{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":27,"outputs":[{"output_type":"stream","text":"['test', 'train.csv', 'sample_submission.csv']\n","name":"stdout"}]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import time\nimport datetime\nimport warnings\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport xgboost as xgb\nimport lightgbm as lgb\nfrom tqdm import tqdm\nimport scipy as sp\nfrom sklearn.svm import NuSVR, SVR\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.linear_model import LassoCV, RidgeCV\nfrom sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.preprocessing import RobustScaler\nfrom sklearn.model_selection import KFold, cross_val_score\nwarnings.filterwarnings(\"ignore\")\nwarnings.simplefilter(action='ignore', category=FutureWarning)\nfrom numpy import inf\npd.options.display.precision = 15","execution_count":28,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport gc\nfrom joblib import Parallel, delayed","execution_count":29,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def classic_sta_lta(x, Ns, Nl):\n sta = np.cumsum(x ** 2)\n sta = np.require(sta, dtype=np.float)\n lta = sta.copy()\n lta[Nl:-Ns] = lta[Nl:-Ns] - lta[:-Nl-Ns]\n lta /= Nl\n sta[Nl+Ns-1:] = sta[Nl+Ns-1:] - sta[Nl-1:-Ns]\n sta /= Ns\n sta[:Nl - 1 + Ns] = 0\n dtiny = np.finfo(0.0).tiny\n idx = lta < dtiny\n lta[idx] = dtiny\n return sta / lta","execution_count":30,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Taken from [Abhishek's Kernel](https://www.kaggle.com/abhishek/quite-a-few-features-1-51). Features used here are different though."},{"metadata":{"trusted":true},"cell_type":"code","source":"class FeatureGenerator(object):\n def __init__(self, dtype, n_jobs=1, chunk_size=None):\n self.chunk_size = chunk_size\n self.dtype = dtype\n self.filename = None\n self.n_jobs = n_jobs\n self.test_files = []\n if self.dtype == 'train':\n self.filename = '../input/train.csv'\n self.total_data = int(629145481 / self.chunk_size)\n else:\n submission = pd.read_csv('../input/sample_submission.csv')\n for seg_id in submission.seg_id.values:\n self.test_files.append((seg_id, '../input/test/' + seg_id + '.csv'))\n self.total_data = int(len(submission))\n\n def read_chunks(self):\n if self.dtype == 'train':\n iter_df = pd.read_csv(self.filename, iterator=True, chunksize=self.chunk_size,\n dtype={'acoustic_data': np.float64, 'time_to_failure': np.float64})\n for counter, df in enumerate(iter_df):\n x = df.acoustic_data.values\n y = df.time_to_failure.values[-1]\n seg_id = 'train_' + str(counter)\n del df\n yield seg_id, x, y\n else:\n for seg_id, f in self.test_files:\n df = pd.read_csv(f, dtype={'acoustic_data': np.float64})\n x = df.acoustic_data.values[-self.chunk_size:]\n del df\n yield seg_id, x, -999\n\n def features(self, x, y, seg_id):\n feature_dict = dict()\n feature_dict['target'] = y\n feature_dict['seg_id'] = seg_id\n\n # create features here\n feature_dict['mean'] = np.mean(x)\n feature_dict['max'] = np.max(x)\n feature_dict['min'] = np.min(x)\n feature_dict['std'] = np.std(x)\n feature_dict['var'] = np.var(x)\n feature_dict['quantile_03'] = np.quantile(x, 0.03)\n feature_dict['skew'] = sp.stats.skew(x)\n feature_dict['kurtosis'] = sp.stats.kurtosis(x)\n feature_dict['moment_3'] = sp.stats.moment(x, 3)\n \n pct_change = pd.Series(x).pct_change()\n pct_change[pct_change == -inf] = 0\n pct_change[pct_change == inf] = 0\n feature_dict['pct_change_mean'] = pct_change.mean()\n rate_change = pd.Series(x).pct_change().pct_change()\n rate_change[rate_change == -inf] = 0\n rate_change[rate_change == inf] = 0\n feature_dict['rate_change_max'] = rate_change.max()\n feature_dict['rate_change_mean'] = rate_change.mean()\n feature_dict['classic_sta_lta_mean'] = classic_sta_lta(x, 100, 5000).mean()\n \n window_size = 10\n x_roll_std = pd.Series(x).rolling(window_size).std().dropna().values\n feature_dict['q03_roll_std_' + str(window_size)] = np.quantile(x_roll_std, 0.03)\n window_size = 150\n x_roll_std = pd.Series(x).rolling(window_size).std().dropna().values\n feature_dict['q03_roll_std_' + str(window_size)] = np.quantile(x_roll_std, 0.03)\n \n return feature_dict\n \n def generate(self):\n feature_list = []\n res = Parallel(n_jobs=self.n_jobs,\n backend='threading')(delayed(self.features)(x, y, s)\n for s, x, y in tqdm(self.read_chunks(), total=self.total_data))\n for r in res:\n feature_list.append(r)\n return pd.DataFrame(feature_list)\n \ntraining_fg = FeatureGenerator(dtype='train', n_jobs=10, chunk_size=150000)\ntraining_data = training_fg.generate()\n\ntest_fg = FeatureGenerator(dtype='test', n_jobs=10, chunk_size=150000)\ntest_data = test_fg.generate()\n ","execution_count":31,"outputs":[{"output_type":"stream","text":"4195it [03:47, 18.44it/s] \n100%|██████████| 2624/2624 [02:06<00:00, 20.66it/s]\n","name":"stderr"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = training_data.drop(['target', 'seg_id'], axis=1)\nX_test = test_data.drop(['target', 'seg_id'], axis=1)\ntest_segs = test_data.seg_id\ny = training_data.target","execution_count":32,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"folds = KFold(n_splits=5, shuffle=True, random_state=42)\noof_preds = np.zeros((len(X), 1))\ntest_preds = np.zeros((len(X_test), 1))","execution_count":33,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"params = {\n \"learning_rate\": 0.01,\n \"max_depth\": 3,\n \"n_estimators\": 10000,\n \"min_child_weight\": 4,\n \"colsample_bytree\": 1,\n \"subsample\": 0.9,\n \"nthread\": 12,\n \"random_state\": 42\n}","execution_count":34,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for fold_, (trn_, val_) in enumerate(folds.split(X)):\n print(\"Current Fold: {}\".format(fold_))\n trn_x, trn_y = X.iloc[trn_], y.iloc[trn_]\n val_x, val_y = X.iloc[val_], y.iloc[val_]\n\n clf = xgb.XGBRegressor(**params)\n clf.fit(\n trn_x, trn_y,\n eval_set=[(trn_x, trn_y), (val_x, val_y)],\n eval_metric='mae',\n verbose=150,\n early_stopping_rounds=100\n )\n val_pred = clf.predict(val_x, ntree_limit=clf.best_ntree_limit)\n test_fold_pred = clf.predict(X_test, ntree_limit=clf.best_ntree_limit)\n print(\"MAE = {}\".format(mean_absolute_error(val_y, val_pred)))\n oof_preds[val_, :] = val_pred.reshape((-1, 1))\n test_preds += test_fold_pred.reshape((-1, 1))\ntest_preds /= 5\n\noof_score = mean_absolute_error(y, oof_preds)\nprint(\"Mean MAE = {}\".format(oof_score))","execution_count":35,"outputs":[{"output_type":"stream","text":"Current Fold: 0\n[0]\tvalidation_0-mae:5.16502\tvalidation_1-mae:5.14112\nMultiple eval metrics have been passed: 'validation_1-mae' will be used for early stopping.\n\nWill train until validation_1-mae hasn't improved in 100 rounds.\n[150]\tvalidation_0-mae:2.13414\tvalidation_1-mae:2.19956\n[300]\tvalidation_0-mae:1.96297\tvalidation_1-mae:2.09104\nStopping. Best iteration:\n[270]\tvalidation_0-mae:1.97196\tvalidation_1-mae:2.08829\n\nMAE = 2.0882864178712217\nCurrent Fold: 1\n[0]\tvalidation_0-mae:5.16701\tvalidation_1-mae:5.13132\nMultiple eval metrics have been passed: 'validation_1-mae' will be used for early stopping.\n\nWill train until validation_1-mae hasn't improved in 100 rounds.\n[150]\tvalidation_0-mae:2.14262\tvalidation_1-mae:2.16153\n[300]\tvalidation_0-mae:1.97079\tvalidation_1-mae:2.03739\nStopping. Best iteration:\n[268]\tvalidation_0-mae:1.97963\tvalidation_1-mae:2.03706\n\nMAE = 2.03706235715146\nCurrent Fold: 2\n[0]\tvalidation_0-mae:5.20054\tvalidation_1-mae:4.99898\nMultiple eval metrics have been passed: 'validation_1-mae' will be used for early stopping.\n\nWill train until validation_1-mae hasn't improved in 100 rounds.\n[150]\tvalidation_0-mae:2.14608\tvalidation_1-mae:2.13386\n[300]\tvalidation_0-mae:1.98032\tvalidation_1-mae:2.01795\nStopping. Best iteration:\n[296]\tvalidation_0-mae:1.98126\tvalidation_1-mae:2.0172\n\nMAE = 2.0171964087101815\nCurrent Fold: 3\n[0]\tvalidation_0-mae:5.12031\tvalidation_1-mae:5.31888\nMultiple eval metrics have been passed: 'validation_1-mae' will be used for early stopping.\n\nWill train until validation_1-mae hasn't improved in 100 rounds.\n[150]\tvalidation_0-mae:2.14373\tvalidation_1-mae:2.14109\n[300]\tvalidation_0-mae:1.97747\tvalidation_1-mae:2.00758\nStopping. Best iteration:\n[347]\tvalidation_0-mae:1.96757\tvalidation_1-mae:2.00578\n\nMAE = 2.0057769817779563\nCurrent Fold: 4\n[0]\tvalidation_0-mae:5.1477\tvalidation_1-mae:5.21121\nMultiple eval metrics have been passed: 'validation_1-mae' will be used for early stopping.\n\nWill train until validation_1-mae hasn't improved in 100 rounds.\n[150]\tvalidation_0-mae:2.12839\tvalidation_1-mae:2.23443\n[300]\tvalidation_0-mae:1.96446\tvalidation_1-mae:2.06745\n[450]\tvalidation_0-mae:1.9365\tvalidation_1-mae:2.0676\nStopping. Best iteration:\n[387]\tvalidation_0-mae:1.94744\tvalidation_1-mae:2.06499\n\nMAE = 2.064994124860618\nMean MAE = 2.042663258074287\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(clf.feature_importances_)","execution_count":36,"outputs":[{"output_type":"stream","text":"[0.01522332 0.01312999 0.01295322 0.0183396 0.0111583 0.01447534\n 0.01787995 0.6591948 0.15391296 0.01395802 0.01176884 0.02993136\n 0.01381253 0.01426176 0. ]\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(training_data.columns)","execution_count":37,"outputs":[{"output_type":"stream","text":"Index(['classic_sta_lta_mean', 'kurtosis', 'max', 'mean', 'min', 'moment_3',\n 'pct_change_mean', 'q03_roll_std_10', 'q03_roll_std_150', 'quantile_03',\n 'rate_change_max', 'rate_change_mean', 'seg_id', 'skew', 'std',\n 'target', 'var'],\n dtype='object')\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature_importance = pd.concat([pd.Series(list(set(list(training_data)) - set(['seg_id', 'target']))), pd.Series(clf.feature_importances_)], axis = 1, keys = ['feature', 'importance'])","execution_count":38,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature_importance.sort_values(by = ['importance'], ascending = False, inplace = True)","execution_count":39,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature_importance","execution_count":40,"outputs":[{"output_type":"execute_result","execution_count":40,"data":{"text/plain":" feature importance\n7 min 0.659194827079773\n8 var 0.153912961483002\n11 max 0.029931357130408\n3 q03_roll_std_10 0.018339600414038\n6 rate_change_max 0.017879946157336\n0 quantile_03 0.015223322436213\n5 q03_roll_std_150 0.014475340023637\n13 std 0.014261762611568\n9 rate_change_mean 0.013958019204438\n12 moment_3 0.013812527060509\n1 classic_sta_lta_mean 0.013129988685250\n2 pct_change_mean 0.012953219935298\n10 skew 0.011768844909966\n4 mean 0.011158297769725\n14 kurtosis 0.000000000000000","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
featureimportance
7min0.659194827079773
8var0.153912961483002
11max0.029931357130408
3q03_roll_std_100.018339600414038
6rate_change_max0.017879946157336
0quantile_030.015223322436213
5q03_roll_std_1500.014475340023637
13std0.014261762611568
9rate_change_mean0.013958019204438
12moment_30.013812527060509
1classic_sta_lta_mean0.013129988685250
2pct_change_mean0.012953219935298
10skew0.011768844909966
4mean0.011158297769725
14kurtosis0.000000000000000
\n
"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame(columns=['seg_id', 'time_to_failure'])\nsubmission.seg_id = test_segs\nsubmission.time_to_failure = test_preds\nsubmission.to_csv('submission.csv', index=False)","execution_count":41,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}