{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nfrom sklearn.model_selection import ShuffleSplit\nfrom sklearn.linear_model import HuberRegressor, LinearRegression\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\nfrom tqdm import tqdm_notebook\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.svm import NuSVR, SVR\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.model_selection import ShuffleSplit\n\nfrom scipy.signal import hilbert\nfrom scipy.signal import hann\nfrom scipy.signal import convolve\nfrom scipy import stats\n\nimport matplotlib.pyplot as plt\nimport gc\nimport dask.dataframe as dd\nimport xgboost as xgb\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64})\n#print(\"train shape\", train.shape)\n#pd.set_option(\"display.precision\", 15)  # show more decimals\n#train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"94c878e444b076ed778305b3e75ae763f18ee7dd"},"cell_type":"code","source":"print(\"train shape\", train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"36511a7f6d48c5e8c0a88ec54cdefd8a2bdf2f72"},"cell_type":"code","source":"pd.set_option(\"display.precision\", 15)  # show more decimals","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"68645334172f79246eb733a2288de99308008774"},"cell_type":"code","source":"#train.head(15)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"683a65954832771449287af5e7e69b5c544345d4"},"cell_type":"code","source":"def chunkize(n_chunks):\n    rolling_mean = []\n    last_time = []\n    init_idx = 0\n    for _ in range(n_chunks):  # 629M / 150k = 4194\n        x = train.iloc[init_idx:init_idx + 150000]\n        last_time.append(x.time_to_failure.values[-1])\n        rolling_mean.append(x.acoustic_data.abs().mean())\n        init_idx += 150000\n\n    rolling_mean = np.array(rolling_mean)\n    last_time = np.array(last_time)\n    \n    return rolling_mean, last_time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b90c646f34be4f6ca7ed5859fcc43bc5b4ca4b72"},"cell_type":"code","source":"#rolling_mean, last_time = chunkize(4194)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b84e49bb57439a35bdc144556e3372f340692237"},"cell_type":"code","source":"#splitter = ShuffleSplit(n_splits=1, test_size=.2, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"82938edfa263ad7970d115fde153efec6bb712a9"},"cell_type":"code","source":"#train_index, test_index = list(splitter.split(rolling_mean, last_time))[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c10429834cf88631cae6a8ae6f7a825e12d6d8d0"},"cell_type":"markdown","source":"rm_train = rolling_mean[train_index]\nrm_test  = rolling_mean[test_index]\nlt_train = last_time[train_index]\nlt_test  = last_time[test_index]"},{"metadata":{"trusted":true,"_uuid":"50a3d16735f0ac8d859f36c6e42adf651fee1cb4"},"cell_type":"code","source":"#lr = LinearRegression()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"996ebfbfb71da5d325dd668aeff9eb413fb422eb"},"cell_type":"code","source":"#lr.fit(rolling_mean.reshape(-1,1), last_time)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2cb1e110b3f8e16b10b341b0177eeadf57657966"},"cell_type":"code","source":"#test_files = list(os.listdir('../input/test'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4fcd89349d3c5e7ea012b6a8ddacb4627f251731"},"cell_type":"code","source":"#test_files","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6bfee9b7cec6b71bf2bcf75ad7e77912acd050bc"},"cell_type":"code","source":"#test_dfs = [pd.read_csv('../input/test/{}'.format(file)) for file in test_files]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"171578b1640c02f40e938119c8436a0a56ebaa61"},"cell_type":"code","source":"#test_means = np.array([df.acoustic_data.mean() for df in test_dfs])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dda053b19b480c8d13e1622b560c6f3bd4940041"},"cell_type":"code","source":"#pred_times = lr.predict(test_means.reshape(-1,1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b3de7da1b3d2f4bb5ce9ec1818bd8c1ecd5a8403"},"cell_type":"code","source":"#output = pd.DataFrame()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d2b47e0fa93365086c889863e9ec994970af1188"},"cell_type":"code","source":"#test_files[0][:-4]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5ef68345a6e3da857362dfcad5921442f0b91826"},"cell_type":"code","source":"#output['seg_id'] = [file[:-4] for file in test_files]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7932ba7f9dee14595da11741628d9e7d4ec81586"},"cell_type":"code","source":"#output['time_to_failure'] = pred_times","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8ade02c52167f33c004ce6cc7a846522c99ef481"},"cell_type":"code","source":"#output.to_csv('output_lr.csv', index=False,header=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"739ec28e3deb9dc4da424be31c46f02b757aa30c"},"cell_type":"markdown","source":"Next steps\n- Look at getting more out of the data\n    - Creating new feature (statistical features such as Mean/Std and other for each chunk as well as for the rolling sum in a chunk)\n    - Learning how to anaylyse signal data (looked at STFT, and the \"analysis function for seismic signal data\", the authors aren't quite sure if signal analysis is useful in prediction (they haven't built a model using it))\n    - Looking at other kernels\n        - https://www.kaggle.com/wimwim/rolling-quantiles\n        - https://www.kaggle.com/tsilveira/time-frequency-analysis-with-stft\n        - https://www.kaggle.com/nikitagribov/analysis-function-for-seismic-signal-data\n        - https://www.kaggle.com/zikazika/useful-new-features-and-a-optimised-model (some good anaylsis)\n        - https://www.kaggle.com/artgor/earthquakes-fe-more-features-and-samples (good one for knowing which features to general, I took the feature engineering code from here)\n- Try other models\n    - Stacking (Michael) - https://www.kaggle.com/ashishpatel26/updated-mix-model-with-mxtend-stacking-regression\n    - RNN - https://www.kaggle.com/mayer79/rnn-starter-for-huge-time-series\n    - Scaling data (?) - Basic Feature Benchmark\n    - Generative model - this is more of a moonshot since I've never worked with these before, but it we can model the signal as a composition of different components (first - a gaussian signal (using gaussian process maybe) such that average increases with time, second - a spike, third - gaussian signal that continues before the earthquake, and we model the time periods of the three components as coming from normal distributions with t3~N(0.36 (or whatever is the average time between spike and earthquake), small sigma3), t2~N(small u2, small sigma2) and t1~(the rest of the signal length, i.e. total signal length - t2 - t3, some sigma1(not sure big or small)), then maybe we can model this signal, and then given test signal segment, we can compare what part of the full signal its most likely to match, and give a corresponding time_to_failure. This is probability a simplistic representation, since there are more patterns in the time-to-failure graph that we can take advantage of as shown in [allunia/Shaking Earth](https://www.kaggle.com/allunia/shaking-earth) kernel.\n "},{"metadata":{"trusted":true,"_uuid":"00e8f8cd3bbda84b6596de6e617925713f06c574"},"cell_type":"code","source":"# Create a training file with simple derived features\nrows = 150_000\nsegments = int(np.floor(629145480 / rows))\n\ndef add_trend_feature(arr, abs_values=False):\n    idx = np.array(range(len(arr)))\n    if abs_values:\n        arr = np.abs(arr)\n    lr = LinearRegression()\n    lr.fit(idx.reshape(-1, 1), arr)\n    return lr.coef_[0]\n\ndef classic_sta_lta(x, length_sta, length_lta):\n    \n    sta = np.cumsum(x ** 2)\n\n    # Convert to float\n    sta = np.require(sta, dtype=np.float)\n\n    # Copy for LTA\n    lta = sta.copy()\n\n    # Compute the STA and the LTA\n    sta[length_sta:] = sta[length_sta:] - sta[:-length_sta]\n    sta /= length_sta\n    lta[length_lta:] = lta[length_lta:] - lta[:-length_lta]\n    lta /= length_lta\n\n    # Pad zeros\n    sta[:length_lta - 1] = 0\n\n    # Avoid division by zero by setting zero values to tiny float\n    dtiny = np.finfo(0.0).tiny\n    idx = lta < dtiny\n    lta[idx] = dtiny\n\n    return sta / lta\n\nX_tr = pd.DataFrame(index=range(segments), dtype=np.float64)\n\ny_tr = pd.DataFrame(index=range(segments), dtype=np.float64, columns=['time_to_failure'])\n\ntotal_mean = train['acoustic_data'].mean()\ntotal_std = train['acoustic_data'].std()\ntotal_max = train['acoustic_data'].max()\ntotal_min = train['acoustic_data'].min()\ntotal_sum = train['acoustic_data'].sum()\ntotal_abs_sum = np.abs(train['acoustic_data']).sum()\n\nfor segment in tqdm_notebook(range(segments)):\n    seg = train.iloc[segment*rows:segment*rows+rows]\n    x = pd.Series(seg['acoustic_data'].values)\n    y = seg['time_to_failure'].values[-1]\n    \n    y_tr.loc[segment, 'time_to_failure'] = y\n    X_tr.loc[segment, 'mean'] = x.mean()\n    X_tr.loc[segment, 'std'] = x.std()\n    X_tr.loc[segment, 'max'] = x.max()\n    X_tr.loc[segment, 'min'] = x.min()\n    \n    \n    X_tr.loc[segment, 'mean_change_abs'] = np.mean(np.diff(x))\n    X_tr.loc[segment, 'mean_change_rate'] = np.mean(np.nonzero((np.diff(x) / x[:-1]))[0])\n    X_tr.loc[segment, 'abs_max'] = np.abs(x).max()\n    X_tr.loc[segment, 'abs_min'] = np.abs(x).min()\n    \n    X_tr.loc[segment, 'std_first_50000'] = x[:50000].std()\n    X_tr.loc[segment, 'std_last_50000'] = x[-50000:].std()\n    X_tr.loc[segment, 'std_first_10000'] = x[:10000].std()\n    X_tr.loc[segment, 'std_last_10000'] = x[-10000:].std()\n    \n    X_tr.loc[segment, 'avg_first_50000'] = x[:50000].mean()\n    X_tr.loc[segment, 'avg_last_50000'] = x[-50000:].mean()\n    X_tr.loc[segment, 'avg_first_10000'] = x[:10000].mean()\n    X_tr.loc[segment, 'avg_last_10000'] = x[-10000:].mean()\n    \n    X_tr.loc[segment, 'min_first_50000'] = x[:50000].min()\n    X_tr.loc[segment, 'min_last_50000'] = x[-50000:].min()\n    X_tr.loc[segment, 'min_first_10000'] = x[:10000].min()\n    X_tr.loc[segment, 'min_last_10000'] = x[-10000:].min()\n    \n    X_tr.loc[segment, 'max_first_50000'] = x[:50000].max()\n    X_tr.loc[segment, 'max_last_50000'] = x[-50000:].max()\n    X_tr.loc[segment, 'max_first_10000'] = x[:10000].max()\n    X_tr.loc[segment, 'max_last_10000'] = x[-10000:].max()\n    \n    X_tr.loc[segment, 'max_to_min'] = x.max() / np.abs(x.min())\n    X_tr.loc[segment, 'max_to_min_diff'] = x.max() - np.abs(x.min())\n    X_tr.loc[segment, 'count_big'] = len(x[np.abs(x) > 500])\n    X_tr.loc[segment, 'sum'] = x.sum()\n    \n    X_tr.loc[segment, 'mean_change_rate_first_50000'] = np.mean(np.nonzero((np.diff(x[:50000]) / x[:50000][:-1]))[0])\n    X_tr.loc[segment, 'mean_change_rate_last_50000'] = np.mean(np.nonzero((np.diff(x[-50000:]) / x[-50000:][:-1]))[0])\n    X_tr.loc[segment, 'mean_change_rate_first_10000'] = np.mean(np.nonzero((np.diff(x[:10000]) / x[:10000][:-1]))[0])\n    X_tr.loc[segment, 'mean_change_rate_last_10000'] = np.mean(np.nonzero((np.diff(x[-10000:]) / x[-10000:][:-1]))[0])\n    \n    X_tr.loc[segment, 'q95'] = np.quantile(x, 0.95)\n    X_tr.loc[segment, 'q99'] = np.quantile(x, 0.99)\n    X_tr.loc[segment, 'q05'] = np.quantile(x, 0.05)\n    X_tr.loc[segment, 'q01'] = np.quantile(x, 0.01)\n    \n    X_tr.loc[segment, 'abs_q95'] = np.quantile(np.abs(x), 0.95)\n    X_tr.loc[segment, 'abs_q99'] = np.quantile(np.abs(x), 0.99)\n    X_tr.loc[segment, 'abs_q05'] = np.quantile(np.abs(x), 0.05)\n    X_tr.loc[segment, 'abs_q01'] = np.quantile(np.abs(x), 0.01)\n    \n    X_tr.loc[segment, 'trend'] = add_trend_feature(x)\n    X_tr.loc[segment, 'abs_trend'] = add_trend_feature(x, abs_values=True)\n    X_tr.loc[segment, 'abs_mean'] = np.abs(x).mean()\n    X_tr.loc[segment, 'abs_std'] = np.abs(x).std()\n    \n    X_tr.loc[segment, 'mad'] = x.mad()\n    X_tr.loc[segment, 'kurt'] = x.kurtosis()\n    X_tr.loc[segment, 'skew'] = x.skew()\n    X_tr.loc[segment, 'med'] = x.median()\n    \n    X_tr.loc[segment, 'Hilbert_mean'] = np.abs(hilbert(x)).mean()\n    X_tr.loc[segment, 'Hann_window_mean'] = (convolve(x, hann(150), mode='same') / sum(hann(150))).mean()\n    X_tr.loc[segment, 'classic_sta_lta1_mean'] = classic_sta_lta(x, 500, 10000).mean()\n    X_tr.loc[segment, 'classic_sta_lta2_mean'] = classic_sta_lta(x, 5000, 100000).mean()\n    X_tr.loc[segment, 'classic_sta_lta3_mean'] = classic_sta_lta(x, 3333, 6666).mean()\n    X_tr.loc[segment, 'classic_sta_lta4_mean'] = classic_sta_lta(x, 10000, 25000).mean()\n    X_tr.loc[segment, 'classic_sta_lta5_mean'] = classic_sta_lta(x, 50, 1000).mean()\n    X_tr.loc[segment, 'classic_sta_lta6_mean'] = classic_sta_lta(x, 100, 5000).mean()\n    X_tr.loc[segment, 'classic_sta_lta7_mean'] = classic_sta_lta(x, 333, 666).mean()\n    X_tr.loc[segment, 'classic_sta_lta8_mean'] = classic_sta_lta(x, 4000, 10000).mean()\n    X_tr.loc[segment, 'Moving_average_700_mean'] = x.rolling(window=700).mean().mean(skipna=True)\n    ewma = pd.Series.ewm\n    X_tr.loc[segment, 'exp_Moving_average_300_mean'] = (ewma(x, span=300).mean()).mean(skipna=True)\n    X_tr.loc[segment, 'exp_Moving_average_3000_mean'] = ewma(x, span=3000).mean().mean(skipna=True)\n    X_tr.loc[segment, 'exp_Moving_average_30000_mean'] = ewma(x, span=6000).mean().mean(skipna=True)\n    no_of_std = 3\n    X_tr.loc[segment, 'MA_700MA_std_mean'] = x.rolling(window=700).std().mean()\n    X_tr.loc[segment,'MA_700MA_BB_high_mean'] = (X_tr.loc[segment, 'Moving_average_700_mean'] + no_of_std * X_tr.loc[segment, 'MA_700MA_std_mean']).mean()\n    X_tr.loc[segment,'MA_700MA_BB_low_mean'] = (X_tr.loc[segment, 'Moving_average_700_mean'] - no_of_std * X_tr.loc[segment, 'MA_700MA_std_mean']).mean()\n    X_tr.loc[segment, 'MA_400MA_std_mean'] = x.rolling(window=400).std().mean()\n    X_tr.loc[segment,'MA_400MA_BB_high_mean'] = (X_tr.loc[segment, 'Moving_average_700_mean'] + no_of_std * X_tr.loc[segment, 'MA_400MA_std_mean']).mean()\n    X_tr.loc[segment,'MA_400MA_BB_low_mean'] = (X_tr.loc[segment, 'Moving_average_700_mean'] - no_of_std * X_tr.loc[segment, 'MA_400MA_std_mean']).mean()\n    X_tr.loc[segment, 'MA_1000MA_std_mean'] = x.rolling(window=1000).std().mean()\n    X_tr.drop('Moving_average_700_mean', axis=1, inplace=True)\n    \n    X_tr.loc[segment, 'iqr'] = np.subtract(*np.percentile(x, [75, 25]))\n    X_tr.loc[segment, 'q999'] = np.quantile(x,0.999)\n    X_tr.loc[segment, 'q001'] = np.quantile(x,0.001)\n    X_tr.loc[segment, 'ave10'] = stats.trim_mean(x, 0.1)\n\n    for windows in [10, 100, 1000]:\n        x_roll_std = x.rolling(windows).std().dropna().values\n        x_roll_mean = x.rolling(windows).mean().dropna().values\n        \n        X_tr.loc[segment, 'ave_roll_std_' + str(windows)] = x_roll_std.mean()\n        X_tr.loc[segment, 'std_roll_std_' + str(windows)] = x_roll_std.std()\n        X_tr.loc[segment, 'max_roll_std_' + str(windows)] = x_roll_std.max()\n        X_tr.loc[segment, 'min_roll_std_' + str(windows)] = x_roll_std.min()\n        X_tr.loc[segment, 'q01_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.01)\n        X_tr.loc[segment, 'q05_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.05)\n        X_tr.loc[segment, 'q95_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.95)\n        X_tr.loc[segment, 'q99_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.99)\n        X_tr.loc[segment, 'av_change_abs_roll_std_' + str(windows)] = np.mean(np.diff(x_roll_std))\n        X_tr.loc[segment, 'av_change_rate_roll_std_' + str(windows)] = np.mean(np.nonzero((np.diff(x_roll_std) / x_roll_std[:-1]))[0])\n        X_tr.loc[segment, 'abs_max_roll_std_' + str(windows)] = np.abs(x_roll_std).max()\n        \n        X_tr.loc[segment, 'ave_roll_mean_' + str(windows)] = x_roll_mean.mean()\n        X_tr.loc[segment, 'std_roll_mean_' + str(windows)] = x_roll_mean.std()\n        X_tr.loc[segment, 'max_roll_mean_' + str(windows)] = x_roll_mean.max()\n        X_tr.loc[segment, 'min_roll_mean_' + str(windows)] = x_roll_mean.min()\n        X_tr.loc[segment, 'q01_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.01)\n        X_tr.loc[segment, 'q05_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.05)\n        X_tr.loc[segment, 'q95_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.95)\n        X_tr.loc[segment, 'q99_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.99)\n        X_tr.loc[segment, 'av_change_abs_roll_mean_' + str(windows)] = np.mean(np.diff(x_roll_mean))\n        X_tr.loc[segment, 'av_change_rate_roll_mean_' + str(windows)] = np.mean(np.nonzero((np.diff(x_roll_mean) / x_roll_mean[:-1]))[0])\n        X_tr.loc[segment, 'abs_max_roll_mean_' + str(windows)] = np.abs(x_roll_mean).max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"00b3948bdbfb21820e5bd54eb562ef6429140633"},"cell_type":"code","source":"print(f'{X_tr.shape[0]} samples in new train data and {X_tr.shape[1]} columns.')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"80234ee0a886b25f62454cdb2452a2434ade0f98"},"cell_type":"code","source":"# fillna in new columns\nclassic_sta_lta5_mean_fill = X_tr.loc[X_tr['classic_sta_lta5_mean'] != -np.inf, 'classic_sta_lta5_mean'].mean()\nX_tr.loc[X_tr['classic_sta_lta5_mean'] == -np.inf, 'classic_sta_lta5_mean'] = classic_sta_lta5_mean_fill\nX_tr['classic_sta_lta5_mean'] = X_tr['classic_sta_lta5_mean'].fillna(classic_sta_lta5_mean_fill)\nclassic_sta_lta7_mean_fill = X_tr.loc[X_tr['classic_sta_lta7_mean'] != -np.inf, 'classic_sta_lta7_mean'].mean()\nX_tr.loc[X_tr['classic_sta_lta7_mean'] == -np.inf, 'classic_sta_lta7_mean'] = classic_sta_lta7_mean_fill\nX_tr['classic_sta_lta7_mean'] = X_tr['classic_sta_lta7_mean'].fillna(classic_sta_lta7_mean_fill)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fa0e4cf3f3b03a2a60705f09fb6f461b01fa1ce6"},"cell_type":"code","source":"scaler = StandardScaler()\nscaler.fit(X_tr)\nX_train_scaled = pd.DataFrame(scaler.transform(X_tr), columns=X_tr.columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7ca32486c75e1ae07e8dea4228a88135c9d9e7f2"},"cell_type":"code","source":"del train\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5fced4f4372765646f5a6cf6f8dd2aa7eb21cba3","scrolled":false},"cell_type":"code","source":"submission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id')\nX_test = pd.DataFrame(columns=X_tr.columns, dtype=np.float64, index=submission.index)\nplt.figure(figsize=(22, 16))\n\nfor i, seg_id in enumerate(tqdm_notebook(X_test.index)):\n    seg = pd.read_csv('../input/test/' + seg_id + '.csv')\n    \n    x = pd.Series(seg['acoustic_data'].values)\n    X_test.loc[seg_id, 'mean'] = x.mean()\n    X_test.loc[seg_id, 'std'] = x.std()\n    X_test.loc[seg_id, 'max'] = x.max()\n    X_test.loc[seg_id, 'min'] = x.min()\n        \n    X_test.loc[seg_id, 'mean_change_abs'] = np.mean(np.diff(x))\n    X_test.loc[seg_id, 'mean_change_rate'] = np.mean(np.nonzero((np.diff(x) / x[:-1]))[0])\n    X_test.loc[seg_id, 'abs_max'] = np.abs(x).max()\n    X_test.loc[seg_id, 'abs_min'] = np.abs(x).min()\n    \n    X_test.loc[seg_id, 'std_first_50000'] = x[:50000].std()\n    X_test.loc[seg_id, 'std_last_50000'] = x[-50000:].std()\n    X_test.loc[seg_id, 'std_first_10000'] = x[:10000].std()\n    X_test.loc[seg_id, 'std_last_10000'] = x[-10000:].std()\n    \n    X_test.loc[seg_id, 'avg_first_50000'] = x[:50000].mean()\n    X_test.loc[seg_id, 'avg_last_50000'] = x[-50000:].mean()\n    X_test.loc[seg_id, 'avg_first_10000'] = x[:10000].mean()\n    X_test.loc[seg_id, 'avg_last_10000'] = x[-10000:].mean()\n    \n    X_test.loc[seg_id, 'min_first_50000'] = x[:50000].min()\n    X_test.loc[seg_id, 'min_last_50000'] = x[-50000:].min()\n    X_test.loc[seg_id, 'min_first_10000'] = x[:10000].min()\n    X_test.loc[seg_id, 'min_last_10000'] = x[-10000:].min()\n    \n    X_test.loc[seg_id, 'max_first_50000'] = x[:50000].max()\n    X_test.loc[seg_id, 'max_last_50000'] = x[-50000:].max()\n    X_test.loc[seg_id, 'max_first_10000'] = x[:10000].max()\n    X_test.loc[seg_id, 'max_last_10000'] = x[-10000:].max()\n    \n    X_test.loc[seg_id, 'max_to_min'] = x.max() / np.abs(x.min())\n    X_test.loc[seg_id, 'max_to_min_diff'] = x.max() - np.abs(x.min())\n    X_test.loc[seg_id, 'count_big'] = len(x[np.abs(x) > 500])\n    X_test.loc[seg_id, 'sum'] = x.sum()\n    \n    X_test.loc[seg_id, 'mean_change_rate_first_50000'] = np.mean(np.nonzero((np.diff(x[:50000]) / x[:50000][:-1]))[0])\n    X_test.loc[seg_id, 'mean_change_rate_last_50000'] = np.mean(np.nonzero((np.diff(x[-50000:]) / x[-50000:][:-1]))[0])\n    X_test.loc[seg_id, 'mean_change_rate_first_10000'] = np.mean(np.nonzero((np.diff(x[:10000]) / x[:10000][:-1]))[0])\n    X_test.loc[seg_id, 'mean_change_rate_last_10000'] = np.mean(np.nonzero((np.diff(x[-10000:]) / x[-10000:][:-1]))[0])\n    \n    X_test.loc[seg_id, 'q95'] = np.quantile(x,0.95)\n    X_test.loc[seg_id, 'q99'] = np.quantile(x,0.99)\n    X_test.loc[seg_id, 'q05'] = np.quantile(x,0.05)\n    X_test.loc[seg_id, 'q01'] = np.quantile(x,0.01)\n    \n    X_test.loc[seg_id, 'abs_q95'] = np.quantile(np.abs(x), 0.95)\n    X_test.loc[seg_id, 'abs_q99'] = np.quantile(np.abs(x), 0.99)\n    X_test.loc[seg_id, 'abs_q05'] = np.quantile(np.abs(x), 0.05)\n    X_test.loc[seg_id, 'abs_q01'] = np.quantile(np.abs(x), 0.01)\n    \n    X_test.loc[seg_id, 'trend'] = add_trend_feature(x)\n    X_test.loc[seg_id, 'abs_trend'] = add_trend_feature(x, abs_values=True)\n    X_test.loc[seg_id, 'abs_mean'] = np.abs(x).mean()\n    X_test.loc[seg_id, 'abs_std'] = np.abs(x).std()\n    \n    X_test.loc[seg_id, 'mad'] = x.mad()\n    X_test.loc[seg_id, 'kurt'] = x.kurtosis()\n    X_test.loc[seg_id, 'skew'] = x.skew()\n    X_test.loc[seg_id, 'med'] = x.median()\n    \n    X_test.loc[seg_id, 'Hilbert_mean'] = np.abs(hilbert(x)).mean()\n    X_test.loc[seg_id, 'Hann_window_mean'] = (convolve(x, hann(150), mode='same') / sum(hann(150))).mean()\n    X_test.loc[seg_id, 'classic_sta_lta1_mean'] = classic_sta_lta(x, 500, 10000).mean()\n    X_test.loc[seg_id, 'classic_sta_lta2_mean'] = classic_sta_lta(x, 5000, 100000).mean()\n    X_test.loc[seg_id, 'classic_sta_lta3_mean'] = classic_sta_lta(x, 3333, 6666).mean()\n    X_test.loc[seg_id, 'classic_sta_lta4_mean'] = classic_sta_lta(x, 10000, 25000).mean()\n    X_test.loc[seg_id, 'classic_sta_lta5_mean'] = classic_sta_lta(x, 50, 1000).mean()\n    X_test.loc[seg_id, 'classic_sta_lta6_mean'] = classic_sta_lta(x, 100, 5000).mean()\n    X_test.loc[seg_id, 'classic_sta_lta7_mean'] = classic_sta_lta(x, 333, 666).mean()\n    X_test.loc[seg_id, 'classic_sta_lta8_mean'] = classic_sta_lta(x, 4000, 10000).mean()\n    X_test.loc[seg_id, 'Moving_average_700_mean'] = x.rolling(window=700).mean().mean(skipna=True)\n    ewma = pd.Series.ewm\n    X_test.loc[seg_id, 'exp_Moving_average_300_mean'] = (ewma(x, span=300).mean()).mean(skipna=True)\n    X_test.loc[seg_id, 'exp_Moving_average_3000_mean'] = ewma(x, span=3000).mean().mean(skipna=True)\n    X_test.loc[seg_id, 'exp_Moving_average_30000_mean'] = ewma(x, span=6000).mean().mean(skipna=True)\n    no_of_std = 3\n    X_test.loc[seg_id, 'MA_700MA_std_mean'] = x.rolling(window=700).std().mean()\n    X_test.loc[seg_id,'MA_700MA_BB_high_mean'] = (X_test.loc[seg_id, 'Moving_average_700_mean'] + no_of_std * X_test.loc[seg_id, 'MA_700MA_std_mean']).mean()\n    X_test.loc[seg_id,'MA_700MA_BB_low_mean'] = (X_test.loc[seg_id, 'Moving_average_700_mean'] - no_of_std * X_test.loc[seg_id, 'MA_700MA_std_mean']).mean()\n    X_test.loc[seg_id, 'MA_400MA_std_mean'] = x.rolling(window=400).std().mean()\n    X_test.loc[seg_id,'MA_400MA_BB_high_mean'] = (X_test.loc[seg_id, 'Moving_average_700_mean'] + no_of_std * X_test.loc[seg_id, 'MA_400MA_std_mean']).mean()\n    X_test.loc[seg_id,'MA_400MA_BB_low_mean'] = (X_test.loc[seg_id, 'Moving_average_700_mean'] - no_of_std * X_test.loc[seg_id, 'MA_400MA_std_mean']).mean()\n    X_test.loc[seg_id, 'MA_1000MA_std_mean'] = x.rolling(window=1000).std().mean()\n    X_test.drop('Moving_average_700_mean', axis=1, inplace=True)\n    \n    X_test.loc[seg_id, 'iqr'] = np.subtract(*np.percentile(x, [75, 25]))\n    X_test.loc[seg_id, 'q999'] = np.quantile(x,0.999)\n    X_test.loc[seg_id, 'q001'] = np.quantile(x,0.001)\n    X_test.loc[seg_id, 'ave10'] = stats.trim_mean(x, 0.1)\n    \n    for windows in [10, 100, 1000]:\n        x_roll_std = x.rolling(windows).std().dropna().values\n        x_roll_mean = x.rolling(windows).mean().dropna().values\n        \n        X_test.loc[seg_id, 'ave_roll_std_' + str(windows)] = x_roll_std.mean()\n        X_test.loc[seg_id, 'std_roll_std_' + str(windows)] = x_roll_std.std()\n        X_test.loc[seg_id, 'max_roll_std_' + str(windows)] = x_roll_std.max()\n        X_test.loc[seg_id, 'min_roll_std_' + str(windows)] = x_roll_std.min()\n        X_test.loc[seg_id, 'q01_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.01)\n        X_test.loc[seg_id, 'q05_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.05)\n        X_test.loc[seg_id, 'q95_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.95)\n        X_test.loc[seg_id, 'q99_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.99)\n        X_test.loc[seg_id, 'av_change_abs_roll_std_' + str(windows)] = np.mean(np.diff(x_roll_std))\n        X_test.loc[seg_id, 'av_change_rate_roll_std_' + str(windows)] = np.mean(np.nonzero((np.diff(x_roll_std) / x_roll_std[:-1]))[0])\n        X_test.loc[seg_id, 'abs_max_roll_std_' + str(windows)] = np.abs(x_roll_std).max()\n        \n        X_test.loc[seg_id, 'ave_roll_mean_' + str(windows)] = x_roll_mean.mean()\n        X_test.loc[seg_id, 'std_roll_mean_' + str(windows)] = x_roll_mean.std()\n        X_test.loc[seg_id, 'max_roll_mean_' + str(windows)] = x_roll_mean.max()\n        X_test.loc[seg_id, 'min_roll_mean_' + str(windows)] = x_roll_mean.min()\n        X_test.loc[seg_id, 'q01_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.01)\n        X_test.loc[seg_id, 'q05_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.05)\n        X_test.loc[seg_id, 'q95_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.95)\n        X_test.loc[seg_id, 'q99_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.99)\n        X_test.loc[seg_id, 'av_change_abs_roll_mean_' + str(windows)] = np.mean(np.diff(x_roll_mean))\n        X_test.loc[seg_id, 'av_change_rate_roll_mean_' + str(windows)] = np.mean(np.nonzero((np.diff(x_roll_mean) / x_roll_mean[:-1]))[0])\n        X_test.loc[seg_id, 'abs_max_roll_mean_' + str(windows)] = np.abs(x_roll_mean).max()\n    \n    if i < 12:\n        plt.subplot(6, 4, i + 1)\n        plt.plot(seg['acoustic_data'])\n        plt.title(seg_id)\n\n# fillna in new columns\nX_test.loc[X_test['classic_sta_lta5_mean'] == -np.inf, 'classic_sta_lta5_mean'] = classic_sta_lta5_mean_fill\nX_test.loc[X_test['classic_sta_lta7_mean'] == -np.inf, 'classic_sta_lta7_mean'] = classic_sta_lta7_mean_fill\nX_test_scaled = pd.DataFrame(scaler.transform(X_test), columns=X_test.columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6bf89b266924f7e032f1201e2ebadd4e1e71f293"},"cell_type":"code","source":"params = {'eta': 0.05,\n              'max_depth': 10,\n              'subsample': 0.9,\n              'objective': 'reg:linear',\n              'eval_metric': 'mae',\n              'silent': True,\n              'nthread': 4}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"44c2804ede4bcd750c1aacf1ccf9e306884fc50c"},"cell_type":"code","source":"X=X_train_scaled\nX_test=X_test_scaled","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"861eec1277bb09f77ad0e24a8d6abe7e78b1b550"},"cell_type":"code","source":"splitter = ShuffleSplit(n_splits=1, test_size=.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"108412d6de86aa3cc2d93ddb6330277a5609c7d8"},"cell_type":"code","source":"train_index, valid_index = list(splitter.split(X))[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dba861910ee4ed2ff9cda0b73a1262f1986a1860"},"cell_type":"code","source":"y = y_tr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8ac6299d6218f1f0529907c23bab96ecf4961d9a"},"cell_type":"code","source":"X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]\ny_train, y_valid = y.iloc[train_index], y.iloc[valid_index]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bcfbd59ef3ee51995822c907cd7406e97497d49e"},"cell_type":"code","source":"train_data = xgb.DMatrix(data=X_train, label=y_train, feature_names=X.columns)\nvalid_data = xgb.DMatrix(data=X_valid, label=y_valid, feature_names=X.columns)\n\nwatchlist = [(train_data, 'train'), (valid_data, 'valid_data')]\nmodel = xgb.train(dtrain=train_data, num_boost_round=20000, evals=watchlist, early_stopping_rounds=200, verbose_eval=500, params=params)\ny_pred_valid = model.predict(xgb.DMatrix(X_valid, feature_names=X.columns), ntree_limit=model.best_ntree_limit)\ny_pred = model.predict(xgb.DMatrix(X_test, feature_names=X.columns), ntree_limit=model.best_ntree_limit)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1df35655078e24ffa4964d65b78c0a669fa5e392"},"cell_type":"code","source":"mean_absolute_error(y_valid, y_pred_valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f62170f5ed889f94df9feb8a2cd2fe22ca194dc"},"cell_type":"code","source":"submission['time_to_failure'] = y_pred\n# submission['time_to_failure'] = prediction_lgb_stack\nprint(submission.head())\nsubmission.to_csv('submission_feats_xgb.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"783e2902255afa278e83858b1c2ef57ceede39f7"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}