{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**MODULE IMPORTATION**"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"%%time\nimport matplotlib.pyplot as plt\n\nfrom tqdm import tqdm_notebook, tqdm_gui\n\nfrom catboost import CatBoostRegressor\nfrom sklearn.preprocessing import StandardScaler,LabelEncoder\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.svm import NuSVR, SVR\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.kernel_ridge import KernelRidge\nfrom sklearn.model_selection import StratifiedKFold, KFold, RepeatedKFold\npd.options.display.precision = 15\n\n%matplotlib inline\nimport lightgbm as lgb\nimport xgboost as xgb\nimport time\nimport datetime\nimport gc\nimport seaborn as sns\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nfrom scipy.signal import hilbert\nfrom scipy.signal import hann\nfrom scipy.signal import convolve\nfrom scipy import stats\n\nimport tsfresh\nfrom tsfresh import extract_features\nfrom tsfresh import select_features\nfrom tsfresh.utilities.dataframe_functions import impute\n\nprint('MODULES IMPORTED')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**FEATURE CREATION FUNCTIONS**"},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef add_trend_feature(arr, abs_values=False):\n    idx = np.array(range(len(arr)))\n    if abs_values:\n        arr = np.abs(arr)\n    lr = LinearRegression()\n    lr.fit(idx.reshape(-1, 1), arr)\n    return lr.coef_[0]\n\n\ndef calc_change_rate(x):\n    change = (np.diff(x) / x[:-1]).values\n    change = change[np.nonzero(change)[0]]\n    change = change[~np.isnan(change)]\n    change = change[change != -np.inf]\n    change = change[change != np.inf]\n    return np.mean(change)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tsfresh.feature_extraction import ComprehensiveFCParameters, EfficientFCParameters, MinimalFCParameters\nfrom tsfresh.feature_extraction.feature_calculators import *\n \nfeatures = ['abs_energy','absolute_sum_of_changes','count_above_mean','count_below_mean',\n            'first_location_of_maximum', 'first_location_of_minimum','last_location_of_maximum','last_location_of_minimum',\n            'longest_strike_above_mean','longest_strike_below_mean',\n            'mean_change','mean_abs_change','median','mean_second_derivative_central']\n#'autocorrelation','fft_coefficient','index_mass_quantile','number_peaks'\n\nn_peaks = 5\ncross_threshold = 0\n\ndef row_features(seg,x):\n    df_temp=pd.DataFrame(index=[0])\n    \n    df_temp['sum'] = x.sum()\n    df_temp['mean']= x.mean()\n    df_temp['std'] =x.std()\n    df_temp['var'] =x.var()\n    df_temp['max'] =x.max()\n    df_temp['min'] =x.min()\n    \n    \n    df_temp['trend'] = add_trend_feature(x)\n    df_temp['abs_trend'] = add_trend_feature(x, abs_values=True)\n    df_temp['abs_sum'] = np.abs(x).sum()\n    df_temp['abs_mean'] = np.abs(x).mean()\n    df_temp['abs_std'] = np.abs(x).std()\n    \n    df_temp['mad'] =x.mad()\n    df_temp['kurt'] =x.kurt()\n    df_temp['skew'] =x.skew()\n    \n    df_temp['max_to_min'] = x.max() / np.abs(x.min())\n    df_temp['max_to_min_diff'] = x.max() - np.abs(x.min())\n    df_temp['count_big'] = len(x[np.abs(x) > 500])\n    df_temp['sum'] = x.sum()\n    \n    df_temp['mean_change_rate_first_50000'] = calc_change_rate(x[:50000])\n    df_temp['mean_change_rate_last_50000'] = calc_change_rate(x[-50000:])\n    df_temp['mean_change_rate_first_10000'] = calc_change_rate(x[:10000])\n    df_temp['mean_change_rate_last_10000'] = calc_change_rate(x[-10000:])\n        \n    df_temp['q95'] = np.quantile(x, 0.95)\n    df_temp['q99'] = np.quantile(x, 0.99)\n    df_temp['q05'] = np.quantile(x, 0.05)\n    df_temp['q01'] = np.quantile(x, 0.01)\n    \n    df_temp['abs_q95'] = np.quantile(np.abs(x), 0.95)\n    df_temp['abs_q99'] = np.quantile(np.abs(x), 0.99)\n    df_temp['abs_q05'] = np.quantile(np.abs(x), 0.05)\n    df_temp['abs_q01'] = np.quantile(np.abs(x), 0.01)\n    \n    df_temp['std_first_50000'] = x[:50000].std()\n    df_temp['std_last_50000'] = x[-50000:].std()\n    df_temp['std_first_10000'] = x[:10000].std()\n    df_temp['std_last_10000'] = x[-10000:].std()\n    \n    df_temp['avg_first_50000'] = x[:50000].mean()\n    df_temp['avg_last_50000'] = x[-50000:].mean()\n    df_temp['avg_first_10000'] = x[:10000].mean()\n    df_temp['avg_last_10000'] = x[-10000:].mean()\n    \n    df_temp['min_first_50000'] = x[:50000].min()\n    df_temp['min_last_50000'] = x[-50000:].min()\n    df_temp['min_first_10000'] = x[:10000].min()\n    df_temp['min_last_10000'] = x[-10000:].min()\n    \n    df_temp['max_first_50000'] = x[:50000].max()\n    df_temp['max_last_50000'] = x[-50000:].max()\n    df_temp['max_first_10000'] = x[:10000].max()\n    df_temp['max_last_10000'] = x[-10000:].max()\n    \n    z = np.fft.fft(x)\n    realFFT = np.real(z)\n    imagFFT = np.imag(z)\n    \n    df_temp['Real_mean'] = realFFT.mean()\n    df_temp['Real_std'] = realFFT.std()\n    df_temp['Real_max'] = realFFT.max()\n    df_temp['Real_min'] = realFFT.min()\n    df_temp['Imag_mean'] = imagFFT.mean()\n    df_temp['Imag_std'] = imagFFT.std()\n    df_temp['Imag_max'] = imagFFT.max()\n    df_temp['Imag_min'] = imagFFT.min()\n    \n    df_temp['Hilbert_mean'] = np.abs(hilbert(x)).mean()\n    df_temp['Hann_window_mean'] = (convolve(x, hann(150), mode='same') / sum(hann(150))).mean()\n    df_temp['Moving_average_700_mean'] = x.rolling(window=700).mean().mean(skipna=True)\n    ewma = pd.Series.ewm\n    df_temp['exp_Moving_average_300_mean'] = (ewma(x, span=300).mean()).mean(skipna=True)\n    df_temp['exp_Moving_average_3000_mean'] = ewma(x, span=3000).mean().mean(skipna=True)\n    df_temp['exp_Moving_average_30000_mean'] = ewma(x, span=30000).mean().mean(skipna=True)\n    no_of_std = 3\n    df_temp['MA_700MA_std_mean'] = x.rolling(window=700).std().mean()\n    df_temp['MA_700MA_BB_high_mean'] = (df_temp['Moving_average_700_mean'] + no_of_std * df_temp['MA_700MA_std_mean']).mean()\n    df_temp['MA_700MA_BB_low_mean'] = (df_temp['Moving_average_700_mean'] - no_of_std * df_temp['MA_700MA_std_mean']).mean()\n    df_temp['MA_400MA_std_mean'] = x.rolling(window=400).std().mean()\n    df_temp['MA_400MA_BB_high_mean'] = (df_temp['Moving_average_700_mean'] + no_of_std * df_temp['MA_400MA_std_mean']).mean()\n    df_temp['MA_400MA_BB_low_mean'] = (df_temp['Moving_average_700_mean'] - no_of_std * df_temp['MA_400MA_std_mean']).mean()\n    df_temp['MA_1000MA_std_mean'] = x.rolling(window=1000).std().mean()\n    df_temp.drop('Moving_average_700_mean', axis=1, inplace=True)\n    \n    df_temp['iqr'] = np.subtract(*np.percentile(x, [75, 25]))\n    df_temp['q999'] = np.quantile(x,0.999)\n    df_temp['q001'] = np.quantile(x,0.001)\n    df_temp['ave10'] = stats.trim_mean(x, 0.1)\n\n    for windows in [10, 100, 1000]:\n        x_roll_std = x.rolling(windows).std().dropna().values\n        x_roll_mean = x.rolling(windows).mean().dropna().values\n        \n        df_temp['ave_roll_std_' + str(windows)] = x_roll_std.mean()\n        df_temp['std_roll_std_' + str(windows)] = x_roll_std.std()\n        df_temp['max_roll_std_' + str(windows)] = x_roll_std.max()\n        df_temp['min_roll_std_' + str(windows)] = x_roll_std.min()\n        df_temp['q01_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.01)\n        df_temp['q05_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.05)\n        df_temp['q95_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.95)\n        df_temp['q99_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.99)\n        df_temp['av_change_abs_roll_std_' + str(windows)] = np.mean(np.diff(x_roll_std))\n        df_temp['av_change_rate_roll_std_' + str(windows)] = np.mean(np.nonzero((np.diff(x_roll_std) / x_roll_std[:-1]))[0])\n        df_temp['abs_max_roll_std_' + str(windows)] = np.abs(x_roll_std).max()\n        \n        df_temp['ave_roll_mean_' + str(windows)] = x_roll_mean.mean()\n        df_temp['std_roll_mean_' + str(windows)] = x_roll_mean.std()\n        df_temp['max_roll_mean_' + str(windows)] = x_roll_mean.max()\n        df_temp['min_roll_mean_' + str(windows)] = x_roll_mean.min()\n        df_temp['q01_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.01)\n        df_temp['q05_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.05)\n        df_temp['q95_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.95)\n        df_temp['q99_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.99)\n        df_temp['av_change_abs_roll_mean_' + str(windows)] = np.mean(np.diff(x_roll_mean))\n        df_temp['av_change_rate_roll_mean_' + str(windows)] = np.mean(np.nonzero((np.diff(x_roll_mean) / x_roll_mean[:-1]))[0])\n        df_temp['abs_max_roll_mean_' + str(windows)] = np.abs(x_roll_mean).max()\n    \n    for feature in features: #Features from tsfresh\n        possibles = globals().copy()\n        possibles.update(locals())\n        method = possibles.get(feature)\n        if not method:\n             raise NotImplementedError(\"Method %s not implemented\" % feature)\n        df_temp[feature] = method(x)\n        \n    for lag in range (0,150_000,30_000):\n        df_temp['autocorrelation_'+str(lag)] = autocorrelation(x,lag)\n        \n    df_temp['number_peaks'] = number_peaks(x, n_peaks)\n    df_temp['number_crossing_m'] = number_crossing_m(x, cross_threshold)\n    \n    \n    #print(df_temp.head())\n    return(df_temp)\n\ndef feature_creation(pd_frame,type_ft):\n    segments = int(np.floor(pd_frame.shape[0] / rows))\n    df = pd.DataFrame()\n    if type_ft == 'train' :\n        y_last = pd.DataFrame()\n        for segment in range(segments):\n                #print(segment)\n                seg = pd_frame.iloc[segment*rows:segment*rows+rows]\n                #print(\"SEG IS \", seg)\n                x = pd.Series(seg['acoustic_data'].values)\n                y = pd.Series(seg['time_to_failure'].values)\n                y_last_temp = pd.Series(seg['time_to_failure'].values[-1])\n                #print(\"X IS :\",x )  \n                df_temp=row_features(seg,x)\n                df=pd.concat([df,df_temp])\n                y_last=pd.concat([y_last,pd.DataFrame([y_last_temp])])\n\n        #print(df.head())\n        return(df,y_last)\n    elif type_ft == 'test':\n        y_last = pd.DataFrame()\n        for segment in range(segments):\n                #print(segment)\n                seg = pd_frame.iloc[segment*rows:segment*rows+rows]\n                #print(\"SEG IS \", seg)\n                x = pd.Series(seg['acoustic_data'].values)\n                #print(\"X IS :\",x )  \n                df_temp=row_features(seg,x)\n                df=pd.concat([df,df_temp])\n        #print(df.head())\n        return(df,y_last)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**TRAIN FEATURES CREATION**"},{"metadata":{"trusted":true},"cell_type":"code","source":"chunk_nb = 6\nrows = 150_000\ni=0\ndf_features_tr=pd.DataFrame()\ny_train=pd.DataFrame()\nfor train in tqdm_notebook(pd.read_csv('../input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32},chunksize=rows*chunk_nb),total=int(4194/chunk_nb)):  \n    #print(train.head())\n    a = train\n    ft_tr,y_last = feature_creation(train,'train')\n    #print(\"FT_TR HEAD IS :\",ft_tr)\n    print(df_features_tr.shape)\n    df_features_tr = pd.concat([df_features_tr,ft_tr])\n    y_train = pd.concat([y_train,y_last])\ny_train.columns=['time_to_failure']\nprint('TRAINING IMPORTED')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(y_train.head())\nprint(y_train.shape)\nprint('Before Scaling : \\n',df_features_tr.head(7))\nscale = StandardScaler()\nX_train_scaled = pd.DataFrame(scale.fit_transform(df_features_tr),columns=df_features_tr.columns)\nprint('After Scaling : \\n',X_train_scaled.head())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**TEST FEATURES CREATION**"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id')\nX_test = pd.DataFrame(columns=df_features_tr.columns, dtype=np.float64, index=submission.index)\nplt.figure(figsize=(22, 16))\ndf_features_test=pd.DataFrame()\n\nfor i, seg_id in enumerate(tqdm_notebook(X_test.index)):\n    seg = pd.read_csv('../input/test/' + seg_id + '.csv')\n    #x = pd.Series(seg['acoustic_data'].values)\n    ft_test,zero = feature_creation(seg,'test')\n    df_features_test = pd.concat([df_features_test,ft_test])\nprint('TEST IMPORTED')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scaler = StandardScaler()\nscaler.fit(df_features_test)\nX_test_scaled = pd.DataFrame(scaler.transform(df_features_test), columns=df_features_test.columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_scaled.to_csv('train_features.csv', index=False)\ny_train.to_csv('y_train.csv', index=False)\nX_test_scaled.to_csv('test_features.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}