{"cells":[{"metadata":{},"cell_type":"markdown","source":"This is a part of a series of copy or reproduction of below kernel  \nRef: https://www.kaggle.com/vettejeep/masters-final-project-model-lb-1-392  \n\nV1: First  \nV2: Fix incorrect train data format  \nV3: Remove seg_id.sed_st.seg_end column from scaled_  \nV4: Public  "},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport time\nimport warnings\nimport traceback\nimport numpy as np\nimport pandas as pd\nfrom scipy import stats\nimport scipy.signal as sg\nimport multiprocessing as mp\nfrom scipy.signal import hann\nfrom scipy.signal import hilbert\nfrom scipy.signal import convolve\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.preprocessing import StandardScaler\n\nimport xgboost as xgb\nimport lightgbm as lgb\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.metrics import mean_absolute_error\n\nfrom tqdm import tqdm_notebook\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(os.listdir(\"../input\"))\nprint(os.listdir(\"../input/lanl-vettejeep-createfeature-1\"))\nprint(os.listdir(\"../input/lanl-vettejeep-createfeature-2\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"OUTPUT_DIR = ''\nDATA_DIR = '../input/LANL-Earthquake-Prediction'\n\nSIG_LEN = 150000\nNUM_SEG_PER_PROC = 4000\nNUM_THREADS = 6\n\nNY_FREQ_IDX = 75000  # the test signals are 150k samples long, Nyquist is thus 75k.\nCUTOFF = 18000\nMAX_FREQ_IDX = 20000\nFREQ_STEP = 2500","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Join output features"},{"metadata":{"trusted":true},"cell_type":"code","source":"datadir = '../input/lanl-vettejeep-createfeature-1'\ndf0 = pd.read_csv(os.path.join(datadir, 'train_x_%d.csv' % 0))\ndf1 = pd.read_csv(os.path.join(datadir, 'train_x_%d.csv' % 1))\ndf2 = pd.read_csv(os.path.join(datadir, 'train_x_%d.csv' % 2))\ndatadir = '../input/lanl-vettejeep-createfeature-2'\ndf3 = pd.read_csv(os.path.join(datadir, 'train_x_%d.csv' % 3))\ndf4 = pd.read_csv(os.path.join(datadir, 'train_x_%d.csv' % 4))\ndf5 = pd.read_csv(os.path.join(datadir, 'train_x_%d.csv' % 5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.concat([df0,df1,df2,df3,df4,df5], axis=0, ignore_index=True)\nprint(df.shape)\ndisplay(df.head(5))\ndisplay(df.tail(5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.to_csv(os.path.join(OUTPUT_DIR, 'train_x.csv'), index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# For y"},{"metadata":{"trusted":true},"cell_type":"code","source":"datadir = '../input/lanl-vettejeep-createfeature-1'\ndf0 = pd.read_csv(os.path.join(datadir, 'train_y_%d.csv' % 0))\ndf1 = pd.read_csv(os.path.join(datadir, 'train_y_%d.csv' % 1))\ndf2 = pd.read_csv(os.path.join(datadir, 'train_y_%d.csv' % 2))\ndatadir = '../input/lanl-vettejeep-createfeature-2'\ndf3 = pd.read_csv(os.path.join(datadir, 'train_y_%d.csv' % 3))\ndf4 = pd.read_csv(os.path.join(datadir, 'train_y_%d.csv' % 4))\ndf5 = pd.read_csv(os.path.join(datadir, 'train_y_%d.csv' % 5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_y = pd.concat([df0,df1,df2,df3,df4,df5], axis=0, ignore_index=True)\nprint(df_y.shape)\ndisplay(df_y.tail(5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_y.to_csv(os.path.join(OUTPUT_DIR, 'train_y.csv'), index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create features from test data"},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_trend_feature(arr, abs_values=False):\n    idx = np.array(range(len(arr)))\n    if abs_values:\n        arr = np.abs(arr)\n    lr = LinearRegression()\n    lr.fit(idx.reshape(-1, 1), arr)\n    return lr.coef_[0]\n\ndef classic_sta_lta(x, length_sta, length_lta):\n    sta = np.cumsum(x ** 2)\n    # Convert to float\n    sta = np.require(sta, dtype=np.float)\n    # Copy for LTA\n    lta = sta.copy()\n    # Compute the STA and the LTA\n    sta[length_sta:] = sta[length_sta:] - sta[:-length_sta]\n    sta /= length_sta\n    lta[length_lta:] = lta[length_lta:] - lta[:-length_lta]\n    lta /= length_lta\n    # Pad zeros\n    sta[:length_lta - 1] = 0\n    # Avoid division by zero by setting zero values to tiny float\n    dtiny = np.finfo(0.0).tiny\n    idx = lta < dtiny\n    lta[idx] = dtiny\n    return sta / lta","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def des_bw_filter_lp(cutoff=CUTOFF):  # low pass filter\n    b, a = sg.butter(4, Wn=cutoff/NY_FREQ_IDX)\n    return b, a\n\ndef des_bw_filter_hp(cutoff=CUTOFF):  # high pass filter\n    b, a = sg.butter(4, Wn=cutoff/NY_FREQ_IDX, btype='highpass')\n    return b, a\n\ndef des_bw_filter_bp(low, high):  # band pass filter\n    b, a = sg.butter(4, Wn=(low/NY_FREQ_IDX, high/NY_FREQ_IDX), btype='bandpass')\n    return b, a","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Feature Creation Function\ndef create_features(seg_id, seg, X, st, end):\n    try:\n        X.loc[seg_id, 'seg_id'] = np.int32(seg_id)\n        X.loc[seg_id, 'seg_start'] = np.int32(st)\n        X.loc[seg_id, 'seg_end'] = np.int32(end)\n    except:\n        pass\n\n    xc = pd.Series(seg['acoustic_data'].values)\n    xcdm = xc - np.mean(xc)\n\n    b, a = des_bw_filter_lp(cutoff=18000)\n    xcz = sg.lfilter(b, a, xcdm)\n\n    zc = np.fft.fft(xcz)\n    zc = zc[:MAX_FREQ_IDX]\n\n    # FFT transform values\n    realFFT = np.real(zc)\n    imagFFT = np.imag(zc)\n\n    freq_bands = [x for x in range(0, MAX_FREQ_IDX, FREQ_STEP)]\n    magFFT = np.sqrt(realFFT ** 2 + imagFFT ** 2)\n    phzFFT = np.arctan(imagFFT / realFFT)\n    phzFFT[phzFFT == -np.inf] = -np.pi / 2.0\n    phzFFT[phzFFT == np.inf] = np.pi / 2.0\n    phzFFT = np.nan_to_num(phzFFT)\n\n    for freq in freq_bands:\n        X.loc[seg_id, 'FFT_Mag_01q%d' % freq] = np.quantile(magFFT[freq: freq + FREQ_STEP], 0.01)\n        X.loc[seg_id, 'FFT_Mag_10q%d' % freq] = np.quantile(magFFT[freq: freq + FREQ_STEP], 0.1)\n        X.loc[seg_id, 'FFT_Mag_90q%d' % freq] = np.quantile(magFFT[freq: freq + FREQ_STEP], 0.9)\n        X.loc[seg_id, 'FFT_Mag_99q%d' % freq] = np.quantile(magFFT[freq: freq + FREQ_STEP], 0.99)\n        X.loc[seg_id, 'FFT_Mag_mean%d' % freq] = np.mean(magFFT[freq: freq + FREQ_STEP])\n        X.loc[seg_id, 'FFT_Mag_std%d' % freq] = np.std(magFFT[freq: freq + FREQ_STEP])\n        X.loc[seg_id, 'FFT_Mag_max%d' % freq] = np.max(magFFT[freq: freq + FREQ_STEP])\n\n        X.loc[seg_id, 'FFT_Phz_mean%d' % freq] = np.mean(phzFFT[freq: freq + FREQ_STEP])\n        X.loc[seg_id, 'FFT_Phz_std%d' % freq] = np.std(phzFFT[freq: freq + FREQ_STEP])\n\n    X.loc[seg_id, 'FFT_Rmean'] = realFFT.mean()\n    X.loc[seg_id, 'FFT_Rstd'] = realFFT.std()\n    X.loc[seg_id, 'FFT_Rmax'] = realFFT.max()\n    X.loc[seg_id, 'FFT_Rmin'] = realFFT.min()\n    X.loc[seg_id, 'FFT_Imean'] = imagFFT.mean()\n    X.loc[seg_id, 'FFT_Istd'] = imagFFT.std()\n    X.loc[seg_id, 'FFT_Imax'] = imagFFT.max()\n    X.loc[seg_id, 'FFT_Imin'] = imagFFT.min()\n\n    X.loc[seg_id, 'FFT_Rmean_first_6000'] = realFFT[:6000].mean()\n    X.loc[seg_id, 'FFT_Rstd__first_6000'] = realFFT[:6000].std()\n    X.loc[seg_id, 'FFT_Rmax_first_6000'] = realFFT[:6000].max()\n    X.loc[seg_id, 'FFT_Rmin_first_6000'] = realFFT[:6000].min()\n    X.loc[seg_id, 'FFT_Rmean_first_18000'] = realFFT[:18000].mean()\n    X.loc[seg_id, 'FFT_Rstd_first_18000'] = realFFT[:18000].std()\n    X.loc[seg_id, 'FFT_Rmax_first_18000'] = realFFT[:18000].max()\n    X.loc[seg_id, 'FFT_Rmin_first_18000'] = realFFT[:18000].min()\n\n    del xcz\n    del zc\n\n    b, a = des_bw_filter_lp(cutoff=2500)\n    xc0 = sg.lfilter(b, a, xcdm)\n\n    b, a = des_bw_filter_bp(low=2500, high=5000)\n    xc1 = sg.lfilter(b, a, xcdm)\n\n    b, a = des_bw_filter_bp(low=5000, high=7500)\n    xc2 = sg.lfilter(b, a, xcdm)\n\n    b, a = des_bw_filter_bp(low=7500, high=10000)\n    xc3 = sg.lfilter(b, a, xcdm)\n\n    b, a = des_bw_filter_bp(low=10000, high=12500)\n    xc4 = sg.lfilter(b, a, xcdm)\n\n    b, a = des_bw_filter_bp(low=12500, high=15000)\n    xc5 = sg.lfilter(b, a, xcdm)\n\n    b, a = des_bw_filter_bp(low=15000, high=17500)\n    xc6 = sg.lfilter(b, a, xcdm)\n\n    b, a = des_bw_filter_bp(low=17500, high=20000)\n    xc7 = sg.lfilter(b, a, xcdm)\n\n    b, a = des_bw_filter_hp(cutoff=20000)\n    xc8 = sg.lfilter(b, a, xcdm)\n\n    sigs = [xc, pd.Series(xc0), pd.Series(xc1), pd.Series(xc2), pd.Series(xc3),\n            pd.Series(xc4), pd.Series(xc5), pd.Series(xc6), pd.Series(xc7), pd.Series(xc8)]\n\n    for i, sig in enumerate(sigs):\n        X.loc[seg_id, 'mean_%d' % i] = sig.mean()\n        X.loc[seg_id, 'std_%d' % i] = sig.std()\n        X.loc[seg_id, 'max_%d' % i] = sig.max()\n        X.loc[seg_id, 'min_%d' % i] = sig.min()\n\n        X.loc[seg_id, 'mean_change_abs_%d' % i] = np.mean(np.diff(sig))\n        X.loc[seg_id, 'mean_change_rate_%d' % i] = np.mean(np.nonzero((np.diff(sig) / sig[:-1]))[0])\n        X.loc[seg_id, 'abs_max_%d' % i] = np.abs(sig).max()\n        X.loc[seg_id, 'abs_min_%d' % i] = np.abs(sig).min()\n\n        X.loc[seg_id, 'std_first_50000_%d' % i] = sig[:50000].std()\n        X.loc[seg_id, 'std_last_50000_%d' % i] = sig[-50000:].std()\n        X.loc[seg_id, 'std_first_10000_%d' % i] = sig[:10000].std()\n        X.loc[seg_id, 'std_last_10000_%d' % i] = sig[-10000:].std()\n\n        X.loc[seg_id, 'avg_first_50000_%d' % i] = sig[:50000].mean()\n        X.loc[seg_id, 'avg_last_50000_%d' % i] = sig[-50000:].mean()\n        X.loc[seg_id, 'avg_first_10000_%d' % i] = sig[:10000].mean()\n        X.loc[seg_id, 'avg_last_10000_%d' % i] = sig[-10000:].mean()\n\n        X.loc[seg_id, 'min_first_50000_%d' % i] = sig[:50000].min()\n        X.loc[seg_id, 'min_last_50000_%d' % i] = sig[-50000:].min()\n        X.loc[seg_id, 'min_first_10000_%d' % i] = sig[:10000].min()\n        X.loc[seg_id, 'min_last_10000_%d' % i] = sig[-10000:].min()\n\n        X.loc[seg_id, 'max_first_50000_%d' % i] = sig[:50000].max()\n        X.loc[seg_id, 'max_last_50000_%d' % i] = sig[-50000:].max()\n        X.loc[seg_id, 'max_first_10000_%d' % i] = sig[:10000].max()\n        X.loc[seg_id, 'max_last_10000_%d' % i] = sig[-10000:].max()\n\n        X.loc[seg_id, 'max_to_min_%d' % i] = sig.max() / np.abs(sig.min())\n        X.loc[seg_id, 'max_to_min_diff_%d' % i] = sig.max() - np.abs(sig.min())\n        X.loc[seg_id, 'count_big_%d' % i] = len(sig[np.abs(sig) > 500])\n        X.loc[seg_id, 'sum_%d' % i] = sig.sum()\n\n        X.loc[seg_id, 'mean_change_rate_first_50000_%d' % i] = np.mean(np.nonzero((np.diff(sig[:50000]) / sig[:50000][:-1]))[0])\n        X.loc[seg_id, 'mean_change_rate_last_50000_%d' % i] = np.mean(np.nonzero((np.diff(sig[-50000:]) / sig[-50000:][:-1]))[0])\n        X.loc[seg_id, 'mean_change_rate_first_10000_%d' % i] = np.mean(np.nonzero((np.diff(sig[:10000]) / sig[:10000][:-1]))[0])\n        X.loc[seg_id, 'mean_change_rate_last_10000_%d' % i] = np.mean(np.nonzero((np.diff(sig[-10000:]) / sig[-10000:][:-1]))[0])\n\n        X.loc[seg_id, 'q95_%d' % i] = np.quantile(sig, 0.95)\n        X.loc[seg_id, 'q99_%d' % i] = np.quantile(sig, 0.99)\n        X.loc[seg_id, 'q05_%d' % i] = np.quantile(sig, 0.05)\n        X.loc[seg_id, 'q01_%d' % i] = np.quantile(sig, 0.01)\n\n        X.loc[seg_id, 'abs_q95_%d' % i] = np.quantile(np.abs(sig), 0.95)\n        X.loc[seg_id, 'abs_q99_%d' % i] = np.quantile(np.abs(sig), 0.99)\n        X.loc[seg_id, 'abs_q05_%d' % i] = np.quantile(np.abs(sig), 0.05)\n        X.loc[seg_id, 'abs_q01_%d' % i] = np.quantile(np.abs(sig), 0.01)\n\n        X.loc[seg_id, 'trend_%d' % i] = add_trend_feature(sig)\n        X.loc[seg_id, 'abs_trend_%d' % i] = add_trend_feature(sig, abs_values=True)\n        X.loc[seg_id, 'abs_mean_%d' % i] = np.abs(sig).mean()\n        X.loc[seg_id, 'abs_std_%d' % i] = np.abs(sig).std()\n\n        X.loc[seg_id, 'mad_%d' % i] = sig.mad()\n        X.loc[seg_id, 'kurt_%d' % i] = sig.kurtosis()\n        X.loc[seg_id, 'skew_%d' % i] = sig.skew()\n        X.loc[seg_id, 'med_%d' % i] = sig.median()\n\n        X.loc[seg_id, 'Hilbert_mean_%d' % i] = np.abs(hilbert(sig)).mean()\n        X.loc[seg_id, 'Hann_window_mean'] = (convolve(xc, hann(150), mode='same') / sum(hann(150))).mean()\n\n        X.loc[seg_id, 'classic_sta_lta1_mean_%d' % i] = classic_sta_lta(sig, 500, 10000).mean()\n        X.loc[seg_id, 'classic_sta_lta2_mean_%d' % i] = classic_sta_lta(sig, 5000, 100000).mean()\n        X.loc[seg_id, 'classic_sta_lta3_mean_%d' % i] = classic_sta_lta(sig, 3333, 6666).mean()\n        X.loc[seg_id, 'classic_sta_lta4_mean_%d' % i] = classic_sta_lta(sig, 10000, 25000).mean()\n\n        X.loc[seg_id, 'Moving_average_700_mean_%d' % i] = sig.rolling(window=700).mean().mean(skipna=True)\n        X.loc[seg_id, 'Moving_average_1500_mean_%d' % i] = sig.rolling(window=1500).mean().mean(skipna=True)\n        X.loc[seg_id, 'Moving_average_3000_mean_%d' % i] = sig.rolling(window=3000).mean().mean(skipna=True)\n        X.loc[seg_id, 'Moving_average_6000_mean_%d' % i] = sig.rolling(window=6000).mean().mean(skipna=True)\n\n        ewma = pd.Series.ewm\n        X.loc[seg_id, 'exp_Moving_average_300_mean_%d' % i] = ewma(sig, span=300).mean().mean(skipna=True)\n        X.loc[seg_id, 'exp_Moving_average_3000_mean_%d' % i] = ewma(sig, span=3000).mean().mean(skipna=True)\n        X.loc[seg_id, 'exp_Moving_average_30000_mean_%d' % i] = ewma(sig, span=6000).mean().mean(skipna=True)\n\n        no_of_std = 2\n        X.loc[seg_id, 'MA_700MA_std_mean_%d' % i] = sig.rolling(window=700).std().mean()\n        X.loc[seg_id, 'MA_700MA_BB_high_mean_%d' % i] = (\n                    X.loc[seg_id, 'Moving_average_700_mean_%d' % i] + no_of_std * X.loc[seg_id, 'MA_700MA_std_mean_%d' % i]).mean()\n        X.loc[seg_id, 'MA_700MA_BB_low_mean_%d' % i] = (\n                    X.loc[seg_id, 'Moving_average_700_mean_%d' % i] - no_of_std * X.loc[seg_id, 'MA_700MA_std_mean_%d' % i]).mean()\n        X.loc[seg_id, 'MA_400MA_std_mean_%d' % i] = sig.rolling(window=400).std().mean()\n        X.loc[seg_id, 'MA_400MA_BB_high_mean_%d' % i] = (\n                    X.loc[seg_id, 'Moving_average_700_mean_%d' % i] + no_of_std * X.loc[seg_id, 'MA_400MA_std_mean_%d' % i]).mean()\n        X.loc[seg_id, 'MA_400MA_BB_low_mean_%d' % i] = (\n                    X.loc[seg_id, 'Moving_average_700_mean_%d' % i] - no_of_std * X.loc[seg_id, 'MA_400MA_std_mean_%d' % i]).mean()\n        X.loc[seg_id, 'MA_1000MA_std_mean_%d' % i] = sig.rolling(window=1000).std().mean()\n\n        X.loc[seg_id, 'iqr_%d' % i] = np.subtract(*np.percentile(sig, [75, 25]))\n        X.loc[seg_id, 'q999_%d' % i] = np.quantile(sig, 0.999)\n        X.loc[seg_id, 'q001_%d' % i] = np.quantile(sig, 0.001)\n        X.loc[seg_id, 'ave10_%d' % i] = stats.trim_mean(sig, 0.1)\n\n    for windows in [10, 100, 1000]:\n        x_roll_std = xc.rolling(windows).std().dropna().values\n        x_roll_mean = xc.rolling(windows).mean().dropna().values\n\n        X.loc[seg_id, 'ave_roll_std_' + str(windows)] = x_roll_std.mean()\n        X.loc[seg_id, 'std_roll_std_' + str(windows)] = x_roll_std.std()\n        X.loc[seg_id, 'max_roll_std_' + str(windows)] = x_roll_std.max()\n        X.loc[seg_id, 'min_roll_std_' + str(windows)] = x_roll_std.min()\n        X.loc[seg_id, 'q01_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.01)\n        X.loc[seg_id, 'q05_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.05)\n        X.loc[seg_id, 'q95_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.95)\n        X.loc[seg_id, 'q99_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.99)\n        X.loc[seg_id, 'av_change_abs_roll_std_' + str(windows)] = np.mean(np.diff(x_roll_std))\n        X.loc[seg_id, 'av_change_rate_roll_std_' + str(windows)] = np.mean(\n            np.nonzero((np.diff(x_roll_std) / x_roll_std[:-1]))[0])\n        X.loc[seg_id, 'abs_max_roll_std_' + str(windows)] = np.abs(x_roll_std).max()\n\n        X.loc[seg_id, 'ave_roll_mean_' + str(windows)] = x_roll_mean.mean()\n        X.loc[seg_id, 'std_roll_mean_' + str(windows)] = x_roll_mean.std()\n        X.loc[seg_id, 'max_roll_mean_' + str(windows)] = x_roll_mean.max()\n        X.loc[seg_id, 'min_roll_mean_' + str(windows)] = x_roll_mean.min()\n        X.loc[seg_id, 'q01_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.01)\n        X.loc[seg_id, 'q05_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.05)\n        X.loc[seg_id, 'q95_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.95)\n        X.loc[seg_id, 'q99_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.99)\n        X.loc[seg_id, 'av_change_abs_roll_mean_' + str(windows)] = np.mean(np.diff(x_roll_mean))\n        X.loc[seg_id, 'av_change_rate_roll_mean_' + str(windows)] = np.mean(\n            np.nonzero((np.diff(x_roll_mean) / x_roll_mean[:-1]))[0])\n        X.loc[seg_id, 'abs_max_roll_mean_' + str(windows)] = np.abs(x_roll_mean).max()\n\n    return X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_test_fields():\n    train_X = pd.read_csv(os.path.join(OUTPUT_DIR, 'train_x.csv'))\n    try:\n        train_X.drop(labels=['seg_id', 'seg_start', 'seg_end'], axis=1, inplace=True)\n    except:\n        pass\n\n    submission = pd.read_csv(os.path.join(DATA_DIR, 'sample_submission.csv'), index_col='seg_id')\n    test_X = pd.DataFrame(columns=train_X.columns, dtype=np.float64, index=submission.index)\n\n    print('start for loop')\n    count = 0\n    for seg_id in tqdm_notebook(test_X.index):  # just tqdm in IDE\n        seg = pd.read_csv(os.path.join(DATA_DIR, 'test', str(seg_id) + '.csv'))\n        # train_X = create_features_pk_det(seg_id, seg, train_X, start_idx, end_idx)\n        test_X = create_features(seg_id, seg, test_X, 0, 0)\n\n        if count % 100 == 0:\n            print('working', seg_id)\n        count += 1\n\n    test_X.to_csv(os.path.join(OUTPUT_DIR, 'test_x.csv'), index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def scale_fields(fn_train='train_x.csv', fn_test='test_x.csv', \n                 fn_out_train='scaled_train_X.csv' , fn_out_test='scaled_test_X.csv'):\n    train_X = pd.read_csv(os.path.join(OUTPUT_DIR, fn_train))\n    try:\n        train_X.drop(labels=['seg_id', 'seg_start', 'seg_end'], axis=1, inplace=True)\n    except:\n        pass\n    test_X = pd.read_csv(os.path.join(OUTPUT_DIR, fn_test))\n    test_X.drop(labels=['seg_id', 'seg_start', 'seg_end'], axis=1, inplace=True)\n\n    print('start scaler')\n    scaler = StandardScaler()\n    scaler.fit(train_X)\n    scaled_train_X = pd.DataFrame(scaler.transform(train_X), columns=train_X.columns)\n    scaled_test_X = pd.DataFrame(scaler.transform(test_X), columns=test_X.columns)\n\n    scaled_train_X.to_csv(os.path.join(OUTPUT_DIR, fn_out_train), index=False)\n    scaled_test_X.to_csv(os.path.join(OUTPUT_DIR, fn_out_test), index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nbuild_test_fields()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Scaling"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nscale_fields()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}