{"cells":[{"metadata":{"_uuid":"2ad4b5eaada157c6fe88040ae9032305ccb7be33"},"cell_type":"markdown","source":"Thanks to Lavanya whose kernel I am forking to quickly get started on this competition."},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":false,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true,"_kg_hide-output":false},"cell_type":"code","source":"import gc\nimport os\nimport time\nimport logging\nimport datetime\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport xgboost as xgb\nimport lightgbm as lgb\nfrom scipy import stats\nfrom scipy.signal import hann\nfrom tqdm import tqdm_notebook\nimport matplotlib.pyplot as plt\nfrom scipy.signal import hilbert\nfrom scipy.signal import convolve\nfrom sklearn.svm import NuSVR, SVR\nfrom catboost import CatBoostRegressor\nfrom sklearn.kernel_ridge import KernelRidge\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.model_selection import KFold,StratifiedKFold, RepeatedKFold\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f7dc473a43cbd799a184ca71a27391bb6faa9984"},"cell_type":"markdown","source":"## Load the data\n\nLet's see first what files we have in input directory."},{"metadata":{"_kg_hide-input":false,"_uuid":"45e5498b89764854036baa88eda6ead21d41f382","trusted":true},"cell_type":"code","source":"IS_LOCAL = False\nif(IS_LOCAL):\n    PATH=\"../input/LANL/\"\nelse:\n    PATH=\"../input/\"\nos.listdir(PATH)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"95ef0a76216010678fb760844307b24719eb4bec"},"cell_type":"markdown","source":"We have two files in the **input** directory and another directory, with the **test** data.  \n\nLet's see how many files are in **test** folder."},{"metadata":{"_kg_hide-input":false,"_uuid":"91bda65accbd9172a37799acd2e9198a3477060e","trusted":true},"cell_type":"code","source":"print(\"There are {} files in test folder\".format(len(os.listdir(os.path.join(PATH, 'test' )))))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"68ced52082fc77c768db9ff14fed7cbe43e097d1"},"cell_type":"markdown","source":"\n\nLet's load the train file."},{"metadata":{"_kg_hide-input":false,"_uuid":"bb8916c762c26e98e31a721e4aae673aa12e6c5f","trusted":true},"cell_type":"code","source":"%%time\ntrain_df = pd.read_csv(os.path.join(PATH,'train.csv'), dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32})","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true},"cell_type":"markdown","source":"Let's check the data imported."},{"metadata":{"_kg_hide-input":false,"_uuid":"40c15befb5782fe1d671ae4ad0ca592aff316073","trusted":true},"cell_type":"code","source":"print(\"Train: rows:{} cols:{}\".format(train_df.shape[0], train_df.shape[1]))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"_uuid":"0ee349f0d7f482a0cc4827a3b7515a915e4dd37f","trusted":true},"cell_type":"code","source":"pd.options.display.precision = 15\ntrain_df.head(10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"877ce7e2c62dc0c7b9a86c081754a7f3b19bb364"},"cell_type":"markdown","source":"The dimmension of the data is quite large, in excess of 600 millions rows of data.  \nThe two columns in the train dataset have the following meaning:   \n*  accoustic_data: is the accoustic signal measured in the laboratory experiment;  \n* time to failure: this gives the time until a failure will occurs.\n\nLet's plot 1% of the data. For this we will sample every 100 points of data.  "},{"metadata":{"_kg_hide-input":false,"_uuid":"206af56767581d8e98777da3b0670e11d4e444bd","scrolled":false,"trusted":true},"cell_type":"code","source":"train_ad_sample_df = train_df['acoustic_data'].values[::100]\ntrain_ttf_sample_df = train_df['time_to_failure'].values[::100]\n\ndef plot_acc_ttf_data(train_ad_sample_df, train_ttf_sample_df, title=\"Acoustic data and time to failure: 1% sampled data\"):\n    fig, ax1 = plt.subplots(figsize=(12, 8))\n    plt.title(title)\n    plt.plot(train_ad_sample_df, color='r')\n    ax1.set_ylabel('acoustic data', color='r')\n    plt.legend(['acoustic data'], loc=(0.01, 0.95))\n    ax2 = ax1.twinx()\n    plt.plot(train_ttf_sample_df, color='b')\n    ax2.set_ylabel('time to failure', color='b')\n    plt.legend(['time to failure'], loc=(0.01, 0.9))\n    plt.grid(True)\n\nplot_acc_ttf_data(train_ad_sample_df, train_ttf_sample_df)\ndel train_ad_sample_df\ndel train_ttf_sample_df","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"36c363a5f2e56e54c8ffddd802aa86d91c61d97f"},"cell_type":"markdown","source":"The plot shows only 1% of the full data. \nThe acoustic data shows complex oscilations with variable amplitude. Just before each failure there is an increase in the amplitude of the acoustic data. We see that large amplitudes are also obtained at different moments in time (for example about the mid-time between two succesive failures).  \n\nLet's plot as well the first 1% of the data."},{"metadata":{"_kg_hide-input":false,"_uuid":"845e5a241e545d73cfb1dbaea2cfa464faf8148a","trusted":true},"cell_type":"code","source":"train_ad_sample_df = train_df['acoustic_data'].values[:6291455]\ntrain_ttf_sample_df = train_df['time_to_failure'].values[:6291455]\nplot_acc_ttf_data(train_ad_sample_df, train_ttf_sample_df, title=\"Acoustic data and time to failure: 1% of data\")\ndel train_ad_sample_df\ndel train_ttf_sample_df","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"78948b35358020156422a76c8d60dc132f17417b"},"cell_type":"markdown","source":"On this zoomed-in-time plot we can see that actually the large oscilation before the failure is not quite in the last moment. There are also trains of intense oscilations preceeding the large one and also some oscilations with smaller peaks after the large one. Then, after some minor oscilations, the failure occurs."},{"metadata":{"_uuid":"8dee473309c39d74020b37f2e4863094191c121f"},"cell_type":"markdown","source":"The test segments are 150,000 each.   \nWe split the train data in segments of the same dimmension with the test sets.\n\nWe will create additional aggregation features, calculated on the segments. \n"},{"metadata":{"_kg_hide-input":false,"_uuid":"abcf886f07660f12d170a2182675e15efffefde0","trusted":true},"cell_type":"code","source":"rows = 150000\nsegments = int(np.floor(train_df.shape[0] / rows))\nprint(\"Number of segments: \", segments)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d107cd036c9a931d1e85795f11e5853811a44246"},"cell_type":"markdown","source":"Let's define some computation helper functions."},{"metadata":{"_kg_hide-input":false,"_uuid":"8b46e3d6267988a865b69efc1f92ba90a20ca7b7","trusted":true},"cell_type":"code","source":"def add_trend_feature(arr, abs_values=False):\n    idx = np.array(range(len(arr)))\n    if abs_values:\n        arr = np.abs(arr)\n    lr = LinearRegression()\n    lr.fit(idx.reshape(-1, 1), arr)\n    return lr.coef_[0]\n\ndef classic_sta_lta(x, length_sta, length_lta):\n    sta = np.cumsum(x ** 2)\n    # Convert to float\n    sta = np.require(sta, dtype=np.float)\n    # Copy for LTA\n    lta = sta.copy()\n    # Compute the STA and the LTA\n    sta[length_sta:] = sta[length_sta:] - sta[:-length_sta]\n    sta /= length_sta\n    lta[length_lta:] = lta[length_lta:] - lta[:-length_lta]\n    lta /= length_lta\n    # Pad zeros\n    sta[:length_lta - 1] = 0\n    # Avoid division by zero by setting zero values to tiny float\n    dtiny = np.finfo(0.0).tiny\n    idx = lta < dtiny\n    lta[idx] = dtiny\n    return sta / lta","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0cec83d75c3498355fb232c33ef43dcd2017f24e"},"cell_type":"markdown","source":"## Process train file\n\nNow let's calculate the aggregated functions for train set."},{"metadata":{"_kg_hide-input":false,"_uuid":"c65536bf1bf32037175518c3cc8fc7439e217e9d","trusted":true},"cell_type":"code","source":"train_X = pd.DataFrame(index=range(segments), dtype=np.float64)\ntrain_y = pd.DataFrame(index=range(segments), dtype=np.float64, columns=['time_to_failure'])\ntotal_mean = train_df['acoustic_data'].mean()\ntotal_std = train_df['acoustic_data'].std()\ntotal_max = train_df['acoustic_data'].max()\ntotal_min = train_df['acoustic_data'].min()\ntotal_sum = train_df['acoustic_data'].sum()\ntotal_abs_sum = np.abs(train_df['acoustic_data']).sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"759944135252fbceaf9cee3f8d268a0de7843623"},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"_uuid":"0745eb336b1313d3c46c957d7b18c19c98076743","trusted":true},"cell_type":"code","source":"def create_features(seg_id, seg, X):\n    xc = pd.Series(seg['acoustic_data'].values)\n    zc = np.fft.fft(xc)\n    \n    X.loc[seg_id, 'mean'] = xc.mean()\n    X.loc[seg_id, 'std'] = xc.std()\n    X.loc[seg_id, 'max'] = xc.max()\n    X.loc[seg_id, 'min'] = xc.min()\n    \n    #FFT transform values\n    realFFT = np.real(zc)\n    imagFFT = np.imag(zc)\n    X.loc[seg_id, 'Rmean'] = realFFT.mean()\n    X.loc[seg_id, 'Rstd'] = realFFT.std()\n    X.loc[seg_id, 'Rmax'] = realFFT.max()\n    X.loc[seg_id, 'Rmin'] = realFFT.min()\n    X.loc[seg_id, 'Imean'] = imagFFT.mean()\n    X.loc[seg_id, 'Istd'] = imagFFT.std()\n    X.loc[seg_id, 'Imax'] = imagFFT.max()\n    X.loc[seg_id, 'Imin'] = imagFFT.min()\n    X.loc[seg_id, 'Rmean_last_5000'] = realFFT[-5000:].mean()\n    X.loc[seg_id, 'Rstd__last_5000'] = realFFT[-5000:].std()\n    X.loc[seg_id, 'Rmax_last_5000'] = realFFT[-5000:].max()\n    X.loc[seg_id, 'Rmin_last_5000'] = realFFT[-5000:].min()\n    X.loc[seg_id, 'Rmean_last_15000'] = realFFT[-15000:].mean()\n    X.loc[seg_id, 'Rstd_last_15000'] = realFFT[-15000:].std()\n    X.loc[seg_id, 'Rmax_last_15000'] = realFFT[-15000:].max()\n    X.loc[seg_id, 'Rmin_last_15000'] = realFFT[-15000:].min()\n    \n    X.loc[seg_id, 'mean_change_abs'] = np.mean(np.diff(xc))\n    X.loc[seg_id, 'mean_change_rate'] = np.mean(np.nonzero((np.diff(xc) / xc[:-1]))[0])\n    X.loc[seg_id, 'abs_max'] = np.abs(xc).max()\n    X.loc[seg_id, 'abs_min'] = np.abs(xc).min()\n    \n    X.loc[seg_id, 'std_first_50000'] = xc[:50000].std()\n    X.loc[seg_id, 'std_last_50000'] = xc[-50000:].std()\n    X.loc[seg_id, 'std_first_10000'] = xc[:10000].std()\n    X.loc[seg_id, 'std_last_10000'] = xc[-10000:].std()\n    \n    X.loc[seg_id, 'avg_first_50000'] = xc[:50000].mean()\n    X.loc[seg_id, 'avg_last_50000'] = xc[-50000:].mean()\n    X.loc[seg_id, 'avg_first_10000'] = xc[:10000].mean()\n    X.loc[seg_id, 'avg_last_10000'] = xc[-10000:].mean()\n    \n    X.loc[seg_id, 'min_first_50000'] = xc[:50000].min()\n    X.loc[seg_id, 'min_last_50000'] = xc[-50000:].min()\n    X.loc[seg_id, 'min_first_10000'] = xc[:10000].min()\n    X.loc[seg_id, 'min_last_10000'] = xc[-10000:].min()\n    \n    X.loc[seg_id, 'max_first_50000'] = xc[:50000].max()\n    X.loc[seg_id, 'max_last_50000'] = xc[-50000:].max()\n    X.loc[seg_id, 'max_first_10000'] = xc[:10000].max()\n    X.loc[seg_id, 'max_last_10000'] = xc[-10000:].max()\n    \n    X.loc[seg_id, 'max_to_min'] = xc.max() / np.abs(xc.min())\n    X.loc[seg_id, 'max_to_min_diff'] = xc.max() - np.abs(xc.min())\n    X.loc[seg_id, 'count_big'] = len(xc[np.abs(xc) > 500])\n    X.loc[seg_id, 'sum'] = xc.sum()\n    \n    X.loc[seg_id, 'mean_change_rate_first_50000'] = np.mean(np.nonzero((np.diff(xc[:50000]) / xc[:50000][:-1]))[0])\n    X.loc[seg_id, 'mean_change_rate_last_50000'] = np.mean(np.nonzero((np.diff(xc[-50000:]) / xc[-50000:][:-1]))[0])\n    X.loc[seg_id, 'mean_change_rate_first_10000'] = np.mean(np.nonzero((np.diff(xc[:10000]) / xc[:10000][:-1]))[0])\n    X.loc[seg_id, 'mean_change_rate_last_10000'] = np.mean(np.nonzero((np.diff(xc[-10000:]) / xc[-10000:][:-1]))[0])\n    \n    X.loc[seg_id, 'q95'] = np.quantile(xc, 0.95)\n    X.loc[seg_id, 'q99'] = np.quantile(xc, 0.99)\n    X.loc[seg_id, 'q05'] = np.quantile(xc, 0.05)\n    X.loc[seg_id, 'q01'] = np.quantile(xc, 0.01)\n    \n    X.loc[seg_id, 'abs_q95'] = np.quantile(np.abs(xc), 0.95)\n    X.loc[seg_id, 'abs_q99'] = np.quantile(np.abs(xc), 0.99)\n    X.loc[seg_id, 'abs_q05'] = np.quantile(np.abs(xc), 0.05)\n    X.loc[seg_id, 'abs_q01'] = np.quantile(np.abs(xc), 0.01)\n    \n    X.loc[seg_id, 'trend'] = add_trend_feature(xc)\n    X.loc[seg_id, 'abs_trend'] = add_trend_feature(xc, abs_values=True)\n    X.loc[seg_id, 'abs_mean'] = np.abs(xc).mean()\n    X.loc[seg_id, 'abs_std'] = np.abs(xc).std()\n    \n    X.loc[seg_id, 'mad'] = xc.mad()\n    X.loc[seg_id, 'kurt'] = xc.kurtosis()\n    X.loc[seg_id, 'skew'] = xc.skew()\n    X.loc[seg_id, 'med'] = xc.median()\n    \n    X.loc[seg_id, 'Hilbert_mean'] = np.abs(hilbert(xc)).mean()\n    X.loc[seg_id, 'Hann_window_mean'] = (convolve(xc, hann(150), mode='same') / sum(hann(150))).mean()\n    X.loc[seg_id, 'classic_sta_lta1_mean'] = classic_sta_lta(xc, 500, 10000).mean()\n    X.loc[seg_id, 'classic_sta_lta2_mean'] = classic_sta_lta(xc, 5000, 100000).mean()\n    X.loc[seg_id, 'classic_sta_lta3_mean'] = classic_sta_lta(xc, 3333, 6666).mean()\n    X.loc[seg_id, 'classic_sta_lta4_mean'] = classic_sta_lta(xc, 10000, 25000).mean()\n    X.loc[seg_id, 'Moving_average_700_mean'] = xc.rolling(window=700).mean().mean(skipna=True)\n    X.loc[seg_id, 'Moving_average_1500_mean'] = xc.rolling(window=1500).mean().mean(skipna=True)\n    X.loc[seg_id, 'Moving_average_3000_mean'] = xc.rolling(window=3000).mean().mean(skipna=True)\n    X.loc[seg_id, 'Moving_average_6000_mean'] = xc.rolling(window=6000).mean().mean(skipna=True)\n    ewma = pd.Series.ewm\n    X.loc[seg_id, 'exp_Moving_average_300_mean'] = (ewma(xc, span=300).mean()).mean(skipna=True)\n    X.loc[seg_id, 'exp_Moving_average_3000_mean'] = ewma(xc, span=3000).mean().mean(skipna=True)\n    X.loc[seg_id, 'exp_Moving_average_30000_mean'] = ewma(xc, span=6000).mean().mean(skipna=True)\n    no_of_std = 2\n    X.loc[seg_id, 'MA_700MA_std_mean'] = xc.rolling(window=700).std().mean()\n    X.loc[seg_id,'MA_700MA_BB_high_mean'] = (X.loc[seg_id, 'Moving_average_700_mean'] + no_of_std * X.loc[seg_id, 'MA_700MA_std_mean']).mean()\n    X.loc[seg_id,'MA_700MA_BB_low_mean'] = (X.loc[seg_id, 'Moving_average_700_mean'] - no_of_std * X.loc[seg_id, 'MA_700MA_std_mean']).mean()\n    X.loc[seg_id, 'MA_400MA_std_mean'] = xc.rolling(window=400).std().mean()\n    X.loc[seg_id,'MA_400MA_BB_high_mean'] = (X.loc[seg_id, 'Moving_average_700_mean'] + no_of_std * X.loc[seg_id, 'MA_400MA_std_mean']).mean()\n    X.loc[seg_id,'MA_400MA_BB_low_mean'] = (X.loc[seg_id, 'Moving_average_700_mean'] - no_of_std * X.loc[seg_id, 'MA_400MA_std_mean']).mean()\n    X.loc[seg_id, 'MA_1000MA_std_mean'] = xc.rolling(window=1000).std().mean()\n    \n    X.loc[seg_id, 'iqr'] = np.subtract(*np.percentile(xc, [75, 25]))\n    X.loc[seg_id, 'q999'] = np.quantile(xc,0.999)\n    X.loc[seg_id, 'q001'] = np.quantile(xc,0.001)\n    X.loc[seg_id, 'ave10'] = stats.trim_mean(xc, 0.1)\n    \n    for windows in [10, 100, 1000]:\n        x_roll_std = xc.rolling(windows).std().dropna().values\n        x_roll_mean = xc.rolling(windows).mean().dropna().values\n        \n        X.loc[seg_id, 'ave_roll_std_' + str(windows)] = x_roll_std.mean()\n        X.loc[seg_id, 'std_roll_std_' + str(windows)] = x_roll_std.std()\n        X.loc[seg_id, 'max_roll_std_' + str(windows)] = x_roll_std.max()\n        X.loc[seg_id, 'min_roll_std_' + str(windows)] = x_roll_std.min()\n        X.loc[seg_id, 'q01_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.01)\n        X.loc[seg_id, 'q05_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.05)\n        X.loc[seg_id, 'q95_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.95)\n        X.loc[seg_id, 'q99_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.99)\n        X.loc[seg_id, 'av_change_abs_roll_std_' + str(windows)] = np.mean(np.diff(x_roll_std))\n        X.loc[seg_id, 'av_change_rate_roll_std_' + str(windows)] = np.mean(np.nonzero((np.diff(x_roll_std) / x_roll_std[:-1]))[0])\n        X.loc[seg_id, 'abs_max_roll_std_' + str(windows)] = np.abs(x_roll_std).max()\n        \n        X.loc[seg_id, 'ave_roll_mean_' + str(windows)] = x_roll_mean.mean()\n        X.loc[seg_id, 'std_roll_mean_' + str(windows)] = x_roll_mean.std()\n        X.loc[seg_id, 'max_roll_mean_' + str(windows)] = x_roll_mean.max()\n        X.loc[seg_id, 'min_roll_mean_' + str(windows)] = x_roll_mean.min()\n        X.loc[seg_id, 'q01_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.01)\n        X.loc[seg_id, 'q05_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.05)\n        X.loc[seg_id, 'q95_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.95)\n        X.loc[seg_id, 'q99_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.99)\n        X.loc[seg_id, 'av_change_abs_roll_mean_' + str(windows)] = np.mean(np.diff(x_roll_mean))\n        X.loc[seg_id, 'av_change_rate_roll_mean_' + str(windows)] = np.mean(np.nonzero((np.diff(x_roll_mean) / x_roll_mean[:-1]))[0])\n        X.loc[seg_id, 'abs_max_roll_mean_' + str(windows)] = np.abs(x_roll_mean).max()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5a446450bfd25296d6b98dde3e750e11d96250a4","trusted":true},"cell_type":"code","source":"# iterate over all segments\nfor seg_id in tqdm_notebook(range(segments)):\n    seg = train_df.iloc[seg_id*rows:seg_id*rows+rows]\n    create_features(seg_id, seg, train_X)\n    train_y.loc[seg_id, 'time_to_failure'] = seg['time_to_failure'].values[-1]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c2e064c25b00c08acd8f8973f565a0b29d09704f"},"cell_type":"markdown","source":"Let's check the result. We plot the shape and the head of train_X."},{"metadata":{"_kg_hide-input":false,"_uuid":"ff98b8f65290cfa14a5fed507a5567a589109fc9","trusted":true},"cell_type":"code","source":"train_X.shape","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"_uuid":"dc6dd182bf1819c4dcfe823416efca51ff15d5bf","trusted":true},"cell_type":"code","source":"train_X.head(10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"58196ccf11c59b8bca96279f6bdc7e66eaad6fbe"},"cell_type":"markdown","source":"We scale the data."},{"metadata":{"_kg_hide-input":false,"_uuid":"5e9f4c996ac9842fd3243e4f9d3a6e77b8b65cf7","trusted":true},"cell_type":"code","source":"scaler = StandardScaler()\nscaler.fit(train_X)\nscaled_train_X = pd.DataFrame(scaler.transform(train_X), columns=train_X.columns)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"885370309aca6e12d22a811d3ccc126205de468d"},"cell_type":"markdown","source":"Let's check the obtained dataframe."},{"metadata":{"_kg_hide-input":false,"_uuid":"1212dd6508332a32bfb4e337f3076ee9680584d4","trusted":true},"cell_type":"code","source":"scaled_train_X.head(10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"500232cf8ff46fd1d0a9a43088c815d9624a222a"},"cell_type":"markdown","source":"## Process test data\n\nWe apply the same processing done for the training data to the test data.\n\nWe read the submission file and prepare the test file."},{"metadata":{"_kg_hide-input":false,"_uuid":"38e67f8e457d7c7ce4900b454d3d4a1ab5c42d04","trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id')\ntest_X = pd.DataFrame(columns=train_X.columns, dtype=np.float64, index=submission.index)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b9b80332f6e9af2ee82f8e0feca4e9d1d18925ee"},"cell_type":"markdown","source":"Let's check the shape of the submission and test_X datasets."},{"metadata":{"_kg_hide-input":false,"_uuid":"1f06fee3edb09ac05890fa109808487de395d1de","trusted":true},"cell_type":"code","source":"submission.shape, test_X.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e3b54cd654438c2af63060b03ddb767eacbd70c6","trusted":true},"cell_type":"code","source":"for seg_id in tqdm_notebook(test_X.index):\n    seg = pd.read_csv('../input/test/' + seg_id + '.csv')\n    create_features(seg_id, seg, test_X)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5630b8d9234af698ed83af94980538e9ed8eaa59"},"cell_type":"markdown","source":"We scale also the test data."},{"metadata":{"_kg_hide-input":false,"_uuid":"a18c26409ced2a71c3b7b4926d05455afd5fa26a","trusted":true},"cell_type":"code","source":"scaled_test_X = pd.DataFrame(scaler.transform(test_X), columns=test_X.columns)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"_uuid":"9c5701e9cc1c9076bde818af54fa67ec4ad3eb4b","trusted":true},"cell_type":"code","source":"scaled_test_X.shape","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"_uuid":"7bf4d31bea2f1fd82fbfdc1a4a84f36834daf5c3","trusted":true},"cell_type":"code","source":"scaled_test_X.tail(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0a1cbaa7f99237c103a2d59268bf8ba97856686e"},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"_uuid":"99d43c789dde8cac37270dc16430cf98870788d3","trusted":true},"cell_type":"code","source":"n_fold = 5\nfolds = KFold(n_splits=n_fold, shuffle=True)\ntrain_columns = scaled_train_X.columns.values","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"_uuid":"d0b278f0a14dbd3dee610e2e27a0cc6d55844678","trusted":true},"cell_type":"code","source":"from fastai.tabular import *\n\npredictions = np.zeros([len(test_X),1],dtype='float32')\n\n#run model\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(scaled_train_X,train_y.values)):\n    strLog = \"fold {}\".format(fold_)\n    print(strLog)\n    \n    scaled_train_X['target']= train_y['time_to_failure']\n    \n    procs= [Normalize]\n    data = TabularDataBunch.from_df(path= '.',df= scaled_train_X,dep_var= 'target',valid_idx= val_idx,procs=procs,test_df=scaled_test_X)\n    \n    learn= tabular_learner(data,layers=[150,100],metrics= mean_absolute_error)\n    \n   # learn.lr_find()\n   # learn.recorder.plot()\n    \n    learn.fit_one_cycle(16,0.001)\n    \n    test_predicts= learn.get_preds(ds_type= DatasetType.Test)\n    Y_pred= to_np(test_predicts[0])\n  \n    predictions+= Y_pred / n_fold","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"_uuid":"641d4feeb57e87e24c3f4bc29863c35169f9302f","trusted":false},"cell_type":"code","source":"submission.time_to_failure = predictions\nsubmission.to_csv('submission.csv',index=True)\n\nprint(\"Submission file created successfully\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}