{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"%%time\nimport numpy as np\nimport pandas as pd\nimport glob, scipy\nfrom sklearn import *\nfrom catboost import CatBoostRegressor\n\n#https://www.kaggle.com/andrekos/basic-feature-benchmark-with-quantiles\n#Added quantiles per andrekos kernel\n#Added Mean Absolute Deviation, unbiased kurtosis, unbiased skew per @interneuron suggestion\n\n#https://www.kaggle.com/jsaguiar/baseline-with-abs-and-trend-features\n#Added Trend Features from @jsaguiar\ndef add_trend_feature(arr, abs_values=False):\n    idx = np.array(range(len(arr)))\n    if abs_values:\n        arr = np.abs(arr)\n    lr = linear_model.LinearRegression(n_jobs=-1)\n    lr.fit(idx.reshape(-1, 1), arr)\n    return lr.coef_[0]\n\nsize = 10_000\ntrain = pd.read_csv('../input/train.csv', iterator=True, chunksize=size, dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64})\nstrain = []\ngroup = []\nfor df in train:\n    group.append(df)\n    group = group[-int(150_000/size):]\n    df2 = pd.concat(group)\n    if len(df2)==150_000:\n        seismic_chunk = df2['acoustic_data']\n        fullmean = seismic_chunk.mean()\n        fullstd = seismic_chunk.std()\n        fullmax = seismic_chunk.max()\n        fullmin = seismic_chunk.min()\n        fullmad = seismic_chunk.mad()\n        fullkurtosis = seismic_chunk.kurtosis()\n        fullskew = seismic_chunk.skew()\n        \n        #https://www.kaggle.com/artgor/earthquakes-fe-more-features-and-samples\n        sc_abs = np.abs(seismic_chunk)\n        q01_abs = np.quantile(sc_abs,0.01)\n        q05_abs = np.quantile(sc_abs,0.05)\n        q95_abs = np.quantile(sc_abs,0.95)\n        q99_abs = np.quantile(sc_abs,0.99)\n        roll = seismic_chunk.rolling(100).std().dropna().values\n        q01_roll = np.quantile(roll,0.01)\n        q05_roll = np.quantile(roll,0.05)\n        q95_roll = np.quantile(roll,0.95)\n        q99_roll = np.quantile(roll,0.99)\n        \n        seismic_chunk = df2['acoustic_data'].values\n        q01 = np.quantile(seismic_chunk,0.01)\n        q05 = np.quantile(seismic_chunk,0.05)\n        q95 = np.quantile(seismic_chunk,0.95)\n        q99 = np.quantile(seismic_chunk,0.99)\n        abs_max = np.abs(seismic_chunk).max()\n        abs_mean = np.abs(seismic_chunk).mean()\n        abs_std = np.abs(seismic_chunk).std()\n        trend = add_trend_feature(seismic_chunk)\n        abs_trend = add_trend_feature(seismic_chunk, abs_values=True)\n        lastTTF = df2['time_to_failure'].values[-1]\n\n        strain.append([fullmean, fullstd, fullmax, fullmin, fullmad, fullkurtosis, fullskew, q01, q05, q95, q99, q01_abs, q05_abs, q95_abs, q99_abs, q01_roll, q05_roll, q95_roll, q99_roll, abs_max, abs_mean, abs_std, trend, abs_trend, lastTTF])\nstrain = pd.DataFrame(strain, columns=['fullmean', 'fullstd', 'fullmax', 'fullmin', 'fullmad', 'fullkurtosis', 'fullskew', 'q01', 'q05', 'q95', 'q99', 'q01_abs', 'q05_abs', 'q95_abs', 'q99_abs', 'q01_roll', 'q05_roll', 'q95_roll', 'q99_roll', 'abs_max', 'abs_mean', 'abs_std', 'trend', 'abs_trend', 'time_to_failure'])\n\ntest = glob.glob('../input/test/**')\nstest = []\nfor path in test:\n    df = pd.read_csv(path, dtype={'acoustic_data': np.int16})\n    seg_id = path.split('/')[-1].split('.')[0]\n    seismic_chunk = df['acoustic_data']\n    fullmean = seismic_chunk.mean()\n    fullstd = seismic_chunk.std()\n    fullmax = seismic_chunk.max()\n    fullmin = seismic_chunk.min()\n    fullmad = seismic_chunk.mad()\n    fullkurtosis = seismic_chunk.kurtosis()\n    fullskew = seismic_chunk.skew()\n\n    #https://www.kaggle.com/artgor/earthquakes-fe-more-features-and-samples\n    sc_abs = np.abs(seismic_chunk)\n    q01_abs = np.quantile(sc_abs,0.01)\n    q05_abs = np.quantile(sc_abs,0.05)\n    q95_abs = np.quantile(sc_abs,0.95)\n    q99_abs = np.quantile(sc_abs,0.99)\n    roll = seismic_chunk.rolling(100).std().dropna().values\n    q01_roll = np.quantile(roll,0.01)\n    q05_roll = np.quantile(roll,0.05)\n    q95_roll = np.quantile(roll,0.95)\n    q99_roll = np.quantile(roll,0.99)\n    \n    seismic_chunk = df['acoustic_data'].values\n    q01 = np.quantile(seismic_chunk,0.01)\n    q05 = np.quantile(seismic_chunk,0.05)\n    q95 = np.quantile(seismic_chunk,0.95)\n    q99 = np.quantile(seismic_chunk,0.99)\n    abs_max = np.abs(seismic_chunk).max()\n    abs_mean = np.abs(seismic_chunk).mean()\n    abs_std = np.abs(seismic_chunk).std()\n    trend = add_trend_feature(seismic_chunk)\n    abs_trend = add_trend_feature(seismic_chunk, abs_values=True)\n    stest.append([seg_id, fullmean, fullstd, fullmax, fullmin, fullmad, fullkurtosis, fullskew, q01, q05, q95, q99, q01_abs, q05_abs, q95_abs, q99_abs, q01_roll, q05_roll, q95_roll, q99_roll, abs_max, abs_mean, abs_std, trend, abs_trend])\nstest = pd.DataFrame(stest, columns=['seg_id', 'fullmean', 'fullstd', 'fullmax', 'fullmin', 'fullmad', 'fullkurtosis', 'fullskew', 'q01', 'q05', 'q95', 'q99', 'q01_abs', 'q05_abs', 'q95_abs', 'q99_abs', 'q01_roll', 'q05_roll', 'q95_roll', 'q99_roll', 'abs_max', 'abs_mean', 'abs_std', 'trend', 'abs_trend'])\n\nsub = pd.read_csv('../input/sample_submission.csv')\nprint(strain.shape, stest.shape, sub.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"743d4920be8b863e12a0fd0db56b43e8d05712bd","scrolled":true,"trusted":false},"cell_type":"code","source":"%%time\ncol = [c for c in strain.columns if c not in ['time_to_failure']]\n\n#https://www.kaggle.com/inversion/basic-feature-benchmark\n#https://www.kaggle.com/byfone/basic-feature-feat-catboost\n\nscaler = preprocessing.StandardScaler()\nscaled_train = scaler.fit_transform(strain[col])\nscaled_test = scaler.transform(stest[col])\n\nm = CatBoostRegressor(loss_function='MAE')\nm.fit(scaled_train, strain['time_to_failure'], silent=True)\nprint(metrics.mean_absolute_error(strain['time_to_failure'], m.predict(scaled_train)))\nstest['time_to_failure'] = m.predict(scaled_test)\n\nstest[['seg_id','time_to_failure']].to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2329088ee20d7b3498d788f92003e959c9f416fa","trusted":false},"cell_type":"code","source":"!rm -r catboost_info","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"64622a9eb70c9bb14ff1c61bbf31a5aad202960c"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}