{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from tqdm import tqdm_notebook as tqdm\nimport numpy as np\nimport pandas as pd\n\nraw = pd.read_csv('../input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64})\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"from tsfresh.feature_extraction import feature_calculators\nimport librosa\nimport pywt\n\n\nnp.random.seed(1337)\nnoise = np.random.normal(0, 0.5, 150_000)\n\n\ndef denoise_signal_simple(x, wavelet='db4', level=1):\n    coeff = pywt.wavedec(x, wavelet, mode=\"per\")\n    #univeral threshold\n    uthresh = 10\n    coeff[1:] = (pywt.threshold(i, value=uthresh, mode='hard') for i in coeff[1:])\n    # Reconstruct the signal using the thresholded coefficients\n    return pywt.waverec(coeff, wavelet, mode='per')\n\n\ndef feature_gen(z):\n    X = pd.DataFrame(index=[0], dtype=np.float64)\n    \n    z = z + noise\n    z = z - np.median(z)\n\n    den_sample_simple = denoise_signal_simple(z)\n    mfcc = librosa.feature.mfcc(z)\n    mfcc_mean = mfcc.mean(axis=1)\n    percentile_roll50_std_20 = np.percentile(pd.Series(z).rolling(50).std().dropna().values, 20)\n    \n    X['var_num_peaks_2_denoise_simple'] = feature_calculators.number_peaks(den_sample_simple, 2)\n    X['var_percentile_roll50_std_20'] = percentile_roll50_std_20\n    X['var_mfcc_mean18'] = mfcc_mean[18]\n    X['var_mfcc_mean4'] = mfcc_mean[4]\n    \n    return X\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from joblib import Parallel, delayed\nimport scipy as sp\nimport itertools\nimport gc\n\ndef parse_sample(sample, start):\n    delta = feature_gen(sample['acoustic_data'].values)\n    delta['start'] = start\n    delta['target'] = sample['time_to_failure'].values[-1]\n    return delta\n    \ndef sample_train_gen(df, segment_size=150_000, indices_to_calculate=[0]):\n    result = Parallel(n_jobs=4, temp_folder=\"/tmp\", max_nbytes=None, backend=\"multiprocessing\")(delayed(parse_sample)(df[int(i) : int(i) + segment_size], int(i)) \n                                                                                                for i in tqdm(indices_to_calculate))\n    data = [r.values for r in result]\n    data = np.vstack(data)\n    X = pd.DataFrame(data, columns=result[0].columns)\n    X = X.sort_values(\"start\")\n    return X\n\ndef parse_sample_test(seg_id):\n    sample = pd.read_csv('../input/test/' + seg_id + '.csv', dtype={'acoustic_data': np.int32})\n    delta = feature_gen(sample['acoustic_data'].values)\n    delta['seg_id'] = seg_id\n    return delta\n\ndef sample_test_gen():\n    X = pd.DataFrame()\n    submission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id')\n    result = Parallel(n_jobs=4, temp_folder=\"/tmp\", max_nbytes=None, backend=\"multiprocessing\")(delayed(parse_sample_test)(seg_id) for seg_id in tqdm(submission.index))\n    data = [r.values for r in result]\n    data = np.vstack(data)\n    X = pd.DataFrame(data, columns=result[0].columns)\n    return X\n\nindices_to_calculate = raw.index.values[::150_000][:-1]\n\ntrain = sample_train_gen(raw, indices_to_calculate=indices_to_calculate)\ngc.collect()\ntest = sample_test_gen()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"etq_meta = [\n{\"start\":0,         \"end\":5656574},\n{\"start\":5656574,   \"end\":50085878},\n{\"start\":50085878,  \"end\":104677356},\n{\"start\":104677356, \"end\":138772453},\n{\"start\":138772453, \"end\":187641820},\n{\"start\":187641820, \"end\":218652630},\n{\"start\":218652630, \"end\":245829585},\n{\"start\":245829585, \"end\":307838917},\n{\"start\":307838917, \"end\":338276287},\n{\"start\":338276287, \"end\":375377848},\n{\"start\":375377848, \"end\":419368880},\n{\"start\":419368880, \"end\":461811623},\n{\"start\":461811623, \"end\":495800225},\n{\"start\":495800225, \"end\":528777115},\n{\"start\":528777115, \"end\":585568144},\n{\"start\":585568144, \"end\":621985673},\n{\"start\":621985673, \"end\":629145480},\n]\n\nfor i, etq in enumerate(etq_meta):\n    train.loc[(train['start'] + 150_000 >= etq[\"start\"]) & (train['start'] <= etq[\"end\"] - 150_000), \"eq\"] = i\n\ntrain_sample = train[train[\"eq\"].isin([2, 7, 0, 4, 11, 13, 9, 1, 14, 10])]\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"Mean:   {train_sample['target'].mean():.4}\")\nprint(f\"Median: {train_sample['target'].median():.4}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import mean_absolute_error\nfrom sklearn.model_selection import KFold\nfrom numpy import random\nimport lightgbm as lgb\n\nrandom.seed(1234)\n\nfeatures = ['var_num_peaks_2_denoise_simple','var_percentile_roll50_std_20','var_mfcc_mean4',  'var_mfcc_mean18']\ntarget = train_sample[\"target\"].values\n\ntrain_X = train_sample[features].values\ntest_X = test[features].values\n\nsubmission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id')\noof = np.zeros(len(train_X))\nprediction = np.zeros(len(submission))\n\nn_fold = 3\n\nkf = KFold(n_splits=n_fold, shuffle=True, random_state=1337)\nkf = list(kf.split(np.arange(len(train_sample))))\n\nfor fold_n, (train_index, valid_index) in enumerate(kf):\n    print('Fold', fold_n)\n\n    trn_data = lgb.Dataset(train_X[train_index], label=target[train_index])\n    val_data = lgb.Dataset(train_X[valid_index], label=target[valid_index])\n    \n    params = {'num_leaves': 4,\n      'min_data_in_leaf': 5,\n      'objective':'fair',\n      'max_depth': -1,\n      'learning_rate': 0.02,\n      \"boosting\": \"gbdt\",\n      'boost_from_average': True,\n      \"feature_fraction\": 0.9,\n      \"bagging_freq\": 1,\n      \"bagging_fraction\": 0.5,\n      \"bagging_seed\": 0,\n      \"metric\": 'mae',\n      \"verbosity\": -1,\n      'max_bin': 500,\n      'reg_alpha': 0,\n      'reg_lambda': 0,\n      'seed': 0,\n      'n_jobs': 1\n      }\n\n    clf = lgb.train(params, trn_data, 1000000, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 1000)\n\n    oof[valid_index] += clf.predict(train_X[valid_index], num_iteration=clf.best_iteration)\n    prediction += clf.predict(test_X, num_iteration=clf.best_iteration)\n\nprediction /= n_fold\n\nprint('\\nMAE: ', mean_absolute_error(target, oof))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission['time_to_failure'] = prediction \nprint(submission.head())\nsubmission.to_csv('submission.csv')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}