{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"# The MIT License (MIT)\n\n# Permission is hereby granted, free of charge, to any person obtaining a copy\n# of this software and associated documentation files (the \"Software\"), to deal\n# in the Software without restriction, including without limitation the rights\n# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n# copies of the Software, and to permit persons to whom the Software is\n# furnished to do so, subject to the following conditions:\n\n# The above copyright notice and this permission notice shall be included in all\n# copies or substantial portions of the Software.\n\n# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n# SOFTWARE.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Import Libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from keras.layers import (BatchNormalization,Flatten,Convolution1D,Activation,Input,Dense,LSTM)\nfrom tsfresh.feature_extraction import feature_calculators\nfrom keras.callbacks import ModelCheckpoint,EarlyStopping\nfrom keras.utils import Sequence, to_categorical\nfrom sklearn.metrics import mean_absolute_error\nfrom keras.callbacks import ReduceLROnPlateau\nfrom keras import losses, models, optimizers\nfrom sklearn.model_selection import KFold\nfrom tqdm import tqdm_notebook as tqdm\nfrom joblib import Parallel, delayed\nfrom sklearn import preprocessing\nimport matplotlib.pyplot as plt\nfrom keras import backend as K\nimport tensorflow as tf\nimport lightgbm as lgb\nimport seaborn as sns\nimport random as rn\nimport pandas as pd\nimport numpy as np\nimport scipy as sp\nimport itertools\nimport warnings\nimport librosa\nimport pywt\nimport os\nimport gc\n\n\nwarnings.simplefilter(action='ignore', category=FutureWarning)\npd.options.mode.chained_assignment = None\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Description\n\nThe LANL Earthquake Prediction competition (https://www.kaggle.com/c/LANL-Earthquake-Prediction/overview) requires competitors to predict the time remaining (Time to failure, or TTF) until a laboratory earthquake occurs from real-time seismic data. We are given 150,000 data points of seismic data, which corresponds to 0.0375 seconds of seismic data (ordered in time).\n\nThe first place solution to LANL Earthquake Prediction is a geometric mean of a neural network (NN) solution and LightGBM (LGBM) solution. Since the NN and LGBM algorithms are very different, they each capture different parts of the signal, and blending the two together increases generalization.\n\nTo make predictions, we divide our training data into 150,000-length segments. Instead of using all of the training data, we decided to use only segments from the earthquake cycles that had exhibited higher TTF. This caused the TTF predictions to be biased higher.\n\nThe raw acoustic data itself is noisy; therefore, we utilize various packages to denoise the signal. Additionally, we inject random noise to every segment and remove the median of the segment, because we noticed the mean & median values were increasing as the laboratory experiment went forward in time. This improves generalization.\n\nAdditional solution details can be found on Kaggle's Discussion forum at https://www.kaggle.com/c/LANL-Earthquake-Prediction/discussion/94390."},{"metadata":{},"cell_type":"markdown","source":"## Read in data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# raw train data import\nraw = pd.read_csv('../input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32}) ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Functions for parsing and feature generation."},{"metadata":{"trusted":true},"cell_type":"code","source":"# The normalize function is required to normalize the data for the neural network.\n\ndef normalize(X_train, X_valid, X_test, normalize_opt, excluded_feat):\n    feats = [f for f in X_train.columns if f not in excluded_feat]\n    if normalize_opt != None:\n        if normalize_opt == 'min_max':\n            scaler = preprocessing.MinMaxScaler()\n        elif normalize_opt == 'robust':\n            scaler = preprocessing.RobustScaler()\n        elif normalize_opt == 'standard':\n            scaler = preprocessing.StandardScaler()\n        elif normalize_opt == 'max_abs':\n            scaler = preprocessing.MaxAbsScaler()\n        scaler = scaler.fit(X_train[feats])\n        X_train[feats] = scaler.transform(X_train[feats])\n        X_valid[feats] = scaler.transform(X_valid[feats])\n        X_test[feats] = scaler.transform(X_test[feats])\n    return X_train, X_valid, X_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# functions for feature generation\n# Create random noise for robustness\nnp.random.seed(1337)\nnoise = np.random.normal(0, 0.5, 150_000)\n\n# Mean Absolute Deviation\ndef maddest(d, axis=None):\n    return np.mean(np.absolute(d - np.mean(d, axis)), axis)\n\n# Denoise the raw signal given a segment x\ndef denoise_signal(x, wavelet='db4', level=1):\n    coeff = pywt.wavedec(x, wavelet, mode=\"per\")\n    sigma = (1/0.6745) * maddest(coeff[-level])\n    uthresh = sigma * np.sqrt(2*np.log(len(x)))\n    coeff[1:] = (pywt.threshold(i, value=uthresh, mode='hard') for i in coeff[1:])\n\n    return pywt.waverec(coeff, wavelet, mode='per')\n\n# Denoise the raw signal (simplified) given a segment x\ndef denoise_signal_simple(x, wavelet='db4', level=1):\n    coeff = pywt.wavedec(x, wavelet, mode=\"per\")\n    #univeral threshold\n    uthresh = 10\n    coeff[1:] = (pywt.threshold(i, value=uthresh, mode='hard') for i in coeff[1:])\n    # Reconstruct the signal using the thresholded coefficients\n    return pywt.waverec(coeff, wavelet, mode='per')\n\n# Generate the features given a segment z\ndef feature_gen(z):\n    X = pd.DataFrame(index=[0], dtype=np.float64)\n    \n    # Add noise, subtract median to remove bias from mean/median as time passes in the experiment\n    # Save the result as a new segment, z\n#     z = z + noise\n    z = z - np.median(z)\n\n    # Save denoised versions of z\n    den_sample = denoise_signal(z)\n    den_sample_simple = denoise_signal_simple(z)\n    \n    # Mel-frequency cepstral coefficients\n    mfcc = librosa.feature.mfcc(z)\n    mfcc_mean = mfcc.mean(axis=1)\n    mfcc_denoise_simple = librosa.feature.mfcc(den_sample_simple)\n    mfcc_mean_denoise_simple = mfcc_denoise_simple.mean(axis=1) #0-19\n    \n    # Spectral contrast\n    lib_spectral_contrast_denoise_simple = librosa.feature.spectral_contrast(den_sample_simple).mean(axis=1) #0-6\n    lib_spectral_contrast = librosa.feature.spectral_contrast(z).mean(axis=1) #0-6\n    \n    # Neural network features\n    X['NN_zero_crossings_denoise'] = len(np.where(np.diff(np.sign(den_sample)))[0])\n    X['NN_LGBM_percentile_roll20_std_50'] = np.percentile(pd.Series(z).rolling(20).std().dropna().values, 50)\n    X['NN_q95_roll20_std'] = np.quantile(pd.Series(z).rolling(20).std().dropna().values, 0.95)\n    X['NN_LGBM_mfcc_mean4'] = mfcc_mean[4]\n    X['NN_lib_spectral_contrast0'] = lib_spectral_contrast[0]\n    X['NN_num_peaks_3_denoise'] = feature_calculators.number_peaks(den_sample, 3)\n    X['NN_mfcc_mean_denoise_simple2'] = mfcc_mean_denoise_simple[2]\n    X['NN_mfcc_mean5'] = mfcc_mean[5]\n    X['NN_mfcc_mean2'] = mfcc_mean[2]\n    X['NN_mfcc_mean_denoise_simple5'] = mfcc_mean_denoise_simple[5]\n    X['NN_absquant95'] = np.quantile(np.abs(z), 0.95)\n    X['NN_median_roll50_std_denoise_simple'] = np.median(pd.Series(den_sample_simple).rolling(50).std().dropna().values)\n    X['NN_mfcc_mean_denoise_simple1'] = mfcc_mean_denoise_simple[1]\n    X['NN_quant99'] = np.quantile(z, 0.99)\n    X['NN_lib_zero_cross_rate_denoise_simple'] = librosa.feature.zero_crossing_rate(den_sample_simple)[0].mean()\n    X['NN_fftr_max_denoise'] = np.max(pd.Series(np.abs(np.fft.fft(den_sample)))[0:75000])\n    X['NN_abssumgreater15'] = np.sum(abs(z[np.where(abs(z)>15)]))\n    X['NN_LGBM_mfcc_mean18'] = mfcc_mean[18]\n    X['NN_lib_spectral_contrast_denoise_simple2'] = lib_spectral_contrast_denoise_simple[2]\n    X['NN_fftr_sum'] = np.sum(pd.Series(np.abs(np.fft.fft(z)))[0:75000])\n    X['NN_mfcc_mean_denoise_simple10'] = mfcc_mean_denoise_simple[10]\n    \n    # Extra features only LGBM used.\n    X['LGBM_num_peaks_2_denoise_simple'] = feature_calculators.number_peaks(den_sample_simple, 2)\n    X['LGBM_autocorr5'] = feature_calculators.autocorrelation(pd.Series(z), 5)\n    \n    # Windowed fast fourier transformations\n    fftrhann20000 = np.sum(np.abs(np.fft.fft(np.hanning(len(z))*z)[:20000]))\n    fftrhann20000_denoise = np.sum(np.abs(np.fft.fft(np.hanning(len(z))*den_sample)[:20000]))\n    fftrhann20000_diff_rate = (fftrhann20000 - fftrhann20000_denoise)/fftrhann20000\n    \n    X['LGBM_fftrhann20000_diff_rate'] = fftrhann20000_diff_rate\n    \n    return X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"raw_s = raw[0:10000000]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{"trusted":true},"cell_type":"code","source":"# create train and test sets\n# create train and test sets\ndef parse_sample(sample, start):\n    delta = feature_gen(sample['acoustic_data'].values)\n    delta['start'] = start\n    delta['target'] = sample['time_to_failure'].values[-1]\n    import pdb; pdb.set_trace()\n    return delta\n    \ndef sample_train_gen(df, segment_size=150_000, indices_to_calculate=[0]):\n    result = Parallel(n_jobs=1, temp_folder=\"/tmp\", max_nbytes=None, backend=\"multiprocessing\")(delayed(parse_sample)(df[int(i) : int(i) + segment_size], int(i)) \n                                                                                                for i in tqdm(indices_to_calculate))\n    data = [r.values for r in result]\n    data = np.vstack(data)\n    X = pd.DataFrame(data, columns=result[0].columns)\n    X = X.sort_values(\"start\")\n    return X\n\ndef parse_sample_test(seg_id):\n    sample = pd.read_csv('../input/test/' + seg_id + '.csv', dtype={'acoustic_data': np.int32})\n    delta = feature_gen(sample['acoustic_data'].values)\n    delta['seg_id'] = seg_id\n    return delta\n\ndef sample_test_gen():\n    X = pd.DataFrame()\n    submission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id')\n    result = Parallel(n_jobs=1, temp_folder=\"/tmp\", max_nbytes=None, backend=\"multiprocessing\")(delayed(parse_sample_test)(seg_id) for seg_id in tqdm(submission.index))\n    data = [r.values for r in result]\n    data = np.vstack(data)\n    X = pd.DataFrame(data, columns=result[0].columns)\n    return X\n\nindices_to_calculate = raw.index.values[::150_000][:-1]\n\ntrain = sample_train_gen(raw, indices_to_calculate=indices_to_calculate)\n# train = sample_train_gen(raw_s, indices_to_calculate=indices_to_calculate)\n\n\n# del raw\n# gc.collect()\n# test = sample_test_gen()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prepare Cross-Validation"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# keep CV observations from train set\netq_meta = [\n{\"start\":0,         \"end\":5656574},\n{\"start\":5656574,   \"end\":50085878},\n{\"start\":50085878,  \"end\":104677356},\n{\"start\":104677356, \"end\":138772453},\n{\"start\":138772453, \"end\":187641820},\n{\"start\":187641820, \"end\":218652630},\n{\"start\":218652630, \"end\":245829585},\n{\"start\":245829585, \"end\":307838917},\n{\"start\":307838917, \"end\":338276287},\n{\"start\":338276287, \"end\":375377848},\n{\"start\":375377848, \"end\":419368880},\n{\"start\":419368880, \"end\":461811623},\n{\"start\":461811623, \"end\":495800225},\n{\"start\":495800225, \"end\":528777115},\n{\"start\":528777115, \"end\":585568144},\n{\"start\":585568144, \"end\":621985673},\n{\"start\":621985673, \"end\":629145480},\n]\n\nfor i, etq in enumerate(etq_meta):\n    train.loc[(train['start'] + 150_000 >= etq[\"start\"]) & (train['start'] <= etq[\"end\"] - 150_000), \"eq\"] = i\n\n# We are only keeping segments that belong in these earthquakes\n# This is to make the training distribution more like the testing distribution\ntrain_sample = train[train[\"eq\"].isin([2, 7, 0, 4, 11, 13, 9, 1, 14, 10])]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# delete unnecessary files\ndel train\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# reset the index of the final train set\ntrain_sample=train_sample.reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create time since failure target variable\n# This will be used in the NN as an additional objective\ntargets=train_sample[['target','start']]\ntargets['tsf']=targets['target']-targets['target'].shift(1).fillna(0)\ntargets['tsf']=np.where(targets['tsf']>1.5, targets['tsf'], 0)\ntargets['tsf'].iloc[0]=targets['target'].iloc[0]\n\ntemp_max=0\nfor i in tqdm(range(targets.shape[0])):\n    if targets['tsf'].iloc[i]>0:\n        temp_max=targets['tsf'].iloc[i]\n    else:\n        targets['tsf'].iloc[i]=temp_max\n        \ntargets['tsf']=targets['tsf']-targets['target']\n\n# create a flag target variable for TTF<0.5 secs\n# This will be used in the NN as an additional objective\ntarget=targets['target'].copy().values\ntarget[target>=0.5]=1\ntarget[target<0.5]=0\ntarget=1-target\n\ntargets['binary']=target\ndel target\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import submission file\nsubmission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# delete unnecessary columns\ntrain_sample.drop(['start', 'target', 'eq'],axis=1,inplace=True)\ntest.drop(['seg_id'],axis=1,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# We also need to convert test columns from objects to float64\ntest = test.astype('float64')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define your kfold cross validation\n# We used 3 folds, because we did not see improvements with higher folds\n# We are not scared of shuffling because The whole point of this comp is to be independent of time. Test is shuffled\nn_fold = 3\n\nkf = KFold(n_splits=n_fold, shuffle=True, random_state=1337)\nkf = list(kf.split(np.arange(len(train_sample))))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train the LGBM\n\nThe LGBM is trained using only six features. We train using shuffled 3Fold. The LGBM is averaged over ten runs to improve generalization. Hyperparameters were optimized for cross-validation."},{"metadata":{"trusted":true},"cell_type":"code","source":"LGBM_feats = [feat for feat in train_sample.columns if 'LGBM' in feat]\nprint('The features LGBM is using are:', LGBM_feats)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oof_LGBM = np.zeros(len(train_sample))\nsub_LGBM = np.zeros(len(submission))\nseeds = [0,1,2,3,4,5,6,7,8,9]\n\nfor seed in seeds:\n    print('Seed',seed)\n    for fold_n, (train_index, valid_index) in enumerate(kf):\n        print('Fold', fold_n)\n\n        # Create train and validation data using only LGBM_feats.\n        trn_data = lgb.Dataset(train_sample[LGBM_feats].iloc[train_index], label=targets['target'].iloc[train_index])\n        val_data = lgb.Dataset(train_sample[LGBM_feats].iloc[valid_index], label=targets['target'].iloc[valid_index])\n\n        params = {'num_leaves': 4, # Low number of leaves reduces LGBM complexity\n          'min_data_in_leaf': 5,\n          'objective':'fair', # Fitting to fair objective performed better than fitting to MAE objective\n          'max_depth': -1,\n          'learning_rate': 0.01,\n          \"boosting\": \"gbdt\", \n          'boost_from_average': True,\n          \"feature_fraction\": 0.9,\n          \"bagging_freq\": 1,\n          \"bagging_fraction\": 0.5,\n          \"bagging_seed\": 0,\n          \"metric\": 'mae',\n          \"verbosity\": -1,\n          'max_bin': 500,\n          'reg_alpha': 0, \n          'reg_lambda': 0,\n          'seed': seed,\n          'n_jobs': 1\n          }\n\n        clf = lgb.train(params, trn_data, 1000000, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 1000)\n\n        oof_LGBM[valid_index] += clf.predict(train_sample[LGBM_feats].iloc[valid_index], num_iteration=clf.best_iteration)\n        sub_LGBM += clf.predict(test[LGBM_feats], num_iteration=clf.best_iteration) / n_fold\n        \noof_LGBM = oof_LGBM / len(seeds)\nsub_LGBM = sub_LGBM / len(seeds)\n    \nprint('\\nMAE for LGBM: ', mean_absolute_error(targets['target'], oof_LGBM))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train the NN\n\nThe NN is trained using shuffled 3Fold. It is averaged over 8 runs to improve generalization. Sometimes when running the model, the initial weights are bad which results in bad results in cross-validation. If this happens, we will not use it when we average.\n\nThe model is simultaneously fit to three targets: Time to Failure (TTF), Time Since Failure (TSF), and Binary Target for TTF < 0.5 seconds. The loss weights are 8, 1, and 1, respectively. Because the NN has to focus on the TSF and Binary targets, the weights created seem to be better for predicting TTF. Likely, by fitting the NN this way, it reduces overfitting and increases generalization.\n\nWe use Nadam optimizer."},{"metadata":{"trusted":true},"cell_type":"code","source":"NN_feats = [feat for feat in train_sample.columns if 'NN' in feat]\nprint('The features NN is using are:', NN_feats)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Subset columns to only use the neural network features\ntrain_sample = train_sample[NN_feats]\ntest = test[NN_feats]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Define Neural Network architecture\ndef get_model():\n\n    inp = Input(shape=(1,train_sample.shape[1]))\n    x = BatchNormalization()(inp)\n    x = LSTM(128,return_sequences=True)(x) # LSTM as first layer performed better than Dense.\n    x = Convolution1D(128, (2),activation='relu', padding=\"same\")(x)\n    x = Convolution1D(84, (2),activation='relu', padding=\"same\")(x)\n    x = Convolution1D(64, (2),activation='relu', padding=\"same\")(x)\n\n    x = Flatten()(x)\n\n    x = Dense(64, activation=\"relu\")(x)\n    x = Dense(32, activation=\"relu\")(x)\n    \n    #outputs\n    ttf = Dense(1, activation='relu',name='regressor')(x) # Time to Failure\n    tsf = Dense(1)(x) # Time Since Failure\n    classifier = Dense(1, activation='sigmoid')(x) # Binary for TTF<0.5 seconds\n    \n    model = models.Model(inputs=inp, outputs=[ttf,tsf,classifier])    \n    opt = optimizers.Nadam(lr=0.008)\n\n    # We are fitting to 3 targets simultaneously: Time to Failure (TTF), Time Since Failure (TSF), and Binary for TTF<0.5 seconds\n    # We weight the model to optimize heavily for TTF\n    # Optimizing for TSF and Binary TTF<0.5 helps to reduce overfitting, and helps for generalization.\n    model.compile(optimizer=opt, loss=['mae','mae','binary_crossentropy'],loss_weights=[8,1,1],metrics=['mae'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n=8 # number of NN runs\n\noof_final = np.zeros(len(train_sample))\nsub_final = np.zeros(len(submission))\ni=0\n\n\nwhile i<8:\n    print('Running Model ', i+1)\n    \n    oof = np.zeros(len(train_sample))\n    prediction = np.zeros(len(submission))\n\n    for fold_n, (train_index, valid_index) in enumerate(kf):\n        #define training and validation sets\n\n        train_x=train_sample.iloc[train_index] #training set\n        train_y_ttf=targets['target'].iloc[train_index] #training target(Time to Failure)\n\n        valid_x=train_sample.iloc[valid_index] #validation set\n        valid_y_ttf=targets['target'].iloc[valid_index] #validation target(Time to Failure)\n\n        train_y_tsf=targets['tsf'].iloc[train_index] #training target(Time Since Failure)\n        train_y_clf=targets['binary'].iloc[train_index] #training target(Binary for TTF<0.5 Secs)\n\n        valid_y_tsf=targets['tsf'].iloc[valid_index] #validation target(Time Since Failure)\n        valid_y_clf=targets['binary'].iloc[valid_index] #validation target(Binary for TTF<0.5 Secs)\n\n        #apply min max scaler on training, validation data\n        train_x,valid_x,test_scaled=normalize(train_x.copy(), valid_x.copy(), test.copy(), 'min_max', [])\n\n        #Reshape training,validation,test data for fitting\n        train_x=train_x.values.reshape(train_x.shape[0],1,train_x.shape[1])\n        valid_x=valid_x.values.reshape(valid_x.shape[0],1,valid_x.shape[1])\n        test_scaled=test_scaled.values.reshape(test_scaled.shape[0],1,test_scaled.shape[1])\n\n        #obtain Neural Network Instance\n        model=get_model()\n\n        #setup Neural Network callbacks\n        cb_checkpoint = ModelCheckpoint(\"model.hdf5\", monitor='val_regressor_mean_absolute_error', save_weights_only=True,save_best_only=True, period=1)\n        cb_Early_Stop=EarlyStopping( monitor='val_regressor_mean_absolute_error',patience=20)\n        cb_Reduce_LR = ReduceLROnPlateau(monitor='val_regressor_mean_absolute_error', factor=0.5, patience=5, verbose=0, mode='auto', min_delta=0.0001, cooldown=0, min_lr=0)\n\n        callbacks = [cb_checkpoint,cb_Early_Stop,cb_Reduce_LR] #define callbacks set\n        \n        ### NN seeds setup- Start\n        os.environ['PYTHONHASHSEED'] = '0'\n        np.random.seed(1234)\n        rn.seed(1234)\n        tf.set_random_seed(1234)\n        session_conf = tf.ConfigProto( allow_soft_placement=True)\n        sess = tf.Session(graph=tf.get_default_graph(), config=session_conf)\n        K.set_session(sess)\n        ### NN seeds setup- End\n        \n        \n        model.fit(train_x,[train_y_ttf,train_y_tsf,train_y_clf],\n                  epochs=1000,callbacks=callbacks\n                  ,batch_size=256,verbose=0,\n                  validation_data=(valid_x,[valid_y_ttf,valid_y_tsf,valid_y_clf]))\n\n        model.load_weights(\"model.hdf5\")\n        \n        oof[valid_index] += model.predict(valid_x)[0].ravel()\n        prediction += model.predict(test_scaled)[0].ravel()/n_fold\n        \n        K.clear_session()\n        \n    # Obtain the MAE for this run.\n    model_score=mean_absolute_error(targets['target'], oof)\n    \n    # Sometimes, the NN performs very badly. This happens if the weights are initialized poorly.\n    # If the MAE is < 2, then the model has performed correctly, and we will use it in the average.\n    if model_score < 2:\n        print('MAE: ', model_score,' Averaged')\n        oof_final += oof/n\n        sub_final += prediction/n\n        i+=1 # Increase i, so we know that we completed a successful run.\n        \n    # If the MAE is >= 2, then the NN has performed badly.\n    # We will reject this run in the average.\n    else:\n        print('MAE: ', model_score,' Not Averaged')\n\nprint('\\nMAE for NN: ', mean_absolute_error(targets['target'], oof_final))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Combine NN and LGBM using geometric mean\n\nThe geometric mean requires taking product of N oofs, and then taking root(N) of the product. \n\nSince we have two oofs, we multiply the two, then take square root.\n\nWe found the geometric mean to perform slightly better than mean and median."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Square root requires non-negative values, so let us force minima to something small.\nMIN_VALUE = 0.1\n\n# Correct LGBM predictions\noof_LGBM[oof_LGBM < MIN_VALUE] = MIN_VALUE\nsub_LGBM[sub_LGBM < MIN_VALUE] = MIN_VALUE\n\n# Correct NN predictions\noof_final[oof_final < MIN_VALUE] = MIN_VALUE\nsub_final[sub_final < MIN_VALUE] = MIN_VALUE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('MAE for LGBM was: ', mean_absolute_error(targets['target'], oof_LGBM))\nprint('MAE for NN was  : ', mean_absolute_error(targets['target'], oof_final))\n\noof_geomean = (oof_LGBM * oof_final) ** (1/2)\nsub_geomean = (sub_LGBM * sub_final) ** (1/2)\n\nprint('\\nMAE for geometric mean of LGBM and NN was : ', mean_absolute_error(targets['target'], oof_geomean))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Save oof and sub as numpy arrays\nnp.save('oof_LGBM.npy', oof_LGBM)\nnp.save('sub_LGBM.npy', sub_LGBM)\nnp.save('oof_NN.npy', oof_final)\nnp.save('sub_NN.npy', sub_final)\nnp.save('oof_geomean.npy', oof_geomean)\nnp.save('sub_geomean.npy', sub_geomean)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Save out the geometric mean submission\nsubmission['time_to_failure'] = sub_geomean\nprint(submission.head())\nsubmission.to_csv('submission_geomean.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Save solution using just LGBM\nsubmission['time_to_failure'] = sub_LGBM\nprint(submission.head())\nsubmission.to_csv('submission_LGBM.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Save solution using just NN\nsubmission['time_to_failure'] = sub_final\nprint(submission.head())\nsubmission.to_csv('submission_NN.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}