{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Summary\n\n* Adding multiple target values spanning multiple segments.\n\n# Reference\n* [RNN starter for huge time series](https://www.kaggle.com/mayer79/rnn-starter-for-huge-time-series) \n* [RNN starter notebook](https://www.kaggle.com/devilears/rnn-starter-kernel-with-notebook)\n* [Intro to RNN with LSTM and GRU](https://www.kaggle.com/thebrownviking20/intro-to-recurrent-neural-networks-lstm-gru)\n* [Wavelet denoising](https://www.kaggle.com/tarunpaparaju/lanl-earthquake-prediction-signal-denoising)"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nfrom tqdm import tqdm_notebook\nfrom sklearn.linear_model import Lasso, Ridge\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nfrom numpy.random import seed\nseed(802)\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":1,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain = pd.read_csv(\"../input/train.csv\", \n                         dtype={\"acoustic_data\": np.int16, \n                                \"time_to_failure\": np.float32}).values","execution_count":2,"outputs":[{"output_type":"stream","text":"CPU times: user 2min 23s, sys: 12.4 s, total: 2min 35s\nWall time: 2min 36s\n","name":"stdout"}]},{"metadata":{},"cell_type":"markdown","source":"# Feature generation\n\nFor a given ending position \"last_index\", we split the last 150000 values \nof \"x\" into 150 pieces of length 1000 each. So n_steps * step_length should equal 150000.\nFrom each piece, a set features are extracted. This results in a feature matrix of dimension (150 time steps x features).  "},{"metadata":{"trusted":true},"cell_type":"code","source":"## the simplified one\ndef extract_features(z):\n    z_abs = np.abs(z)\n    return np.c_[z.mean(axis=1),\n                 z_abs.max(axis=1),\n                 (z*(z>=0)*(z<=10)).std(axis=1),\n                 np.transpose(np.quantile(z, q=[0.7, 0.95], axis=1)),\n                 np.transpose(np.quantile(z_abs, q=[0.3, 0.95], axis=1))]\n\n\ndef create_X(x, last_index=None, n_steps=150, step_length=1000):\n    if last_index == None:\n        last_index = len(x)\n       \n    assert last_index - n_steps * step_length >= 0\n\n    temp = x[(last_index - n_steps * step_length):last_index].reshape(n_steps, -1)/3\n    \n    # Extracts features of sequences of full length 1000 and some fractions of it\n    return np.c_[extract_features(temp),\n                 extract_features(temp[:, -step_length // 3:]),\n                 extract_features(temp[:, -step_length // 10:])]","execution_count":3,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Some global variables"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nrows = 150_000\nn_features = create_X(train[:rows,0]).shape[1]\nn_targets = 5\nprint(\"The model RNN is based on {0} features and {1} targets.\\n\".format(n_features, n_targets))","execution_count":4,"outputs":[{"output_type":"stream","text":"The model RNN is based on 21 features and 5 targets.\n\nCPU times: user 16 ms, sys: 0 ns, total: 16 ms\nWall time: 81.9 ms\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 32\neps = 0.5 # weights for the second target (time_after_failure)\nn_steps = 150\nstep_length = rows//n_steps\n\nindex_earthquake_start = np.nonzero(np.diff(train[:,1]) > 0)[0] + 1\ncv_earthquake_index = index_earthquake_start[1]","execution_count":5,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(index_earthquake_start)","execution_count":6,"outputs":[{"output_type":"stream","text":"[  5656574  50085878 104677356 138772453 187641820 218652630 245829585\n 307838917 338276287 375377848 419368880 461811623 495800225 528777115\n 585568144 621985673]\n","name":"stdout"}]},{"metadata":{},"cell_type":"markdown","source":"# Data generator\nThe generator endlessly selects `batch_size` ending positions of sub-time series. For each ending position, the `time_to_failure` serves as target, while the features are created by the function `create_X`."},{"metadata":{"trusted":true},"cell_type":"code","source":"def generator(data, min_index=0, max_index=None, batch_size=32, n_steps=150, step_length=1000):\n    if max_index is None:\n        max_index = len(data) - 1\n     \n    while True:\n        # Pick indices of ending positions\n        seg_length = n_steps * step_length\n        index_range = range(min_index + seg_length, max_index, 20000)\n        rows = np.random.choice(index_range, size=batch_size, replace=False)\n#         for limit in index_earthquake_start: \n#             rows = rows[np.logical_not\\\n#                         (np.logical_and(rows>limit, rows<(limit+160000)))]\n         \n        # Initialize feature matrices and targets\n        samples = np.zeros((batch_size, n_steps, n_features))\n        \n        ## adding a sequence of targets or a single target\n        targets = np.zeros((batch_size, n_targets))\n        \n        for j, row in enumerate(rows):\n            samples[j] = create_X(data[:, 0], \n                                  last_index = row, \n                                  n_steps = n_steps,\n                                  step_length = step_length)\n            \n            if n_targets == 1:\n                ## single target\n                targets[j] = data[row - 1, 1]\n            elif n_targets == 2:\n                ## here the training needs to be chosen as the ones after the first earthquake\n                targets[j,0] = data[row - 1, 1]\n                ## time_after_failure \n                taf_idx = index_earthquake_start[np.sum(row > index_earthquake_start) - 1]\n                targets[j,1] = eps*(data[taf_idx, 1] - targets[j,0])\n            elif n_targets > 2:\n                ## multiple targets (preferably odd number)\n                for i in range(n_targets):\n                    # targets are all in one segments\n                    # targets[j,i] = data[row-1-i*(seg_length//n_targets), 1]  \n                    # targets are spanning multiple neighboring segments\n                    targets[j,i] = data[row - 1 - i*(seg_length//2), 1] \n\n        yield samples, targets","execution_count":7,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Initialize generators\ntrain_gen = generator(train, \n                      batch_size=batch_size,\n                      n_steps=n_steps, \n                      step_length=step_length) \n\n# train_gen = generator(train, \n#                       batch_size=batch_size, \n#                       min_index=cv_earthquake_index,\n#                       n_steps=n_steps, \n#                       step_length=step_length)\n\nvalid_gen = generator(train, \n                      batch_size=batch_size, \n                      max_index=cv_earthquake_index-1,\n                      n_steps=n_steps, \n                      step_length=step_length)\n\n# verify the generator\naux, aux2 = next(train_gen)\nprint(aux.shape, aux2.shape)","execution_count":8,"outputs":[{"output_type":"stream","text":"(32, 150, 21) (32, 5)\n","name":"stdout"}]},{"metadata":{},"cell_type":"markdown","source":"# RNN Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.engine.topology import Layer\nfrom keras.layers import Dense, CuDNNGRU, Dropout, LSTM, CuDNNLSTM, Bidirectional, BatchNormalization\nfrom keras.optimizers import adam, RMSprop\nfrom keras import initializers, regularizers, constraints\nfrom keras.callbacks import ModelCheckpoint\nfrom keras import backend as K\n\nfrom tensorflow import set_random_seed\nset_random_seed(1127)","execution_count":9,"outputs":[{"output_type":"stream","text":"Using TensorFlow backend.\n","name":"stderr"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"# The LSTM architecture\nmodel = Sequential()\n\n''' \nLSTM based RNN (GPU)\n'''\n# # First RNN layer\n# model.add(CuDNNLSTM(units=96, return_sequences=True, input_shape=(None,n_features)))\n# model.add(Dropout(0.2))\n\n# # Second LSTM layer\n# model.add(CuDNNLSTM(units=48, return_sequences=True))\n# model.add(Dropout(0.2))\n\n# # Third LSTM layer\n# model.add(CuDNNLSTM(units=48, return_sequences=True))\n# model.add(Dropout(0.2))\n\n# # Fourth LSTM layer\n# model.add(CuDNNLSTM(units=48))\n\n# model.add(Dense(units=n_targets))\n\n\n''' \nGRU based RNN (GPU)\n'''\n# First RNN layer\nmodel.add(CuDNNGRU(units=50, return_sequences=True, input_shape=(None,n_features)))\nmodel.add(Dropout(0.2))\n\n# Second LSTM layer\nmodel.add(CuDNNGRU(units=50, return_sequences=True))\nmodel.add(Dropout(0.2))\n\n# Third LSTM layer\nmodel.add(CuDNNGRU(units=50, return_sequences=True))\nmodel.add(Dropout(0.2))\n\n# Fourth LSTM layer\nmodel.add(CuDNNGRU(units=50))\n# model.add(Dropout(0.2))\n\n\n# The output layer\nmodel.add(Dense(units=n_targets))\n\n\nmodel.summary()","execution_count":11,"outputs":[{"output_type":"stream","text":"WARNING:tensorflow:From /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\nInstructions for updating:\nColocations handled automatically by placer.\nWARNING:tensorflow:From /opt/conda/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\nInstructions for updating:\nPlease use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n_________________________________________________________________\nLayer (type)                 Output Shape              Param #   \n=================================================================\ncu_dnngru_1 (CuDNNGRU)       (None, None, 50)          10950     \n_________________________________________________________________\ndropout_1 (Dropout)          (None, None, 50)          0         \n_________________________________________________________________\ncu_dnngru_2 (CuDNNGRU)       (None, None, 50)          15300     \n_________________________________________________________________\ndropout_2 (Dropout)          (None, None, 50)          0         \n_________________________________________________________________\ncu_dnngru_3 (CuDNNGRU)       (None, None, 50)          15300     \n_________________________________________________________________\ndropout_3 (Dropout)          (None, None, 50)          0         \n_________________________________________________________________\ncu_dnngru_4 (CuDNNGRU)       (None, 50)                15300     \n_________________________________________________________________\ndense_1 (Dense)              (None, 5)                 255       \n=================================================================\nTotal params: 57,105\nTrainable params: 57,105\nNon-trainable params: 0\n_________________________________________________________________\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Compile and fit model\n\ncb = [ModelCheckpoint(\"model.hdf5\", monitor='val_mean_absolute_error',\n                      save_best_only=True, period=3)]\n\nmodel.compile(optimizer = 'rmsprop',\n              loss = 'logcosh',\n              metrics = ['mae'])\n\n# model.compile(optimizer = 'rmsprop',\n#               loss = 'mae')\n\n\nhistory = model.fit_generator(train_gen,\n                              steps_per_epoch=1000,\n                              epochs=50,\n                              verbose=2,\n                              callbacks=cb,\n                              validation_data=valid_gen,\n                              validation_steps=200\n                             )","execution_count":13,"outputs":[{"output_type":"stream","text":"Epoch 1/30\n  34/1000 [>.............................] - ETA: 7:33 - loss: 2.9283 - mean_absolute_error: 3.5516","name":"stdout"},{"output_type":"error","ename":"KeyboardInterrupt","evalue":"","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)","\u001b[0;32m<ipython-input-13-d7ded00bac8d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     18\u001b[0m                               \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcb\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m                               \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalid_gen\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m                               \u001b[0mvalidation_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m200\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     21\u001b[0m                              )\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/keras/legacy/interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     89\u001b[0m                 warnings.warn('Update your `' + object_name + '` call to the ' +\n\u001b[1;32m     90\u001b[0m                               'Keras 2 API: ' + signature, stacklevel=2)\n\u001b[0;32m---> 91\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     92\u001b[0m         \u001b[0mwrapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_function\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m   1416\u001b[0m             \u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1417\u001b[0m             \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1418\u001b[0;31m             initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m   1419\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1420\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0minterfaces\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlegacy_generator_methods_support\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/keras/engine/training_generator.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(model, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m    179\u001b[0m             \u001b[0mbatch_index\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    180\u001b[0m             \u001b[0;32mwhile\u001b[0m \u001b[0msteps_done\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 181\u001b[0;31m                 \u001b[0mgenerator_output\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_generator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    182\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    183\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgenerator_output\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'__len__'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/keras/utils/data_utils.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    683\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    684\u001b[0m             \u001b[0;32mwhile\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_running\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 685\u001b[0;31m                 \u001b[0minputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mqueue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mblock\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    686\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mqueue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtask_done\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    687\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0minputs\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/multiprocessing/pool.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    636\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    637\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 638\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    639\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mready\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    640\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mTimeoutError\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/multiprocessing/pool.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    633\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    634\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 635\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_event\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    636\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    637\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/threading.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    549\u001b[0m             \u001b[0msignaled\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_flag\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    550\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0msignaled\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 551\u001b[0;31m                 \u001b[0msignaled\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_cond\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    552\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0msignaled\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    553\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/threading.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    293\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m    \u001b[0;31m# restore state no matter what (e.g., KeyboardInterrupt)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    294\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 295\u001b[0;31m                 \u001b[0mwaiter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    296\u001b[0m                 \u001b[0mgotit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    297\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mKeyboardInterrupt\u001b[0m: "]}]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Visualize accuracies\n\nloss = history.history['mean_absolute_error']\nval_loss = history.history['val_mean_absolute_error']\n\n# loss = history.history['loss']\n# val_loss = history.history['val_loss']\nepochs = np.asarray(history.epoch) + 1\n    \nplt.plot(epochs, loss, 'bo', label = \"Training MAE\")\nplt.plot(epochs, val_loss, 'b*', label = \"Validation MAE\")\nplt.title(\"Training and validation loss\")\nplt.xlabel(\"Epochs\")\nplt.legend();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"rows = 150000\nnum_segments = int(np.floor(train.shape[0] / rows))\n\ny_tr = np.zeros(num_segments)\ny_tr_pred = np.zeros((num_segments,n_targets))\n\nfor i in tqdm_notebook(range(num_segments)):\n    x = train[i*rows : i*rows+rows, 0]\n    y_tr[i] = train[i*rows+rows-1, 1]\n    y_tr_pred[i] = model.predict(np.expand_dims(create_X(x), 0))[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"The training MAE is {:.7}.\".format(np.abs(y_tr_pred[:,0] - y_tr).mean()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if n_targets>2:\n    time = np.arange(n_targets).reshape(-1,1)\n    y_tr_extrapolated = np.zeros(num_segments)\n\n    for i in tqdm_notebook(range(num_segments)):\n        clf = Ridge(alpha=0.1)\n        clf.fit(time, y_tr_pred[i])\n        if clf.coef_>0:\n            y_tr_extrapolated[i]  = clf.predict(time[:3]).mean()      \n        else:\n            y_tr_extrapolated[i] = y_tr_pred[i,:3].mean()\n    print(\"The training MAE for extrapolation is {:.7}.\"\\\n          .format(np.abs(y_tr_extrapolated - y_tr).mean()))\n    \n    plt.figure(figsize=(18, 6))\n    plt.plot(y_tr, color='g', label='time_to_failure', linewidth = 2)\n    plt.plot(y_tr_extrapolated, color='b', label='RNN extrapolated prediction')\n    plt.legend(loc='best');\n    plt.title('RNN prediction vs ttf');\nelse:\n    plt.figure(figsize=(18, 6))\n    plt.plot(y_tr, color='g', label='time_to_failure', linewidth = 2)\n    plt.plot(y_tr_pred, color='b', label='RNN predictions')\n    plt.legend(loc='best');\n    plt.title('RNN prediction vs ttf');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load submission file\nsubmission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id', dtype={\"time_to_failure\": np.float32})\nx = None\ny_test_pred = np.zeros((len(submission),n_targets))\n\n\n# Load each test data, create the feature matrix, get numeric prediction\nfor i, seg_id in enumerate(tqdm_notebook(submission.index)):\n  #  print(i)\n    seg = pd.read_csv('../input/test/' + seg_id + '.csv')\n    x = seg['acoustic_data'].values[:]\n    y_test_pred[i] = model.predict(np.expand_dims(create_X(x), 0))[0]\n    if n_targets > 2:\n        clf = Ridge(alpha=0.1)\n        clf.fit(time, y_test_pred[i].reshape(-1,1))\n        if clf.coef_>0:\n            y_pred_extrapolated = clf.predict(time[:n_targets-2]).mean()      \n        else:\n            y_pred_extrapolated = y_test_pred[i,:n_targets-2].mean()\n        \n        submission.time_to_failure[i] = y_pred_extrapolated\n    else:\n        submission.time_to_failure[i] = y_test_pred[i]\n\n# Save\nsubmission.to_csv('submission_lstm_extrapolated.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.DataFrame(y_tr_pred).to_csv('y_tr_pred.csv',index=False)\npd.DataFrame(y_test_pred).to_csv('y_test_pred.csv',index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.1"}},"nbformat":4,"nbformat_minor":1}