{"cells":[{"metadata":{},"cell_type":"markdown","source":"This kernel is base on [MichaelMayer's](https://www.kaggle.com/mayer79/rnn-starter-for-huge-time-series) oringinal work here I add more dense layer with activation function relu, set shuffle=True and learning rate to 0.001 to improve score. "},{"metadata":{},"cell_type":"markdown","source":"# BASIC IDEA OF THE KERNEL\n\nThe data consists of a one dimensional time series x with 600 Mio data points. \nAt test time, we will see a time series of length 150'000 to predict the next earthquake.\nThe idea of this kernel is to randomly sample chunks of length 150'000 from x, derive some\nfeatures and use them to update weights of a recurrent neural net with 150'000 / 1000 = 150 time steps. "},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\nimport os\nfrom tqdm import tqdm\n\n# Fix seeds\nfrom numpy.random import seed\nseed(639)\nfrom tensorflow import set_random_seed\nset_random_seed(5944)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%time\nfloat_data = pd.read_csv(\"../input/train.csv\", dtype={\"acoustic_data\": np.float32, \"time_to_failure\": np.float32}).values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Helper function for the data generator. Extracts mean, standard deviation, and quantiles per time step.\n# Can easily be extended. Expects a two dimensional array.\ndef extract_features(z):\n     return np.c_[z.mean(axis=1), \n                  z.min(axis=1),\n                  z.max(axis=1),\n                  z.std(axis=1)]\n\n# For a given ending position \"last_index\", we split the last 150'000 values \n# of \"x\" into 150 pieces of length 1000 each. So n_steps * step_length should equal 150'000.\n# From each piece, a set features are extracted. This results in a feature matrix \n# of dimension (150 time steps x features).  \ndef create_X(x, last_index=None, n_steps=150, step_length=1000):\n    if last_index == None:\n        last_index=len(x)\n       \n    assert last_index - n_steps * step_length >= 0\n\n    # Reshaping and approximate standardization with mean 5 and std 3.\n    temp = (x[(last_index - n_steps * step_length):last_index].reshape(n_steps, -1) - 5 ) / 3\n    \n    # Extracts features of sequences of full length 1000, of the last 100 values and finally also \n    # of the last 10 observations. \n    return np.c_[extract_features(temp),\n                 extract_features(temp[:, -step_length // 10:]),\n                 extract_features(temp[:, -step_length // 100:])]\n\n# Query \"create_X\" to figure out the number of features\nn_features = create_X(float_data[0:150000]).shape[1]\nprint(\"Our RNN is based on %i features\"% n_features)\n    \n# The generator endlessly selects \"batch_size\" ending positions of sub-time series. For each ending position,\n# the \"time_to_failure\" serves as target, while the features are created by the function \"create_X\".\ndef generator(data, min_index=0, max_index=None, batch_size=16, n_steps=150, step_length=1000):\n    if max_index is None:\n        max_index = len(data) - 1\n     \n    while True:\n        # Pick indices of ending positions\n        rows = np.random.randint(min_index + n_steps * step_length, max_index, size=batch_size)\n         \n        # Initialize feature matrices and targets\n        samples = np.zeros((batch_size, n_steps, n_features))\n        targets = np.zeros(batch_size, )\n        \n        for j, row in enumerate(rows):\n            samples[j] = create_X(data[:, 0], last_index=row, n_steps=n_steps, step_length=step_length)\n            targets[j] = data[row - 1, 1]\n        yield samples, targets","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 32\n\n# Position of second (of 16) earthquake. Used to have a clean split\n# between train and validation\nsecond_earthquake = 50085877\nfloat_data[second_earthquake, 1]\n\n# Initialize generators\ntrain_gen = generator(float_data, batch_size=batch_size) # Use this for better score\n# train_gen = generator(float_data, batch_size=batch_size, min_index=second_earthquake + 1)\nvalid_gen = generator(float_data, batch_size=batch_size, max_index=second_earthquake)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Here I add one more dense layer with relu which improve the score<a id=\"3\"></a>"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define model\nfrom keras.models import Sequential\nfrom keras.layers import Dense, CuDNNGRU, CuDNNLSTM, Bidirectional, LSTM, Dropout,BatchNormalization\nfrom keras.optimizers import adam\nfrom keras.callbacks import ModelCheckpoint\nn_steps = 100\n\ncb = [ModelCheckpoint(\"model.hdf5\", save_best_only=True, period=3)]\n\nmodel = Sequential()\nmodel.add(CuDNNGRU(48, input_shape=(None, n_features)))\nmodel.add(Dense(10, activation='relu'))\nmodel.add(Dense(10, activation='relu'))\nmodel.add(Dense(1))\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Here I set shuffle=True and learning rate to 0.001 to improve score<a id=\"3\"></a>"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Compile and fit model, Here I set learning rate to 0.001 which improve the score\nmodel.compile(optimizer=adam(lr=0.001), loss=\"mae\")\n\n# Here I set shuffle=True to improve score\nhist = model.fit_generator(train_gen,\n                              steps_per_epoch=1000,\n                              epochs=30,\n                              verbose=0,\n                              callbacks=cb,\n                              shuffle=True,\n                              validation_data=valid_gen,\n                              validation_steps=200)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visualize loss<a id=\"3\"></a>"},{"metadata":{"trusted":true},"cell_type":"code","source":"loss = hist.history['loss']\nval_loss = hist.history['val_loss']\nepochs = np.asarray(hist.epoch) + 1\n\nplt.figure(figsize=(15,5))\nplt.plot(epochs, loss)\nplt.plot(epochs, val_loss)\nplt.title('Loss over epochs', weight='bold', fontsize=22)\nplt.xlabel('Epochs', fontsize=16)\nplt.ylabel('Loss', fontsize=16)\nplt.legend(['Training loss', 'Validation loss'], fontsize=16)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load submission file\nsubmission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id', dtype={\"time_to_failure\": np.float32})\n\n# Load each test data, create the feature matrix, get numeric prediction\nfor i, seg_id in enumerate(tqdm(submission.index)):\n  #  print(i)\n    seg = pd.read_csv('../input/test/' + seg_id + '.csv')\n    x = seg['acoustic_data'].values\n    submission.time_to_failure[i] = model.predict(np.expand_dims(create_X(x), 0))\n\nsubmission.head()\n\n# Save\nsubmission.to_csv('submission.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Reference\n[MichaelMayer's](https://www.kaggle.com/mayer79/rnn-starter-for-huge-time-series)  \n"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}