{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install keras-tcn","execution_count":1,"outputs":[{"output_type":"stream","text":"Collecting keras-tcn\n Downloading https://files.pythonhosted.org/packages/a4/f7/f584a9b82c7c7110165949b3e9f1f6e0859979589eaed2b2bd70747e723a/keras_tcn-2.6.7-py2.py3-none-any.whl\nRequirement already satisfied: keras in /opt/conda/lib/python3.6/site-packages (from keras-tcn) (2.2.4)\nRequirement already satisfied: numpy in /opt/conda/lib/python3.6/site-packages (from keras-tcn) (1.16.2)\nRequirement already satisfied: keras-applications>=1.0.6 in /opt/conda/lib/python3.6/site-packages (from keras->keras-tcn) (1.0.7)\nRequirement already satisfied: keras-preprocessing>=1.0.5 in /opt/conda/lib/python3.6/site-packages (from keras->keras-tcn) (1.0.9)\nRequirement already satisfied: h5py in /opt/conda/lib/python3.6/site-packages (from keras->keras-tcn) (2.9.0)\nRequirement already satisfied: scipy>=0.14 in /opt/conda/lib/python3.6/site-packages (from keras->keras-tcn) (1.1.0)\nRequirement already satisfied: pyyaml in /opt/conda/lib/python3.6/site-packages (from keras->keras-tcn) (3.12)\nRequirement already satisfied: six>=1.9.0 in /opt/conda/lib/python3.6/site-packages (from keras->keras-tcn) (1.12.0)\nInstalling collected packages: keras-tcn\nSuccessfully installed keras-tcn-2.6.7\n","name":"stdout"}]},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"# BASIC IDEA OF THE KERNEL\n\n# The data consists of a one dimensional time series x with 600 Mio data points. \n# At test time, we will see a time series of length 150'000 to predict the next earthquake.\n# The idea of this kernel is to randomly sample chunks of length 150'000 from x, derive some\n# features and use them to update weights of a recurrent neural net with 150'000 / 1000 = 150\n# time steps. \n\nimport numpy as np \nimport pandas as pd\nimport os\nfrom tqdm import tqdm\n\n# Fix seeds\nfrom numpy.random import seed\nseed(639)\nfrom tensorflow import set_random_seed\nset_random_seed(5944)\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":2,"outputs":[{"output_type":"stream","text":"['test', 'train.csv', 'sample_submission.csv']\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#import\nfloat_data = pd.read_csv(\"../input/train.csv\", dtype={\"acoustic_data\": np.float32, \"time_to_failure\": np.float32}).values","execution_count":3,"outputs":[{"output_type":"stream","text":"CPU times: user 2min 16s, sys: 12.8 s, total: 2min 28s\nWall time: 2min 30s\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"float_data.shape","execution_count":4,"outputs":[{"output_type":"execute_result","execution_count":4,"data":{"text/plain":"(629145480, 2)"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Helper function for the data generator. Extracts mean, standard deviation, and quantiles per time step.\n# Can easily be extended. Expects a two dimensional array.\ndef extract_features(z):\n return np.c_[z.mean(axis=1), \n z.min(axis=1),\n z.max(axis=1),\n z.std(axis=1)]\n\n# For a given ending position \"last_index\", we split the last 150'000 values \n# of \"x\" into 150 pieces of length 1000 each. So n_steps * step_length should equal 150'000.\n# From each piece, a set features are extracted. This results in a feature matrix \n# of dimension (150 time steps x features). \ndef create_X(x, last_index=None, n_steps=150, step_length=1000):\n if last_index == None:\n last_index=len(x)\n \n assert last_index - n_steps * step_length >= 0\n\n # Reshaping and approximate standardization with mean 5 and std 3.\n temp = (x[(last_index - n_steps * step_length):last_index].reshape(n_steps, -1) - 5 ) / 3\n \n # Extracts features of sequences of full length 1000, of the last 100 values and finally also \n # of the last 10 observations. \n return np.c_[extract_features(temp),\n extract_features(temp[:, -step_length // 10:]),\n extract_features(temp[:, -step_length // 100:])]\n\n# Query \"create_X\" to figure out the number of features\nn_features = create_X(float_data[0:150000]).shape[1]\nprint(\"Our RNN is based on %i features\"% n_features)\n \n# The generator endlessly selects \"batch_size\" ending positions of sub-time series. For each ending position,\n# the \"time_to_failure\" serves as target, while the features are created by the function \"create_X\".\ndef generator(data, min_index=0, max_index=None, batch_size=16, n_steps=150, step_length=1000):\n if max_index is None:\n max_index = len(data) - 1\n \n while True:\n # Pick indices of ending positions\n rows = np.random.randint(min_index + n_steps * step_length, max_index, size=batch_size)\n \n # Initialize feature matrices and targets\n samples = np.zeros((batch_size, n_steps, n_features))\n targets = np.zeros(batch_size, )\n \n for j, row in enumerate(rows):\n samples[j] = create_X(data[:, 0], last_index=row, n_steps=n_steps, step_length=step_length)\n targets[j] = data[row - 1, 1]\n yield samples, targets\n \nbatch_size = 32\n\n# Position of second (of 16) earthquake. Used to have a clean split\n# between train and validation\nsecond_earthquake = 50085877\nfloat_data[second_earthquake, 1]\n\n# Initialize generators\ntrain_gen = generator(float_data, batch_size=batch_size) # Use this for better score\n# train_gen = generator(float_data, batch_size=batch_size, min_index=second_earthquake + 1)\nvalid_gen = generator(float_data, batch_size=batch_size, max_index=second_earthquake)","execution_count":5,"outputs":[{"output_type":"stream","text":"Our RNN is based on 12 features\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define model\nfrom keras.models import Sequential\nfrom keras.layers import Dense, CuDNNGRU\nfrom keras.optimizers import adam\nfrom keras.callbacks import ModelCheckpoint\nfrom keras.models import Input, Model\nfrom tcn import TCN\ncb = [ModelCheckpoint(\"model.hdf5\", save_best_only=True, period=3)]\n\ni = Input(batch_shape=(batch_size, None, n_features))\n\no = TCN(return_sequences=False,dilations=[1, 2, 4, 8, 16, 32,64])(i)# The TCN layers are here.\no = Dense(16)(o)\no = Dense(1)(o)\n\nmodel = Model(inputs=[i], outputs=[o])\nmodel.compile(optimizer=adam(lr=0.0005), loss=\"mae\")\nmodel.summary()\n\n\n# Compile and fit model\n\n\nhistory = model.fit_generator(train_gen,\n steps_per_epoch=1000,\n epochs=30,\n verbose=0,\n callbacks=cb,\n validation_data=valid_gen,\n validation_steps=200)","execution_count":null,"outputs":[{"output_type":"stream","text":"WARNING:tensorflow:From /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\nInstructions for updating:\nColocations handled automatically by placer.\n","name":"stdout"},{"output_type":"stream","text":"Using TensorFlow backend.\n","name":"stderr"},{"output_type":"stream","text":"__________________________________________________________________________________________________\nLayer (type) Output Shape Param # Connected to \n==================================================================================================\ninput_1 (InputLayer) (32, None, 12) 0 \n__________________________________________________________________________________________________\nconv1d_1 (Conv1D) (32, None, 64) 832 input_1[0][0] \n__________________________________________________________________________________________________\nconv1d_2 (Conv1D) (32, None, 64) 8256 conv1d_1[0][0] \n__________________________________________________________________________________________________\nactivation_1 (Activation) (32, None, 64) 0 conv1d_2[0][0] \n__________________________________________________________________________________________________\nspatial_dropout1d_1 (SpatialDro (32, None, 64) 0 activation_1[0][0] \n__________________________________________________________________________________________________\nconv1d_3 (Conv1D) (32, None, 64) 8256 spatial_dropout1d_1[0][0] \n__________________________________________________________________________________________________\nactivation_2 (Activation) (32, None, 64) 0 conv1d_3[0][0] \n__________________________________________________________________________________________________\nspatial_dropout1d_2 (SpatialDro (32, None, 64) 0 activation_2[0][0] \n__________________________________________________________________________________________________\nconv1d_4 (Conv1D) (32, None, 64) 4160 conv1d_1[0][0] \n__________________________________________________________________________________________________\nadd_1 (Add) (32, None, 64) 0 conv1d_4[0][0] \n spatial_dropout1d_2[0][0] \n__________________________________________________________________________________________________\nconv1d_5 (Conv1D) (32, None, 64) 8256 add_1[0][0] \n__________________________________________________________________________________________________\nactivation_3 (Activation) (32, None, 64) 0 conv1d_5[0][0] \n__________________________________________________________________________________________________\nspatial_dropout1d_3 (SpatialDro (32, None, 64) 0 activation_3[0][0] \n__________________________________________________________________________________________________\nconv1d_6 (Conv1D) (32, None, 64) 8256 spatial_dropout1d_3[0][0] \n__________________________________________________________________________________________________\nactivation_4 (Activation) (32, None, 64) 0 conv1d_6[0][0] \n__________________________________________________________________________________________________\nspatial_dropout1d_4 (SpatialDro (32, None, 64) 0 activation_4[0][0] \n__________________________________________________________________________________________________\nconv1d_7 (Conv1D) (32, None, 64) 4160 add_1[0][0] \n__________________________________________________________________________________________________\nadd_2 (Add) (32, None, 64) 0 conv1d_7[0][0] \n spatial_dropout1d_4[0][0] \n__________________________________________________________________________________________________\nconv1d_8 (Conv1D) (32, None, 64) 8256 add_2[0][0] \n__________________________________________________________________________________________________\nactivation_5 (Activation) (32, None, 64) 0 conv1d_8[0][0] \n__________________________________________________________________________________________________\nspatial_dropout1d_5 (SpatialDro (32, None, 64) 0 activation_5[0][0] \n__________________________________________________________________________________________________\nconv1d_9 (Conv1D) (32, None, 64) 8256 spatial_dropout1d_5[0][0] \n__________________________________________________________________________________________________\nactivation_6 (Activation) (32, None, 64) 0 conv1d_9[0][0] \n__________________________________________________________________________________________________\nspatial_dropout1d_6 (SpatialDro (32, None, 64) 0 activation_6[0][0] \n__________________________________________________________________________________________________\nconv1d_10 (Conv1D) (32, None, 64) 4160 add_2[0][0] \n__________________________________________________________________________________________________\nadd_3 (Add) (32, None, 64) 0 conv1d_10[0][0] \n spatial_dropout1d_6[0][0] \n__________________________________________________________________________________________________\nconv1d_11 (Conv1D) (32, None, 64) 8256 add_3[0][0] \n__________________________________________________________________________________________________\nactivation_7 (Activation) (32, None, 64) 0 conv1d_11[0][0] \n__________________________________________________________________________________________________\nspatial_dropout1d_7 (SpatialDro (32, None, 64) 0 activation_7[0][0] \n__________________________________________________________________________________________________\nconv1d_12 (Conv1D) (32, None, 64) 8256 spatial_dropout1d_7[0][0] \n__________________________________________________________________________________________________\nactivation_8 (Activation) (32, None, 64) 0 conv1d_12[0][0] \n__________________________________________________________________________________________________\nspatial_dropout1d_8 (SpatialDro (32, None, 64) 0 activation_8[0][0] \n__________________________________________________________________________________________________\nconv1d_13 (Conv1D) (32, None, 64) 4160 add_3[0][0] \n__________________________________________________________________________________________________\nadd_4 (Add) (32, None, 64) 0 conv1d_13[0][0] \n spatial_dropout1d_8[0][0] \n__________________________________________________________________________________________________\nconv1d_14 (Conv1D) (32, None, 64) 8256 add_4[0][0] \n__________________________________________________________________________________________________\nactivation_9 (Activation) (32, None, 64) 0 conv1d_14[0][0] \n__________________________________________________________________________________________________\nspatial_dropout1d_9 (SpatialDro (32, None, 64) 0 activation_9[0][0] \n__________________________________________________________________________________________________\nconv1d_15 (Conv1D) (32, None, 64) 8256 spatial_dropout1d_9[0][0] \n__________________________________________________________________________________________________\nactivation_10 (Activation) (32, None, 64) 0 conv1d_15[0][0] \n__________________________________________________________________________________________________\nspatial_dropout1d_10 (SpatialDr (32, None, 64) 0 activation_10[0][0] \n__________________________________________________________________________________________________\nconv1d_16 (Conv1D) (32, None, 64) 4160 add_4[0][0] \n__________________________________________________________________________________________________\nadd_5 (Add) (32, None, 64) 0 conv1d_16[0][0] \n spatial_dropout1d_10[0][0] \n__________________________________________________________________________________________________\nconv1d_17 (Conv1D) (32, None, 64) 8256 add_5[0][0] \n__________________________________________________________________________________________________\nactivation_11 (Activation) (32, None, 64) 0 conv1d_17[0][0] \n__________________________________________________________________________________________________\nspatial_dropout1d_11 (SpatialDr (32, None, 64) 0 activation_11[0][0] \n__________________________________________________________________________________________________\nconv1d_18 (Conv1D) (32, None, 64) 8256 spatial_dropout1d_11[0][0] \n__________________________________________________________________________________________________\nactivation_12 (Activation) (32, None, 64) 0 conv1d_18[0][0] \n__________________________________________________________________________________________________\nspatial_dropout1d_12 (SpatialDr (32, None, 64) 0 activation_12[0][0] \n__________________________________________________________________________________________________\nconv1d_19 (Conv1D) (32, None, 64) 4160 add_5[0][0] \n__________________________________________________________________________________________________\nadd_6 (Add) (32, None, 64) 0 conv1d_19[0][0] \n spatial_dropout1d_12[0][0] \n__________________________________________________________________________________________________\nconv1d_20 (Conv1D) (32, None, 64) 8256 add_6[0][0] \n__________________________________________________________________________________________________\nactivation_13 (Activation) (32, None, 64) 0 conv1d_20[0][0] \n__________________________________________________________________________________________________\nspatial_dropout1d_13 (SpatialDr (32, None, 64) 0 activation_13[0][0] \n__________________________________________________________________________________________________\nconv1d_21 (Conv1D) (32, None, 64) 8256 spatial_dropout1d_13[0][0] \n__________________________________________________________________________________________________\nactivation_14 (Activation) (32, None, 64) 0 conv1d_21[0][0] \n__________________________________________________________________________________________________\nspatial_dropout1d_14 (SpatialDr (32, None, 64) 0 activation_14[0][0] \n__________________________________________________________________________________________________\nadd_8 (Add) (32, None, 64) 0 spatial_dropout1d_2[0][0] \n spatial_dropout1d_4[0][0] \n spatial_dropout1d_6[0][0] \n spatial_dropout1d_8[0][0] \n spatial_dropout1d_10[0][0] \n spatial_dropout1d_12[0][0] \n spatial_dropout1d_14[0][0] \n__________________________________________________________________________________________________\nlambda_1 (Lambda) (32, 64) 0 add_8[0][0] \n__________________________________________________________________________________________________\ndense_1 (Dense) (32, 16) 1040 lambda_1[0][0] \n__________________________________________________________________________________________________\ndense_2 (Dense) (32, 1) 17 dense_1[0][0] \n==================================================================================================\nTotal params: 142,433\nTrainable params: 142,433\nNon-trainable params: 0\n__________________________________________________________________________________________________\nWARNING:tensorflow:From /opt/conda/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\nInstructions for updating:\nUse tf.cast instead.\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Visualize accuracies\nimport matplotlib.pyplot as plt\n\ndef perf_plot(history, what = 'loss'):\n x = history.history[what]\n val_x = history.history['val_' + what]\n epochs = np.asarray(history.epoch) + 1\n \n plt.plot(epochs, x, 'bo', label = \"Training \" + what)\n plt.plot(epochs, val_x, 'b', label = \"Validation \" + what)\n plt.title(\"Training and validation \" + what)\n plt.xlabel(\"Epochs\")\n plt.legend()\n plt.show()\n return None\n\nperf_plot(history)\n\n# Load submission file\nsubmission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id', dtype={\"time_to_failure\": np.float32})\n\n# Load each test data, create the feature matrix, get numeric prediction\nfor i, seg_id in enumerate(tqdm(submission.index)):\n # print(i)\n seg = pd.read_csv('../input/test/' + seg_id + '.csv')\n x = seg['acoustic_data'].values\n submission.time_to_failure[i] = model.predict(np.expand_dims(create_X(x), 0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Save\nsubmission.to_csv('submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}