{"cells":[{"metadata":{"id":"e9JaFrY5qvPA","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\ndf = pd.read_csv(\n    '../input/train.csv',\n    dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64})","execution_count":1,"outputs":[]},{"metadata":{"id":"W4QFrMiBq_Ve","colab_type":"text"},"cell_type":"markdown","source":"According to the dataset description, the acoustic data is sampled in blocks of 4k samples. A test data block is 150_000 samples long and doesn't necessarily start / end at a 4k sample threshold."},{"metadata":{"id":"edJTF57frxDJ","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"def fft_analysis(x: np.array, n=None, k=128, order=False) -> np.array:\n    \"\"\"\n    Perform an FFT op on a block of data.\n\n    Returns a tensor with shape (k, 3) where\n      0 - frequency\n      1 - amplitude\n      2 - phase\n    \"\"\"\n    if x.ndim > 1:\n        x = x.reshape(-1)\n    if n is None:\n        n = x.size\n    yf = np.fft.fft(x, n) * 1/n\n    yf = yf[:n//2]\n    if order:\n        sig = np.argsort(np.abs(yf))[-k:][::-1]\n    else:\n        sig = np.argpartition(np.abs(yf), -k)[-k:]\n    Y = np.empty((k, 3))\n    Y[:, 0] = sig # frequencies\n    yf_sig = yf[sig]\n    Y[:, 1] = 2.0 * np.abs(yf_sig) # amplitude\n    Y[:, 2] = np.angle(yf_sig) # phase\n    return Y\n\ndef feature_gen(x: np.array, strides=512, k=128) -> np.array:\n  \"\"\"\n  Given data block of 150_000 values, generate a set of 4k windows over the\n  data and get its FFT data.\n  \"\"\"\n  WINDOW = 4 * 1024\n  data = []\n  for i in range(x.shape[0] // strides):\n    begin = i * strides\n    end = begin + WINDOW\n    data.append(fft_analysis(x[begin:end], k=k))\n  return np.stack(data)","execution_count":2,"outputs":[]},{"metadata":{"id":"8rGOUicStJZ-","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"from tensorflow import keras\n\nclass DataTrainTestSplit(object):\n    def __init__(self, df, sample_size, split=0.1):\n        n_samples = df.shape[0] // sample_size\n        perm = np.random.permutation(n_samples)\n        test_samples = int(np.floor(n_samples * split))\n        self.train_slice = perm[:-test_samples]\n        self.test_slice = perm[-test_samples:]\n    def train(self):\n        return self.train_slice\n    def test(self):\n        return self.test_slice\n        \nclass Generator(keras.utils.Sequence):\n  sample_size = 150_000\n  STRIDES = 2 * 1024\n  NDIMS = 128\n  STEPS = sample_size // STRIDES\n\n  def __init__(self, df, indices, batch_size=32):\n    self.dataframe = df\n    self.indices = indices\n    self.batch_size = batch_size\n\n  def __len__(self):\n    return int((self.indices.shape[0] - 1) / self.batch_size) + 1\n  \n  def __getitem__(self, index: int):\n    s_begin = index * self.batch_size\n    s_end = min(s_begin + self.batch_size, self.indices.shape[0])\n\n    X_list = []\n    y = np.empty((s_end - s_begin))\n    for m in range(s_begin, s_end):\n        begin = self.indices[m] * self.sample_size\n        end = begin + self.sample_size\n        X_list.append(\n            feature_gen(self.dataframe['acoustic_data'].values[begin:end],\n                        strides=Generator.STRIDES, k=Generator.NDIMS)\n        )\n        y[m - s_begin] = self.dataframe['time_to_failure'].iloc[end - 1]\n\n    X = np.stack(X_list)\n    return X, y","execution_count":3,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow import keras\nimport tensorflow.keras.backend as K\n\nclass Decision(keras.layers.Layer):\n    \"\"\"\n    input_shape: (samples, channels)\n    output_shape: (samples, output_dim)\n    \"\"\"\n    def __init__(self, output_dim, **kwargs):\n        self.output_dim = output_dim\n        self.activation = keras.activations.get('relu')\n        super(Decision, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        self.bias = self.add_weight(\n            shape=(self.output_dim,),\n            name='bias',\n            trainable=True)\n\n        self.u = self.add_weight(\n            shape=(int(input_shape[-1]), self.output_dim,),\n            name='u',\n            trainable=True)\n\n        self.v = self.add_weight(\n            shape=(self.output_dim, self.output_dim,),\n            name='v',\n            trainable=True)\n        super(Decision, self).build(input_shape)  # Be sure to call this at the end\n\n    def call(self, x):\n        m = K.dot(self.activation(K.dot(x, self.u)), self.v)\n        output = K.bias_add(m, self.bias, data_format='channels_last')\n        pos = self.activation(output)\n        neg = output - pos\n        return [neg, pos]\n\n\n    def compute_output_shape(self, input_shape):\n        shape = input_shape\n        shape[-1] = self.output_dim\n        return [shape, shape]\n\n","execution_count":4,"outputs":[]},{"metadata":{"id":"PhbKkCw_uTdY","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"import tensorflow as tf\n\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.models import Model, Sequential\n\ndef make_model():\n    inp = Input(shape=(Generator.STEPS, Generator.NDIMS, 3))\n\n    h1_outs = Decision(64)(inp)\n    h2_outs = []\n    for h1 in h1_outs:\n        h2_outs.extend(Decision(16)(h1))\n\n    h2_out = Concatenate(axis=3)(h2_outs)\n\n    h2_mix = Dense(32)(h2_out)\n\n    h3_out = MaxPooling2D((1, Generator.NDIMS))(h2_mix)\n    h3_out = Reshape((Generator.STEPS, 32,))(h3_out)\n\n    common = LSTM(128)(h3_out)\n    mix = Dense(64)(common)\n    outp = Dense(1)(mix)\n\n    return Model(inp, outp)\n\n","execution_count":5,"outputs":[]},{"metadata":{"id":"iR2h1erru__q","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"filepath=\"model.ckpt.hdf5\"\ncb_checkpoint = keras.callbacks.ModelCheckpoint(\n    filepath, monitor='val_mean_absolute_error',\n    save_best_only=True, mode='min')\n\ncb_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)\n\n# cross-validation\nfor fold in range(5):\n    print('Generating model {0}'.format(fold))\n    slices = DataTrainTestSplit(df, Generator.sample_size, split=0.2)\n    model = make_model()\n    model.compile(optimizer='adam', loss='mse', metrics=['mae'])\n    history = model.fit_generator(Generator(df, slices.train(), batch_size=64),\n                              validation_data=Generator(df, slices.test()),\n                              callbacks=[cb_stop, cb_checkpoint],\n                              epochs=100)\n\n","execution_count":6,"outputs":[{"output_type":"stream","text":"WARNING:tensorflow:From /opt/conda/lib/python3.6/site-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\nInstructions for updating:\nColocations handled automatically by placer.\nWARNING:tensorflow:From /opt/conda/lib/python3.6/site-packages/tensorflow/python/keras/utils/losses_utils.py:170: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\nInstructions for updating:\nUse tf.cast instead.\nWARNING:tensorflow:From /opt/conda/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\nInstructions for updating:\nUse tf.cast instead.\n27/27 [==============================] - 32s 1s/step - loss: 12.0071 - mean_absolute_error: 2.7607\n53/53 [==============================] - 96s 2s/step - loss: 14.3663 - mean_absolute_error: 3.0732 - val_loss: 12.0071 - val_mean_absolute_error: 2.7607\n27/27 [==============================] - 30s 1s/step - loss: 10.9597 - mean_absolute_error: 2.6758\n53/53 [==============================] - 92s 2s/step - loss: 13.7336 - mean_absolute_error: 2.9811 - val_loss: 10.9597 - val_mean_absolute_error: 2.6758\n27/27 [==============================] - 30s 1s/step - loss: 13.4582 - mean_absolute_error: 3.0487\n53/53 [==============================] - 91s 2s/step - loss: 14.1997 - mean_absolute_error: 3.0647 - val_loss: 13.4582 - val_mean_absolute_error: 3.0487\n27/27 [==============================] - 29s 1s/step - loss: 12.9907 - mean_absolute_error: 2.9275\n53/53 [==============================] - 89s 2s/step - loss: 14.5127 - mean_absolute_error: 3.1079 - val_loss: 12.9907 - val_mean_absolute_error: 2.9275\n27/27 [==============================] - 31s 1s/step - loss: 12.1758 - mean_absolute_error: 2.9443\n53/53 [==============================] - 92s 2s/step - loss: 16.2384 - mean_absolute_error: 3.2401 - val_loss: 12.1758 - val_mean_absolute_error: 2.9443\n","name":"stdout"}]},{"metadata":{"id":"oNN9HpUBwgGY","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport matplotlib.pyplot as plt\n\nplt.figure()\nplt.plot(history.history['loss'], label='loss')\nplt.plot(history.history['val_loss'], label='val_loss')\nplt.legend()\nplt.title('Model loss')\nplt.ylabel('Loss')\nplt.xlabel('Epoch')\nplt.plot()\n\nplt.figure()\nplt.plot(history.history['mean_absolute_error'], label='mae')\nplt.plot(history.history['val_mean_absolute_error'], label='val_mae')\nplt.legend()\nplt.title('Mean Absolute Error')\nplt.ylabel('Error')\nplt.xlabel('Epoch')\nplt.plot()\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_weights('model.ckpt.hdf5')\nmodel.evaluate_generator(Generator(df, slices.test()))","execution_count":9,"outputs":[{"output_type":"execute_result","execution_count":9,"data":{"text/plain":"[10.499158841592294, 2.6518514]"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm_notebook\n\nsubmission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id', dtype={'time_to_failure': np.float32})\n\nfor seg_id in tqdm_notebook(submission.index):\n    seg = pd.read_csv('../input/test/' + seg_id + '.csv')\n    X = feature_gen(seg['acoustic_data'].values, strides=Generator.STRIDES, k=Generator.NDIMS)\n    y = model.predict(X[np.newaxis, :])\n    submission.loc[seg_id]['time_to_failure'] = y\nsubmission.to_csv('submission.csv')","execution_count":null,"outputs":[]}],"metadata":{"colab":{"name":"LANL.ipynb","version":"0.3.2","provenance":[],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"nbformat":4,"nbformat_minor":1}