{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\n# Any results you write to the current directory are saved as output.\n\ndf = pd.read_csv(\n    '../input/train.csv',\n    dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport numpy as np\n\ntf.set_random_seed(42)\nnp.random.seed(42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DataTrainTestSplit(object):\n    def __init__(self, df, sample_size, split=0.1):\n        n_samples = df.shape[0] // sample_size\n        perm = np.random.permutation(n_samples)\n        test_samples = int(np.floor(n_samples * split))\n        self.train_slice = perm[:-test_samples]\n        self.test_slice = perm[-test_samples:]\n    def train(self):\n        return self.train_slice\n    def test(self):\n        return self.test_slice\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow import keras\n\nclass FeatureGenerator(keras.utils.Sequence):\n    sample_size = 150_000\n    STRIDES = 4 * 1024\n    NDIMS = 512\n    STEPS = sample_size // STRIDES\n\n    def __init__(self, df, indices, batch_size=32):\n        self.dataframe = df\n        self.indices = indices\n        self.batch_size = batch_size\n\n    def __len__(self):\n        return int((self.indices.shape[0] - 1) / self.batch_size) + 1\n\n    @classmethod\n    def generate(self, x):\n        \"\"\"\n        Return the applitude of the first NDIMS frequencies\n        \"\"\"\n        WINDOW = 4 * 1024\n        data = []\n        for i in range(FeatureGenerator.STEPS):\n            begin = i * FeatureGenerator.STRIDES\n            end = begin + WINDOW\n            wndn = x[begin:end]\n            n = wndn.size\n            yf = np.fft.fft(wndn, n) * 1/n\n            yf = yf[:FeatureGenerator.NDIMS]\n            data.append(2.0 * np.abs(yf))\n        return np.stack(data)\n\n        \n    def __getitem__(self, index: int):\n        s_begin = index * self.batch_size\n        s_end = min(s_begin + self.batch_size, self.indices.shape[0])\n\n        X_list = []\n        y = np.empty((s_end - s_begin))\n        for m in range(s_begin, s_end):\n            begin = self.indices[m] * self.sample_size\n            end = begin + self.sample_size\n            X_list.append(\n                FeatureGenerator.generate(self.dataframe['acoustic_data'].values[begin:end])\n            )\n            y[m - s_begin] = self.dataframe['time_to_failure'].iloc[end - 1]\n\n        X = np.stack(X_list)\n        return X, y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\n\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.models import Model, Sequential\n\ndef make_block(ndims, inp):\n    blocks = []\n    for i in range(ndims // 64):\n        k = Dense(64)(inp)\n        q = Dense(64)(inp)\n        kq = Multiply()([k, q])\n        v = Dense(64, activation='softmax')(kq)\n        blocks.append(v)\n    join = Concatenate(axis=2)(blocks)\n    ff_sum = Add()([inp, join])\n    ff_drop = Dropout(0.1)(ff_sum)\n    return ff_drop\n    \ndef make_model():\n    inp = Input(shape=(FeatureGenerator.STEPS, FeatureGenerator.NDIMS))\n    layer1 = make_block(512, inp)\n    layer1_seq = LSTM(256, return_sequences=True)(layer1)\n    layer2 = make_block(256, layer1_seq)\n    layer3 = LSTM(128)(layer2)\n    outp = Dense(1)(layer3)\n    model = Model(inp, outp)\n    model.compile('adam',\n                  loss=['mse'],\n                  metrics=['mae'])\n    return model\n\nmodel = make_model()\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"filepath=\"model.ckpt.hdf5\"\ncb_checkpoint = keras.callbacks.ModelCheckpoint(\n    filepath, monitor='val_mean_absolute_error',\n    save_best_only=True, mode='min')\n\ncb_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)\n\n# cross-validation\nfor fold in range(5):\n    print('Generating model {0}'.format(fold))\n    slices = DataTrainTestSplit(df, FeatureGenerator.sample_size, split=0.2)\n    model = make_model()\n    history = model.fit_generator(FeatureGenerator(df, slices.train(), batch_size=64),\n                              validation_data=FeatureGenerator(df, slices.test(), batch_size=64),\n                              callbacks=[cb_stop, cb_checkpoint],\n                              epochs=100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport matplotlib.pyplot as plt\n\nplt.figure()\nplt.plot(history.history['loss'], label='loss')\nplt.plot(history.history['val_loss'], label='val_loss')\nplt.legend()\nplt.title('Model loss')\nplt.ylabel('Loss')\nplt.xlabel('Epoch')\nplt.plot()\n\nplt.figure()\nplt.plot(history.history['mean_absolute_error'], label='mae')\nplt.plot(history.history['val_mean_absolute_error'], label='val_mae')\nplt.legend()\nplt.title('Mean Absolute Error')\nplt.ylabel('Error')\nplt.xlabel('Epoch')\nplt.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_weights('model.ckpt.hdf5')\nmodel.evaluate_generator(FeatureGenerator(df, slices.test()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm_notebook\n\nsubmission = pd.read_csv(\n    '../input/sample_submission.csv', index_col='seg_id', dtype={'time_to_failure': np.float32})\n\nfor seg_id in tqdm_notebook(submission.index):\n    seg = pd.read_csv('../input/test/' + seg_id + '.csv')\n    X = FeatureGenerator.generate(seg['acoustic_data'].values)\n    y = model.predict(X[np.newaxis, :])\n    submission.loc[seg_id]['time_to_failure'] = y\nsubmission.to_csv('submission.csv')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}