{"cells":[{"metadata":{"_kg_hide-input":false,"_uuid":"e1baa7518f18a7ff1f2767df1b29c051955502ff","trusted":true},"cell_type":"code","source":"%matplotlib inline\n\nfrom os import listdir, makedirs\nfrom os.path import isfile, join, basename, splitext, isfile, exists\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm_notebook\n\nimport tensorflow as tf\nimport keras.backend as K\nfrom keras import metrics\n\nimport keras\nfrom keras.engine.input_layer import Input\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport random, os, sys\nfrom keras.models import *\nfrom keras.layers import *\nfrom keras.callbacks import *\nfrom keras.initializers import *\nimport tensorflow as tf\nfrom keras.engine.topology import Layer\n\npd.set_option('precision', 30)\nnp.set_printoptions(precision = 30)\n\nnp.random.seed(368)\ntf.set_random_seed(368)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ccfffa3051a10a0eb8c74527bd8d788c49a340f2","trusted":true},"cell_type":"code","source":"%%time\ntrain_df = pd.read_csv('../input/train.csv', dtype={'acoustic_data': np.int8, 'time_to_failure': np.float32})","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"_kg_hide-output":false,"_uuid":"30679acc658f8437d3126d4353068f7d40588a29","trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"693ff1759a78bcc26123e47b011355b54a8ac6ab","trusted":true},"cell_type":"code","source":"X_train = train_df.acoustic_data.values\ny_train = train_df.time_to_failure.values","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"10f0836f9ac55fd57430633125ee33293b8c3bab"},"cell_type":"markdown","source":"Find complete segments in the training data (time to failure goes to zero)"},{"metadata":{"_uuid":"f0f90d3483c4349050cb3e9ada89cfaf974bb0c4","trusted":true},"cell_type":"code","source":"ends_mask = np.less(y_train[:-1], y_train[1:])\nsegment_ends = np.nonzero(ends_mask)\n\ntrain_segments = []\nstart = 0\nfor end in segment_ends[0]:\n    train_segments.append((start, end))\n    start = end\n    \nprint(train_segments)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2e1194a9e1726095db78e14183c971a1dcbc89a1","trusted":true},"cell_type":"code","source":"plt.title('Segment sizes')\n_ = plt.bar(np.arange(len(train_segments)), [ s[1] - s[0] for s in train_segments])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5ad80d3483b09c18d07bb62229a802d2284cd0e3"},"cell_type":"markdown","source":"The generator samples randomly from the segmens without crossing the boundaries"},{"metadata":{"_uuid":"389b43c4107c58029e048522b9b29a38662769c8","trusted":true},"cell_type":"code","source":"class EarthQuakeRandom(keras.utils.Sequence):\n\n    def __init__(self, x, y, x_mean, x_std, segments, ts_length, batch_size, steps_per_epoch):\n        self.x = x\n        self.y = y\n        self.segments = segments\n        self.ts_length = ts_length\n        self.batch_size = batch_size\n        self.steps_per_epoch = steps_per_epoch\n        self.segments_size = np.array([s[1] - s[0] for s in segments])\n        self.segments_p = self.segments_size / self.segments_size.sum()\n        self.x_mean = x_mean\n        self.x_std = x_std\n\n    def get_batch_size(self):\n        return self.batch_size\n\n    def get_ts_length(self):\n        return self.ts_length\n\n    def get_segments(self):\n        return self.segments\n\n    def get_segments_p(self):\n        return self.segments_p\n\n    def get_segments_size(self):\n        return self.segments_size\n\n    def __len__(self):\n        return self.steps_per_epoch\n\n    def __getitem__(self, idx):\n        segment_index = np.random.choice(range(len(self.segments)), p=self.segments_p)\n        segment = self.segments[segment_index]\n        end_indexes = np.random.randint(segment[0] + self.ts_length, segment[1], size=self.batch_size)\n\n        x_batch = np.empty((self.batch_size, self.ts_length))\n        y_batch = np.empty(self.batch_size, )\n\n        for i, end in enumerate(end_indexes):\n            x_batch[i, :] = self.x[end - self.ts_length: end]\n            y_batch[i] = self.y[end - 1]\n            \n        #x_batch = (x_batch - self.x_mean)/self.x_std\n\n        return np.expand_dims(x_batch, axis=2), y_batch","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d7985fae8438678b6542b73dc80b64ceb35bde22"},"cell_type":"markdown","source":"We could use any segments for training / validation"},{"metadata":{"_uuid":"a2287d88c18d90e7f1cf41917265698354b48624","trusted":true},"cell_type":"code","source":"t_segments = [train_segments[i] for i in [ 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]]\nv_segments = [train_segments[i] for i in [ 0, 1, 2, 3]]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4c2ed95db99225cf1e8226eee68114f31f982c6a"},"cell_type":"markdown","source":"I think it does not make big difference but lets not leak into the validation data and calculate mean and standrad deviation on the training data only."},{"metadata":{"_uuid":"fa319e6bdca9dd9866856d514fec8ac093cf818d","trusted":true},"cell_type":"code","source":"x_sum = 0.\ncount = 0\n\nfor s in t_segments:\n    x_sum += X_train[s[0]:s[1]].sum()\n    count += (s[1] - s[0])\n\nX_train_mean = x_sum/count\n\nx2_sum = 0.\nfor s in t_segments:\n    x2_sum += np.power(X_train[s[0]:s[1]] - X_train_mean, 2).sum()\n\nX_train_std =  np.sqrt(x2_sum/count)\n\nprint(X_train_mean, X_train_std)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5cae20837d7405650f9b09891e5e71a156abf883","trusted":true},"cell_type":"code","source":"train_gen = EarthQuakeRandom(\n    x = X_train, \n    y = y_train,\n    x_mean = X_train_mean, \n    x_std = X_train_std,\n    segments = t_segments,\n    ts_length = 150000,\n    batch_size = 64,\n    steps_per_epoch = 400\n)\n\nvalid_gen = EarthQuakeRandom(\n    x = X_train, \n    y = y_train,\n    x_mean = X_train_mean, \n    x_std = X_train_std,\n    segments = v_segments,\n    ts_length = 150000,\n    batch_size = 64,\n    steps_per_epoch = 400\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_gen)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"70ab7209fe41fac92e9ed89ed0617ce383bd0453"},"cell_type":"markdown","source":"Use convolutional layers to learn the features and reduce the time sequence length "},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/shujian/transformer-with-lstm\n\ntry:\n    from dataloader import TokenList, pad_to_longest\n    # for transformer\nexcept: pass\n\n\n\nembed_size = 60\n\nclass LayerNormalization(Layer):\n    def __init__(self, eps=1e-6, **kwargs):\n        self.eps = eps\n        super(LayerNormalization, self).__init__(**kwargs)\n    def build(self, input_shape):\n        self.gamma = self.add_weight(name='gamma', shape=input_shape[-1:],\n                                     initializer=Ones(), trainable=True)\n        self.beta = self.add_weight(name='beta', shape=input_shape[-1:],\n                                    initializer=Zeros(), trainable=True)\n        super(LayerNormalization, self).build(input_shape)\n    def call(self, x):\n        mean = K.mean(x, axis=-1, keepdims=True)\n        std = K.std(x, axis=-1, keepdims=True)\n        return self.gamma * (x - mean) / (std + self.eps) + self.beta\n    def compute_output_shape(self, input_shape):\n        return input_shape\n\nclass ScaledDotProductAttention():\n    def __init__(self, d_model, attn_dropout=0.1):\n        self.temper = np.sqrt(d_model)\n        self.dropout = Dropout(attn_dropout)\n    def __call__(self, q, k, v, mask):\n        attn = Lambda(lambda x:K.batch_dot(x[0],x[1],axes=[2,2])/self.temper)([q, k])\n        if mask is not None:\n            mmask = Lambda(lambda x:(-1e+10)*(1-x))(mask)\n            attn = Add()([attn, mmask])\n        attn = Activation('softmax')(attn)\n        attn = self.dropout(attn)\n        output = Lambda(lambda x:K.batch_dot(x[0], x[1]))([attn, v])\n        return output, attn\n\nclass MultiHeadAttention():\n    # mode 0 - big martixes, faster; mode 1 - more clear implementation\n    def __init__(self, n_head, d_model, d_k, d_v, dropout, mode=0, use_norm=True):\n        self.mode = mode\n        self.n_head = n_head\n        self.d_k = d_k\n        self.d_v = d_v\n        self.dropout = dropout\n        if mode == 0:\n            self.qs_layer = Dense(n_head*d_k, use_bias=False)\n            self.ks_layer = Dense(n_head*d_k, use_bias=False)\n            self.vs_layer = Dense(n_head*d_v, use_bias=False)\n        elif mode == 1:\n            self.qs_layers = []\n            self.ks_layers = []\n            self.vs_layers = []\n            for _ in range(n_head):\n                self.qs_layers.append(TimeDistributed(Dense(d_k, use_bias=False)))\n                self.ks_layers.append(TimeDistributed(Dense(d_k, use_bias=False)))\n                self.vs_layers.append(TimeDistributed(Dense(d_v, use_bias=False)))\n        self.attention = ScaledDotProductAttention(d_model)\n        self.layer_norm = LayerNormalization() if use_norm else None\n        self.w_o = TimeDistributed(Dense(d_model))\n\n    def __call__(self, q, k, v, mask=None):\n        d_k, d_v = self.d_k, self.d_v\n        n_head = self.n_head\n\n        if self.mode == 0:\n            qs = self.qs_layer(q)  # [batch_size, len_q, n_head*d_k]\n            ks = self.ks_layer(k)\n            vs = self.vs_layer(v)\n\n            def reshape1(x):\n                s = tf.shape(x)   # [batch_size, len_q, n_head * d_k]\n                x = tf.reshape(x, [s[0], s[1], n_head, d_k])\n                x = tf.transpose(x, [2, 0, 1, 3])  \n                x = tf.reshape(x, [-1, s[1], d_k])  # [n_head * batch_size, len_q, d_k]\n                return x\n            qs = Lambda(reshape1)(qs)\n            ks = Lambda(reshape1)(ks)\n            vs = Lambda(reshape1)(vs)\n\n            if mask is not None:\n                mask = Lambda(lambda x:K.repeat_elements(x, n_head, 0))(mask)\n            head, attn = self.attention(qs, ks, vs, mask=mask)  \n                \n            def reshape2(x):\n                s = tf.shape(x)   # [n_head * batch_size, len_v, d_v]\n                x = tf.reshape(x, [n_head, -1, s[1], s[2]]) \n                x = tf.transpose(x, [1, 2, 0, 3])\n                x = tf.reshape(x, [-1, s[1], n_head*d_v])  # [batch_size, len_v, n_head * d_v]\n                return x\n            head = Lambda(reshape2)(head)\n        elif self.mode == 1:\n            heads = []; attns = []\n            for i in range(n_head):\n                qs = self.qs_layers[i](q)   \n                ks = self.ks_layers[i](k) \n                vs = self.vs_layers[i](v) \n                head, attn = self.attention(qs, ks, vs, mask)\n                heads.append(head); attns.append(attn)\n            head = Concatenate()(heads) if n_head > 1 else heads[0]\n            attn = Concatenate()(attns) if n_head > 1 else attns[0]\n\n        outputs = self.w_o(head)\n        outputs = Dropout(self.dropout)(outputs)\n        if not self.layer_norm: return outputs, attn\n        # outputs = Add()([outputs, q]) # sl: fix\n        return self.layer_norm(outputs), attn\n\nclass PositionwiseFeedForward():\n    def __init__(self, d_hid, d_inner_hid, dropout=0.1):\n        self.w_1 = Conv1D(d_inner_hid, 1, activation='relu')\n        self.w_2 = Conv1D(d_hid, 1)\n        self.layer_norm = LayerNormalization()\n        self.dropout = Dropout(dropout)\n    def __call__(self, x):\n        output = self.w_1(x) \n        output = self.w_2(output)\n        output = self.dropout(output)\n        output = Add()([output, x])\n        return self.layer_norm(output)\n\nclass EncoderLayer():\n    def __init__(self, d_model, d_inner_hid, n_head, d_k, d_v, dropout=0.1):\n        self.self_att_layer = MultiHeadAttention(n_head, d_model, d_k, d_v, dropout=dropout)\n        self.pos_ffn_layer  = PositionwiseFeedForward(d_model, d_inner_hid, dropout=dropout)\n    def __call__(self, enc_input, mask=None):\n        output, slf_attn = self.self_att_layer(enc_input, enc_input, enc_input, mask=mask)\n        output = self.pos_ffn_layer(output)\n        return output, slf_attn\n\n\ndef GetPosEncodingMatrix(max_len, d_emb):\n    pos_enc = np.array([\n        [pos / np.power(10000, 2 * (j // 2) / d_emb) for j in range(d_emb)] \n        if pos != 0 else np.zeros(d_emb) \n            for pos in range(max_len)\n            ])\n    pos_enc[1:, 0::2] = np.sin(pos_enc[1:, 0::2]) # dim 2i\n    pos_enc[1:, 1::2] = np.cos(pos_enc[1:, 1::2]) # dim 2i+1\n    return pos_enc\n\ndef GetPadMask(q, k):\n    ones = K.expand_dims(K.ones_like(q, 'float32'), -1)\n    mask = K.cast(K.expand_dims(K.not_equal(k, 0), 1), 'float32')\n    mask = K.batch_dot(ones, mask, axes=[2,1])\n    return mask\n\ndef GetSubMask(s):\n    len_s = tf.shape(s)[1]\n    bs = tf.shape(s)[:1]\n    mask = K.cumsum(tf.eye(len_s, batch_shape=bs), 1)\n    return mask\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def CnnTransformerModel():\n    i = Input(shape = (150000, 1))\n    \n    x = BatchNormalization(axis=-1, momentum=0.99, epsilon=0.001, center=True, scale=True, beta_initializer='zeros', gamma_initializer='ones', moving_mean_initializer='zeros', moving_variance_initializer='ones', beta_regularizer=None, gamma_regularizer=None, beta_constraint=None, gamma_constraint=None)(i)\n\n    x = Convolution1D( 8, kernel_size = 10, strides = 10, activation='relu')(x)\n    \n    x = BatchNormalization(axis=-1, momentum=0.99, epsilon=0.001, center=True, scale=True, beta_initializer='zeros', gamma_initializer='ones', moving_mean_initializer='zeros', moving_variance_initializer='ones', beta_regularizer=None, gamma_regularizer=None, beta_constraint=None, gamma_constraint=None)(x)\n\n    x = Convolution1D(16, kernel_size = 10, strides = 10, activation='relu')(x)\n    \n    x = BatchNormalization(axis=-1, momentum=0.99, epsilon=0.001, center=True, scale=True, beta_initializer='zeros', gamma_initializer='ones', moving_mean_initializer='zeros', moving_variance_initializer='ones', beta_regularizer=None, gamma_regularizer=None, beta_constraint=None, gamma_constraint=None)(x)\n\n    x = Convolution1D(32, kernel_size = 10, strides = 10, activation='relu')(x)\n    \n    x = BatchNormalization(axis=-1, momentum=0.99, epsilon=0.001, center=True, scale=True, beta_initializer='zeros', gamma_initializer='ones', moving_mean_initializer='zeros', moving_variance_initializer='ones', beta_regularizer=None, gamma_regularizer=None, beta_constraint=None, gamma_constraint=None)(x)\n\n    x = Convolution1D(64, kernel_size = 10, strides = 10, activation='relu')(x)\n\n    x = Bidirectional(CuDNNLSTM(128, return_sequences = True, return_state = False))(x)\n    \n    x = Bidirectional(CuDNNLSTM(64, return_sequences = True, return_state = False))(x)\n    \n    x, slf_attn = MultiHeadAttention(n_head=5, d_model=300, d_k=64, d_v=64, dropout=0.3)(x, x, x)\n    \n    avg_pool = GlobalAveragePooling1D()(x)\n    \n    avg_pool = Dense(60,activation = 'relu')(avg_pool)\n    \n    y = Dense(1,activation = 'relu')(avg_pool)\n    \n\n    return Model(inputs = [i], outputs = [y])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = CnnTransformerModel()\nmodel.compile(loss='mean_squared_error', optimizer='adam',metrics = ['mean_absolute_error'])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Visualizing The Model Architecture**"},{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.display import SVG\nfrom keras.utils.vis_utils import model_to_dot\nSVG(model_to_dot(model,show_shapes = True).create(prog='dot', format='svg'))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5bed51171db8bf8c5e1731119fd31205bfd80a82"},"cell_type":"markdown","source":"Train the model with early stopping"},{"metadata":{"_kg_hide-input":false,"_kg_hide-output":false,"_uuid":"e1061c7cde0687450300f516735aad0cc5dbf08f","trusted":true},"cell_type":"code","source":"import time\nstart_time = time.time()\nhist = model.fit_generator(\n    generator =  train_gen,\n    epochs = 25, \n    verbose = 1, \n    validation_data = valid_gen,\n)\nprint(\"--- %s seconds ---\" % (time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9b4790925984514e64ca5a9b46de8b309062e0cf","trusted":true},"cell_type":"code","source":"plt.plot(hist.history['loss'])\nplt.plot(hist.history['val_loss'])\nplt.title('Model Mean Squared Error / Loss')\nplt.ylabel('MSE/Loss')\nplt.xlabel('Epoch')\n_= plt.legend(['Train', 'Test'], loc='upper left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(hist.history['mean_absolute_error'])\nplt.plot(hist.history['val_mean_absolute_error'])\nplt.title('Model Mean Absolute Error')\nplt.ylabel('MAE')\nplt.xlabel('Epoch')\n_= plt.legend(['Train', 'Test'], loc='upper left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\ndel train_gen\ndel valid_gen\ndel X_train\ndel y_train\ndel train_df\ngc.collect()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save_weights('./trained_model.h5', overwrite=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c3340d6d6ce75585f90c98f1728b1cd664d7f33f"},"cell_type":"markdown","source":"Load and normalize the test data"},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_test(ts_length = 150000):\n    base_dir = '../input/test/'\n    test_files = [f for f in listdir(base_dir) if isfile(join(base_dir, f))]\n\n    ts = np.empty([len(test_files), ts_length])\n    ids = []\n    \n    i = 0\n    for f in tqdm_notebook(test_files):\n        ids.append(splitext(f)[0])\n        t_df = pd.read_csv(base_dir + f, dtype={\"acoustic_data\": np.int8})\n        ts[i, :] = t_df['acoustic_data'].values\n        i = i + 1\n\n    return ts, ids","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0f005579ea08913f4f68a3749bd761df6cef2b1b","trusted":true},"cell_type":"code","source":"test_data, test_ids = load_test()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0c3b7a864a9f53af142a08883def46c3866c5464","trusted":true},"cell_type":"code","source":"X_test = test_data\nX_test = np.expand_dims(X_test, 2)\nX_test.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bf9b36929e5228d4d94b3b7ad1b9011bf088ac44"},"cell_type":"markdown","source":"Load best model and predict"},{"metadata":{"_uuid":"435449fda2bf96635e67d69f56227e140c4cea99","trusted":true},"cell_type":"code","source":"y_pred = model.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9aaf9fb44edba5879a75c68820527d9180d2b3c6","trusted":true},"cell_type":"code","source":"submission_df = pd.DataFrame({'seg_id': test_ids, 'time_to_failure': y_pred[:, 0]})","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9b9d5c63161f637de2e39b59e8e4d7c2f3049581","trusted":true},"cell_type":"code","source":"submission_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<a href=\"submission.csv\"> Download File </a>\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}