{"cells":[{"metadata":{"trusted":true,"_uuid":"e1baa7518f18a7ff1f2767df1b29c051955502ff","_kg_hide-input":false},"cell_type":"code","source":"dropout_rate = 0.05\nl2_prefct = 0.1\nlearning_rate = 0.0001\ntrain_epochs = 250\n%matplotlib inline\n\nfrom os import listdir, makedirs\nfrom os.path import isfile, join, basename, splitext, isfile, exists\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm_notebook\n\nimport tensorflow as tf\nimport keras.backend as K\n\nimport keras\nfrom keras.engine.input_layer import Input\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport random, os, sys\nfrom keras.models import *\nfrom keras.layers import *\nfrom keras.callbacks import *\nfrom keras.optimizers import *\nfrom keras.regularizers import *\nfrom keras.initializers import *\nimport tensorflow as tf\nfrom keras.engine.topology import Layer\n\npd.set_option('precision', 30)\nnp.set_printoptions(precision = 30)\n\nnp.random.seed(368)\ntf.set_random_seed(368)","execution_count":1,"outputs":[{"output_type":"stream","text":"Using TensorFlow backend.\n","name":"stderr"}]},{"metadata":{"trusted":true,"_uuid":"ccfffa3051a10a0eb8c74527bd8d788c49a340f2"},"cell_type":"code","source":"%%time\ntrain_df = pd.read_csv('../input/train.csv', dtype={'acoustic_data': np.int8, 'time_to_failure': np.float32})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"30679acc658f8437d3126d4353068f7d40588a29","_kg_hide-input":false,"_kg_hide-output":false},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"693ff1759a78bcc26123e47b011355b54a8ac6ab"},"cell_type":"code","source":"X_train = train_df.acoustic_data.values\ny_train = train_df.time_to_failure.values","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"10f0836f9ac55fd57430633125ee33293b8c3bab"},"cell_type":"markdown","source":"Find complete segments in the training data (time to failure goes to zero)"},{"metadata":{"trusted":true,"_uuid":"f0f90d3483c4349050cb3e9ada89cfaf974bb0c4"},"cell_type":"code","source":"ends_mask = np.less(y_train[:-1], y_train[1:])\nsegment_ends = np.nonzero(ends_mask)\n\ntrain_segments = []\nstart = 0\nfor end in segment_ends[0]:\n    train_segments.append((start, end))\n    start = end\n    \nprint(train_segments)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e1194a9e1726095db78e14183c971a1dcbc89a1"},"cell_type":"code","source":"plt.title('Segment sizes')\n_ = plt.bar(np.arange(len(train_segments)), [ s[1] - s[0] for s in train_segments])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5ad80d3483b09c18d07bb62229a802d2284cd0e3"},"cell_type":"markdown","source":"The generator samples randomly from the segmens without crossing the boundaries"},{"metadata":{"trusted":true,"_uuid":"389b43c4107c58029e048522b9b29a38662769c8"},"cell_type":"code","source":"class EarthQuakeRandom(keras.utils.Sequence):\n\n    def __init__(self, x, y, x_mean, x_std, segments, ts_length, batch_size, steps_per_epoch):\n        self.x = x\n        self.y = y\n        self.segments = segments\n        self.ts_length = ts_length\n        self.batch_size = batch_size\n        self.steps_per_epoch = steps_per_epoch\n        self.segments_size = np.array([s[1] - s[0] for s in segments])\n        self.segments_p = self.segments_size / self.segments_size.sum()\n        self.x_mean = x_mean\n        self.x_std = x_std\n\n    def get_batch_size(self):\n        return self.batch_size\n\n    def get_ts_length(self):\n        return self.ts_length\n\n    def get_segments(self):\n        return self.segments\n\n    def get_segments_p(self):\n        return self.segments_p\n\n    def get_segments_size(self):\n        return self.segments_size\n\n    def __len__(self):\n        return self.steps_per_epoch\n\n    def __getitem__(self, idx):\n        segment_index = np.random.choice(range(len(self.segments)), p=self.segments_p)\n        segment = self.segments[segment_index]\n        end_indexes = np.random.randint(segment[0] + self.ts_length, segment[1], size=self.batch_size)\n\n        x_batch = np.empty((self.batch_size, self.ts_length))\n        y_batch = np.empty(self.batch_size, )\n\n        for i, end in enumerate(end_indexes):\n            x_batch[i, :] = self.x[end - self.ts_length: end]\n            y_batch[i] = self.y[end - 1]\n            \n        x_batch = (x_batch - self.x_mean)/self.x_std\n\n        return np.expand_dims(x_batch, axis=2), y_batch","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d7985fae8438678b6542b73dc80b64ceb35bde22"},"cell_type":"markdown","source":"We could use any segments for training / validation"},{"metadata":{"trusted":true,"_uuid":"a2287d88c18d90e7f1cf41917265698354b48624"},"cell_type":"code","source":"t_segments = [train_segments[i] for i in [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]]\nv_segments = [train_segments[i] for i in [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4c2ed95db99225cf1e8226eee68114f31f982c6a"},"cell_type":"markdown","source":"I think it does not make big difference but lets not leak into the validation data and calculate mean and standrad deviation on the training data only."},{"metadata":{"trusted":true,"_uuid":"fa319e6bdca9dd9866856d514fec8ac093cf818d"},"cell_type":"code","source":"x_sum = 0.\ncount = 0\n\nfor s in t_segments:\n    x_sum += X_train[s[0]:s[1]].sum()\n    count += (s[1] - s[0])\n\nX_train_mean = x_sum/count\n\nx2_sum = 0.\nfor s in t_segments:\n    x2_sum += np.power(X_train[s[0]:s[1]] - X_train_mean, 2).sum()\n\nX_train_std =  np.sqrt(x2_sum/count)\n\nprint(X_train_mean, X_train_std)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5cae20837d7405650f9b09891e5e71a156abf883"},"cell_type":"code","source":"train_gen = EarthQuakeRandom(\n    x = X_train, \n    y = y_train,\n    x_mean = X_train_mean, \n    x_std = X_train_std,\n    segments = t_segments,\n    ts_length = 150000,\n    batch_size = 16,\n    steps_per_epoch = 400\n)\n\nvalid_gen = EarthQuakeRandom(\n    x = X_train, \n    y = y_train,\n    x_mean = X_train_mean, \n    x_std = X_train_std,\n    segments = v_segments,\n    ts_length = 150000,\n    batch_size = 16,\n    steps_per_epoch = 400\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_gen)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"70ab7209fe41fac92e9ed89ed0617ce383bd0453"},"cell_type":"markdown","source":"Use convolutional layers to learn the features and reduce the time sequence length "},{"metadata":{"trusted":true},"cell_type":"code","source":"def Xeption_1D_Model(dropout_rate):\n    i = Input(shape = (150000,1))\n    \n    x = i\n    for ii in range(3):\n        x = Convolution1D(64, 7, strides = 4, activation='relu', padding='valid')(x)\n        x = BatchNormalization()(x)\n        x = Dropout(dropout_rate)(x)\n\n    #return Model(inputs = [i], outputs = [x])\n    \n    residual = Convolution1D(64, 1, strides = 4, padding='same')(x)\n    residual = Dropout(dropout_rate)(residual)\n    residual = BatchNormalization()(residual)\n    \n    x = Convolution1D(64, 7, strides = 1, padding='same')(x)\n    x = Dropout(dropout_rate)(x)\n    x = BatchNormalization()(x)\n    x = Activation('relu')(x)\n    x = SeparableConv1D(64, 7, strides = 1, padding='same')(x)\n    x = Dropout(dropout_rate)(x)\n    x = BatchNormalization()(x)\n\n    x = MaxPooling1D(7, strides=4, padding='same')(x)\n    x = Dropout(dropout_rate)(x)\n    x = keras.layers.add([x, residual])\n    \n    #return Model(inputs = [i], outputs = [x])\n    \n    residual = Convolution1D(128, 1, strides = 4, padding='same')(x)\n    residual = Dropout(dropout_rate)(residual)\n    residual = BatchNormalization()(residual)\n    \n    x = Convolution1D(128, 7, strides = 1, padding='same')(x)\n    x = Dropout(dropout_rate)(x)\n    x = BatchNormalization()(x)\n    x = Activation('relu')(x)\n    x = SeparableConv1D(128, 7, strides = 1, padding='same')(x)\n    x = Dropout(dropout_rate)(x)\n    x = BatchNormalization()(x)\n\n    x = MaxPooling1D(7, strides=4, padding='same')(x)\n    x = Dropout(dropout_rate)(x)\n    x = keras.layers.add([x, residual])\n    \n    #return Model(inputs = [i], outputs = [x])\n    \n    \n    for ii in range(8):\n        residual = Convolution1D(192, 1, strides = 1, padding='same')(x)\n        residual = Dropout(dropout_rate)(residual)\n        residual = BatchNormalization()(residual)\n        \n        x = Activation('relu')(x)\n        x = SeparableConv1D(192, 7, strides = 1, padding='same')(x)\n        x = Dropout(dropout_rate)(x)\n        x = BatchNormalization()(x)\n        \n        x = Activation('relu')(x)\n        x = SeparableConv1D(192, 7, strides = 1, padding='same')(x)\n        x = Dropout(dropout_rate)(x)\n        x = BatchNormalization()(x)\n        \n        x = Activation('relu')(x)\n        x = SeparableConv1D(192, 7, strides = 1, padding='same')(x)\n        x = Dropout(dropout_rate)(x)\n        x = BatchNormalization()(x)\n        x = keras.layers.add([x, residual])\n        \n    #return Model(inputs = [i], outputs = [x])\n    \n    residual = Convolution1D(384, 1, strides = 4, padding='same')(x)\n    residual = Dropout(dropout_rate)(residual)\n    residual = BatchNormalization()(residual)\n    \n    x = SeparableConv1D(384, 7, strides = 1, padding='same')(x)\n    x = Dropout(dropout_rate)(x)\n    x = BatchNormalization()(x)\n    x = Activation('relu')(x)\n    x = SeparableConv1D(384, 7, strides = 1, padding='same')(x)\n    x = Dropout(dropout_rate)(x)\n    x = BatchNormalization()(x)\n\n    x = MaxPooling1D(7, strides=4, padding='same')(x)\n    x = Dropout(dropout_rate)(x)\n    x = keras.layers.add([x, residual])\n    \n    #return Model(inputs = [i], outputs = [x])\n    \n    residual = Convolution1D(512, 1, strides = 4, padding='same')(x)\n    residual = Dropout(dropout_rate)(residual)\n    residual = BatchNormalization()(residual)\n    \n    x = SeparableConv1D(512, 7, strides = 1, padding='same')(x)\n    x = Dropout(dropout_rate)(x)\n    x = BatchNormalization()(x)\n    x = Activation('relu')(x)\n    x = SeparableConv1D(512, 7, strides = 1, padding='same')(x)\n    x = Dropout(dropout_rate)(x)\n    x = BatchNormalization()(x)\n\n    x = MaxPooling1D(7, strides=4, padding='same')(x)\n    x = Dropout(dropout_rate)(x)\n    x = keras.layers.add([x, residual])\n    \n    #return Model(inputs = [i], outputs = [x])\n    \n    x = GlobalAveragePooling1D()(x)\n    x = Dense(1000, activation='relu')(x)\n    x = Dropout(dropout_rate)(x)\n\n    x = Dense(1000, activation='relu')(x)\n    x = Dropout(dropout_rate)(x)\n\n    x = Dense(1)(x)\n    \n    return Model(inputs = [i], outputs = [x])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78f8084ea9e762c4e59bd933abea41704ad99bdd","_kg_hide-input":false,"_kg_hide-output":false},"cell_type":"code","source":"model = Xeption_1D_Model(dropout_rate)\nfor layer in model.layers:\n    layer.W_regularizer = l2(l2_prefct)\n#adam = Adam(lr=0.001)\nadam = Adam(lr=learning_rate)\n\nmodel.compile(loss='mean_absolute_error', optimizer=adam)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5bed51171db8bf8c5e1731119fd31205bfd80a82"},"cell_type":"markdown","source":"Train the model with early stopping"},{"metadata":{"trusted":true,"_uuid":"e1061c7cde0687450300f516735aad0cc5dbf08f","_kg_hide-input":false,"_kg_hide-output":false},"cell_type":"code","source":"import time\nstart_time = time.time()\nhist = model.fit_generator(\n    generator =  train_gen,\n    epochs = train_epochs, \n    verbose = 1, \n    validation_data = valid_gen,\n    callbacks = [\n        EarlyStopping(monitor='val_loss', patience = 400, verbose = 1)\n    ]\n)\nprint(\"--- %s seconds ---\" % (time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b4790925984514e64ca5a9b46de8b309062e0cf"},"cell_type":"code","source":"plt.plot(hist.history['loss'])\nplt.plot(hist.history['val_loss'])\nplt.title('Model loss')\nplt.ylabel('Loss')\nplt.xlabel('Epoch')\n_= plt.legend(['Train', 'Test'], loc='upper left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\ndel train_gen\ndel valid_gen\ndel X_train\ndel y_train\ndel train_df\ngc.collect()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save_weights('./trained_model.h5', overwrite=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c3340d6d6ce75585f90c98f1728b1cd664d7f33f"},"cell_type":"markdown","source":"Load and normalize the test data"},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_test(ts_length = 150000):\n    base_dir = '../input/test/'\n    test_files = [f for f in listdir(base_dir) if isfile(join(base_dir, f))]\n\n    ts = np.empty([len(test_files), ts_length])\n    ids = []\n    \n    i = 0\n    for f in tqdm_notebook(test_files):\n        ids.append(splitext(f)[0])\n        t_df = pd.read_csv(base_dir + f, dtype={\"acoustic_data\": np.int8})\n        ts[i, :] = t_df['acoustic_data'].values\n        i = i + 1\n\n    return ts, ids","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f005579ea08913f4f68a3749bd761df6cef2b1b"},"cell_type":"code","source":"test_data, test_ids = load_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c3b7a864a9f53af142a08883def46c3866c5464"},"cell_type":"code","source":"X_test = ((test_data - X_train_mean)/ X_train_std)\nX_test = np.expand_dims(X_test, 2)\nX_test.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bf9b36929e5228d4d94b3b7ad1b9011bf088ac44"},"cell_type":"markdown","source":"Load best model and predict"},{"metadata":{"trusted":true,"_uuid":"435449fda2bf96635e67d69f56227e140c4cea99"},"cell_type":"code","source":"y_pred = model.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9aaf9fb44edba5879a75c68820527d9180d2b3c6"},"cell_type":"code","source":"submission_df = pd.DataFrame({'seg_id': test_ids, 'time_to_failure': y_pred[:, 0]})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b9d5c63161f637de2e39b59e8e4d7c2f3049581"},"cell_type":"code","source":"submission_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}