{"cells":[{"metadata":{"trusted":true,"_uuid":"e1baa7518f18a7ff1f2767df1b29c051955502ff","_kg_hide-input":true},"cell_type":"code","source":"%matplotlib inline\n\nfrom os import listdir, makedirs\nfrom os.path import isfile, join, basename, splitext, isfile, exists\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm_notebook\n\nimport tensorflow as tf\nimport keras.backend as K\n\nimport keras\nfrom keras.models import Sequential, Model\nfrom keras.layers import Dropout, Dense, Flatten, BatchNormalization\nfrom keras.layers import Convolution1D, ZeroPadding1D, MaxPooling1D, GlobalAveragePooling1D, GlobalMaxPooling1D\nfrom keras.layers import Concatenate, Average, Maximum, CuDNNLSTM, CuDNNGRU, Bidirectional, TimeDistributed\nfrom keras.callbacks import Callback, EarlyStopping, ModelCheckpoint\nfrom keras.engine.input_layer import Input\nfrom keras.models import load_model\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\npd.set_option('precision', 30)\nnp.set_printoptions(precision = 30)\n\nnp.random.seed(7723)\ntf.set_random_seed(1090)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ccfffa3051a10a0eb8c74527bd8d788c49a340f2"},"cell_type":"code","source":"%%time\ntrain_df = pd.read_csv('../input/train.csv', dtype={'acoustic_data': np.int8, 'time_to_failure': np.float32})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"30679acc658f8437d3126d4353068f7d40588a29","_kg_hide-input":false,"_kg_hide-output":false},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"693ff1759a78bcc26123e47b011355b54a8ac6ab"},"cell_type":"code","source":"X_train = train_df.acoustic_data.values\ny_train = train_df.time_to_failure.values","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"10f0836f9ac55fd57430633125ee33293b8c3bab"},"cell_type":"markdown","source":"Find complete segments in the training data (time to failure goes to zero)"},{"metadata":{"trusted":true,"_uuid":"f0f90d3483c4349050cb3e9ada89cfaf974bb0c4"},"cell_type":"code","source":"ends_mask = np.less(y_train[:-1], y_train[1:])\nsegment_ends = np.nonzero(ends_mask)\n\ntrain_segments = []\nstart = 0\nfor end in segment_ends[0]:\n    train_segments.append((start, end))\n    start = end\n    \nprint(train_segments)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e1194a9e1726095db78e14183c971a1dcbc89a1"},"cell_type":"code","source":"plt.title('Segment sizes')\n_ = plt.bar(np.arange(len(train_segments)), [ s[1] - s[0] for s in train_segments])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5ad80d3483b09c18d07bb62229a802d2284cd0e3"},"cell_type":"markdown","source":"The generator samples randomly from the segmens without crossing the boundaries"},{"metadata":{"trusted":true,"_uuid":"389b43c4107c58029e048522b9b29a38662769c8"},"cell_type":"code","source":"class EarthQuakeRandom(keras.utils.Sequence):\n\n    def __init__(self, x, y, x_mean, x_std, segments, ts_length, batch_size, steps_per_epoch):\n        self.x = x\n        self.y = y\n        self.segments = segments\n        self.ts_length = ts_length\n        self.batch_size = batch_size\n        self.steps_per_epoch = steps_per_epoch\n        self.segments_size = np.array([s[1] - s[0] for s in segments])\n        self.segments_p = self.segments_size / self.segments_size.sum()\n        self.x_mean = x_mean\n        self.x_std = x_std\n\n    def get_batch_size(self):\n        return self.batch_size\n\n    def get_ts_length(self):\n        return self.ts_length\n\n    def get_segments(self):\n        return self.segments\n\n    def get_segments_p(self):\n        return self.segments_p\n\n    def get_segments_size(self):\n        return self.segments_size\n\n    def __len__(self):\n        return self.steps_per_epoch\n\n    def __getitem__(self, idx):\n        segment_index = np.random.choice(range(len(self.segments)), p=self.segments_p)\n        segment = self.segments[segment_index]\n        end_indexes = np.random.randint(segment[0] + self.ts_length, segment[1], size=self.batch_size)\n\n        x_batch = np.empty((self.batch_size, self.ts_length))\n        y_batch = np.empty(self.batch_size, )\n\n        for i, end in enumerate(end_indexes):\n            x_batch[i, :] = self.x[end - self.ts_length: end]\n            y_batch[i] = self.y[end - 1]\n            \n        x_batch = (x_batch - self.x_mean)/self.x_std\n\n        return np.expand_dims(x_batch, axis=2), y_batch","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d7985fae8438678b6542b73dc80b64ceb35bde22"},"cell_type":"markdown","source":"We could use any segments for training / validation"},{"metadata":{"trusted":true,"_uuid":"a2287d88c18d90e7f1cf41917265698354b48624"},"cell_type":"code","source":"t_segments = [train_segments[i] for i in [ 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]]\nv_segments = [train_segments[i] for i in [ 0, 1, 2, 3]]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4c2ed95db99225cf1e8226eee68114f31f982c6a"},"cell_type":"markdown","source":"I think it does not make big difference but lets not leak into the validation data and calculate mean and standrad deviation on the training data only."},{"metadata":{"trusted":true,"_uuid":"fa319e6bdca9dd9866856d514fec8ac093cf818d"},"cell_type":"code","source":"x_sum = 0.\ncount = 0\n\nfor s in t_segments:\n    x_sum += X_train[s[0]:s[1]].sum()\n    count += (s[1] - s[0])\n\nX_train_mean = x_sum/count\n\nx2_sum = 0.\nfor s in t_segments:\n    x2_sum += np.power(X_train[s[0]:s[1]] - X_train_mean, 2).sum()\n\nX_train_std =  np.sqrt(x2_sum/count)\n\nprint(X_train_mean, X_train_std)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5cae20837d7405650f9b09891e5e71a156abf883"},"cell_type":"code","source":"train_gen = EarthQuakeRandom(\n    x = X_train, \n    y = y_train,\n    x_mean = X_train_mean, \n    x_std = X_train_std,\n    segments = t_segments,\n    ts_length = 150000,\n    batch_size = 64,\n    steps_per_epoch = 400\n)\n\nvalid_gen = EarthQuakeRandom(\n    x = X_train, \n    y = y_train,\n    x_mean = X_train_mean, \n    x_std = X_train_std,\n    segments = v_segments,\n    ts_length = 150000,\n    batch_size = 64,\n    steps_per_epoch = 400\n)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"70ab7209fe41fac92e9ed89ed0617ce383bd0453"},"cell_type":"markdown","source":"Use convolutional layers to learn the features and reduce the time sequence length "},{"metadata":{"scrolled":false,"trusted":true,"_uuid":"23f507cdee66c43250648a2d86d95c0ab88b0fe6","_kg_hide-input":false,"_kg_hide-output":false},"cell_type":"code","source":"def CnnRnnModel():\n    i = Input(shape = (150000, 1))\n    \n    x = Convolution1D( 8, kernel_size = 10, strides = 10, activation='relu')(i)\n    x = Convolution1D(16, kernel_size = 10, strides = 10, activation='relu')(x)\n    x = Convolution1D(16, kernel_size = 10, strides = 10, activation='relu')(x)\n    x = CuDNNGRU(24, return_sequences = False, return_state = False)(x)\n    y = Dense(1)(x)\n\n    return Model(inputs = [i], outputs = [y])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78f8084ea9e762c4e59bd933abea41704ad99bdd","_kg_hide-input":true},"cell_type":"code","source":"model = CnnRnnModel()\nmodel.compile(loss='mean_absolute_error', optimizer='adam')\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5bed51171db8bf8c5e1731119fd31205bfd80a82"},"cell_type":"markdown","source":"Train the model with early stopping"},{"metadata":{"trusted":true,"_uuid":"e1061c7cde0687450300f516735aad0cc5dbf08f","_kg_hide-input":true},"cell_type":"code","source":"hist = model.fit_generator(\n    generator =  train_gen,\n    epochs = 50, \n    verbose = 0, \n    validation_data = valid_gen,\n    callbacks = [\n        EarlyStopping(monitor='val_loss', patience = 5, verbose = 1),\n        ModelCheckpoint(filepath='cnn_rnn.h5', monitor='val_loss', save_best_only=True, verbose=1)]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b4790925984514e64ca5a9b46de8b309062e0cf"},"cell_type":"code","source":"plt.plot(hist.history['loss'])\nplt.plot(hist.history['val_loss'])\nplt.title('Model loss')\nplt.ylabel('Loss')\nplt.xlabel('Epoch')\n_= plt.legend(['Train', 'Test'], loc='upper left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6660ea2f2eef5750d137f4b18652a35a4ad4a2a6","_kg_hide-input":true},"cell_type":"code","source":"def load_test(ts_length = 150000):\n    base_dir = '../input/test/'\n    test_files = [f for f in listdir(base_dir) if isfile(join(base_dir, f))]\n\n    ts = np.empty([len(test_files), ts_length])\n    ids = []\n    \n    i = 0\n    for f in tqdm_notebook(test_files):\n        ids.append(splitext(f)[0])\n        t_df = pd.read_csv(base_dir + f, dtype={\"acoustic_data\": np.int8})\n        ts[i, :] = t_df['acoustic_data'].values\n        i = i + 1\n\n    return ts, ids","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c3340d6d6ce75585f90c98f1728b1cd664d7f33f"},"cell_type":"markdown","source":"Load and normalize the test data"},{"metadata":{"trusted":true,"_uuid":"0f005579ea08913f4f68a3749bd761df6cef2b1b"},"cell_type":"code","source":"test_data, test_ids = load_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c3b7a864a9f53af142a08883def46c3866c5464"},"cell_type":"code","source":"X_test = ((test_data - X_train_mean)/ X_train_std).astype('float32')\nX_test = np.expand_dims(X_test, 2)\nX_test.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bf9b36929e5228d4d94b3b7ad1b9011bf088ac44"},"cell_type":"markdown","source":"Load best model and predict"},{"metadata":{"trusted":true,"_uuid":"dc801ecd84c9df193753ed5e8d3a62f61965a4ab"},"cell_type":"code","source":"model = load_model('cnn_rnn.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"435449fda2bf96635e67d69f56227e140c4cea99"},"cell_type":"code","source":"y_pred = model.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9aaf9fb44edba5879a75c68820527d9180d2b3c6"},"cell_type":"code","source":"submission_df = pd.DataFrame({'seg_id': test_ids, 'time_to_failure': y_pred[:, 0]})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b9d5c63161f637de2e39b59e8e4d7c2f3049581"},"cell_type":"code","source":"submission_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"94535dfa59bebec7fed4317ad7ced46f77d84c8b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}