{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nDATADIR = \"../input/LANL-Earthquake-Prediction\"\n\nprint(os.listdir(DATADIR))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# second cell\ndf = pd.read_csv(os.path.join(DATADIR, 'train.csv'),\n                 dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class SignalFeatures(object):\n    SEQUENCE_LENGHT = 150_000\n\n    S_MEAN = 4\n    S_STD = 10\n\n    def __init__(self, normalize=True):\n        self.normalize = normalize\n\n    def shape(self):\n        return (self.SEQUENCE_LENGHT, 1)\n\n    def generate(self, df: pd.DataFrame, predict=False):\n        \"\"\" The performance of this function when vectorized is 10x of\n        an iterative loop.\n        \"\"\"\n        X = df['acoustic_data'].values[:, np.newaxis]\n        if self.normalize:\n            X = (X - self.S_MEAN) / self.S_STD\n        if predict:\n            return X\n        y = df['time_to_failure'].iloc[df.shape[0] - 1]\n        return X, np.array([y])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import lanl_generator_py as lanl_generator\n\ndef get_generators(df, folder: lanl_generator.FoldGenerator, index: int):\n    \"\"\" Use 3x the train data with a random offset on the start of the segment.\n        This should maintain a similar distribution between train and test datasets.\n    \"\"\"\n    train_indices, eval_indices = folder[index]\n    gen_train = lanl_generator.SegmentGenerator(\n        df, SignalFeatures(), train_indices, rand_offset=50_000)\n    gen_eval = lanl_generator.SegmentGenerator(\n        df, SignalFeatures(), eval_indices)\n    return gen_train, gen_eval\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import lanl_generator_py as lanl_generator\n\nclasses = lanl_generator.get_lanl_classes()\ndf_segments = lanl_generator.classify_segments(df, classes)\nfolder = lanl_generator.FoldGenerator(df_segments)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm.autonotebook import tqdm\n\ndef generate_data(gen):\n    X_data = []\n    Y_data = []\n    for i in tqdm(range(len(gen))):\n        x, y = gen[i]\n        X_data.append(x)\n        Y_data.append(y)\n\n    return np.vstack(X_data), np.concatenate(Y_data)\n\ndef prediction_error(model, classes, gen_eval):\n    X_eval, Y_eval = generate_data(gen_eval)\n    y_pred = model.predict(X_eval)\n    serr = (y_pred - Y_eval).reshape(-1)\n    \n    err = np.abs(serr)\n    bins = np.digitize(Y_eval, classes)\n    hist = np.bincount(bins.reshape(-1), minlength=classes.size)\n    errhist = np.zeros((classes.size))\n    for e, b in zip(err, bins):\n        errhist[b] += e\n    errhist /= (hist + 1.0e-9)\n\n    hdensity = hist / np.sum(hist)\n    return serr, errhist, hdensity","execution_count":41,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\n\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.models import Model, Sequential\nimport tensorflow.keras.backend as K\n\ndef make_layer(inp, index, filters, kernel_size, strides=1):\n    W_REG = 0.02\n    c = Conv1D(filters, kernel_size, strides=strides,\n               name='layer_{0}_conv'.format(index),\n               activation='relu',\n               kernel_regularizer=keras.regularizers.l2(W_REG))(inp)\n    p = MaxPooling1D(name='layer_{0}_pool'.format(index))(c)\n    d = Dropout(0.1)(p)\n    return d\n\nclass range_initializer(keras.initializers.Initializer):\n    def __init__(self, vmax, vmean):\n        self.vmax = vmax\n        self.vmean = vmean\n\n    def get_config(self):\n        return {'vmax': self.vmax, 'vmean': self.vmean}\n\n    def __call__(self, shape, dtype=None, partition_info=None):\n        if len(shape) != 2 or shape[1] != 1:\n            raise ValueError('Expected shape (N, 1), got ', shape)\n        return np.arange(shape[0])[:, np.newaxis] / shape[0] * self.vmax - self.vmean\n\ndef make_model():\n    W_REG = 0.05\n    inp = Input(shape=SignalFeatures().shape())\n    \n    params = [\n        (16, 16, 4),\n        (32, 16, 4),\n        (48, 16, 4),\n        (64, 8, 2),\n        (32, 8, 2),\n    ]\n\n    layer_in = inp\n    for i, param in enumerate(params):\n        output = make_layer(layer_in, i, *param)\n        layer_in = output\n\n    s = Dense(16,\n              kernel_regularizer=keras.regularizers.l2(W_REG))(output)\n    summary = Flatten()(s)\n\n    last = Dense(32, name='conv-to-ttf', activation='softmax')(summary)\n    out = Dense(1, name='ttf',\n                kernel_initializer=range_initializer(16.0, 5.6),\n                bias_initializer=keras.initializers.Constant(5.6))(last)\n    model = Model(inp, out)\n    model.compile(optimizer=keras.optimizers.Adam(lr=0.001), loss='mae', metrics=['mae'])\n    return model\n\n","execution_count":46,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"K.clear_session()\nmodel = make_model()\nmodel.summary()\nmodel.save('signal-conv.h5')","execution_count":47,"outputs":[{"output_type":"stream","text":"_________________________________________________________________\nLayer (type)                 Output Shape              Param #   \n=================================================================\ninput_1 (InputLayer)         (None, 150000, 1)         0         \n_________________________________________________________________\nlayer_0_conv (Conv1D)        (None, 37497, 16)         272       \n_________________________________________________________________\nlayer_0_pool (MaxPooling1D)  (None, 18748, 16)         0         \n_________________________________________________________________\ndropout (Dropout)            (None, 18748, 16)         0         \n_________________________________________________________________\nlayer_1_conv (Conv1D)        (None, 4684, 32)          8224      \n_________________________________________________________________\nlayer_1_pool (MaxPooling1D)  (None, 2342, 32)          0         \n_________________________________________________________________\ndropout_1 (Dropout)          (None, 2342, 32)          0         \n_________________________________________________________________\nlayer_2_conv (Conv1D)        (None, 582, 48)           24624     \n_________________________________________________________________\nlayer_2_pool (MaxPooling1D)  (None, 291, 48)           0         \n_________________________________________________________________\ndropout_2 (Dropout)          (None, 291, 48)           0         \n_________________________________________________________________\nlayer_3_conv (Conv1D)        (None, 142, 64)           24640     \n_________________________________________________________________\nlayer_3_pool (MaxPooling1D)  (None, 71, 64)            0         \n_________________________________________________________________\ndropout_3 (Dropout)          (None, 71, 64)            0         \n_________________________________________________________________\nlayer_4_conv (Conv1D)        (None, 32, 32)            16416     \n_________________________________________________________________\nlayer_4_pool (MaxPooling1D)  (None, 16, 32)            0         \n_________________________________________________________________\ndropout_4 (Dropout)          (None, 16, 32)            0         \n_________________________________________________________________\ndense (Dense)                (None, 16, 16)            528       \n_________________________________________________________________\nflatten (Flatten)            (None, 256)               0         \n_________________________________________________________________\nconv-to-ttf (Dense)          (None, 32)                8224      \n_________________________________________________________________\nttf (Dense)                  (None, 1)                 33        \n=================================================================\nTotal params: 82,961\nTrainable params: 82,961\nNon-trainable params: 0\n_________________________________________________________________\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"ckpt_filepath=\"signal-conv.{0}.ckpt.hdf5\"\n\ncv_history = []\nerror_measurements = []\n\nfor fold in range(len(folder)):\n    model = make_model()\n    gen_train, gen_eval = get_generators(df, folder, fold)\n    cb_checkpoint = keras.callbacks.ModelCheckpoint(\n        ckpt_filepath.format(fold), monitor='val_loss', verbose=False,\n        save_best_only=True, mode='min')\n\n    cb_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=5)\n\n    history = model.fit_generator(\n        gen_train,\n        validation_data=gen_eval,\n        callbacks=[cb_checkpoint, cb_stop],\n        epochs=50)\n\n    cv_history.append(history)\n\n    measurements = prediction_error(model, classes, gen_eval)\n    error_measurements.append(measurements)","execution_count":null,"outputs":[{"output_type":"stream","text":"Epoch 1/50\n26/26 [==============================] - 2s 88ms/step - loss: 3.4336 - mean_absolute_error: 2.2350\n106/106 [==============================] - 10s 98ms/step - loss: 4.7713 - mean_absolute_error: 2.5850 - val_loss: 3.4336 - val_mean_absolute_error: 2.2350\nEpoch 2/50\n26/26 [==============================] - 2s 83ms/step - loss: 2.9044 - mean_absolute_error: 2.1995: 1s - loss: 2.9290 - mean_absolute_er\n106/106 [==============================] - 9s 89ms/step - loss: 3.3156 - mean_absolute_error: 2.3928 - val_loss: 2.9044 - val_mean_absolute_error: 2.1995\nEpoch 3/50\n26/26 [==============================] - 2s 83ms/step - loss: 2.6640 - mean_absolute_error: 2.1730\n106/106 [==============================] - 9s 90ms/step - loss: 2.8658 - mean_absolute_error: 2.2826 - val_loss: 2.6640 - val_mean_absolute_error: 2.1730\nEpoch 4/50\n26/26 [==============================] - 2s 85ms/step - loss: 2.5183 - mean_absolute_error: 2.1597\n106/106 [==============================] - 10s 90ms/step - loss: 2.7423 - mean_absolute_error: 2.3201 - val_loss: 2.5183 - val_mean_absolute_error: 2.1597\nEpoch 5/50\n26/26 [==============================] - 2s 85ms/step - loss: 2.4361 - mean_absolute_error: 2.1542\n106/106 [==============================] - 10s 90ms/step - loss: 2.5294 - mean_absolute_error: 2.2130 - val_loss: 2.4361 - val_mean_absolute_error: 2.1542\nEpoch 6/50\n26/26 [==============================] - 2s 84ms/step - loss: 2.4659 - mean_absolute_error: 2.2379\n106/106 [==============================] - 10s 90ms/step - loss: 2.4794 - mean_absolute_error: 2.2235 - val_loss: 2.4659 - val_mean_absolute_error: 2.2379\nEpoch 7/50\n26/26 [==============================] - 2s 83ms/step - loss: 2.6825 - mean_absolute_error: 2.4894\n106/106 [==============================] - 10s 90ms/step - loss: 2.4988 - mean_absolute_error: 2.2804 - val_loss: 2.6825 - val_mean_absolute_error: 2.4894\nEpoch 8/50\n26/26 [==============================] - 2s 85ms/step - loss: 2.3125 - mean_absolute_error: 2.1429\n106/106 [==============================] - 10s 93ms/step - loss: 2.4247 - mean_absolute_error: 2.2432 - val_loss: 2.3125 - val_mean_absolute_error: 2.1429\nEpoch 9/50\n26/26 [==============================] - 2s 85ms/step - loss: 2.3047 - mean_absolute_error: 2.1574\n106/106 [==============================] - 10s 91ms/step - loss: 2.3901 - mean_absolute_error: 2.2339 - val_loss: 2.3047 - val_mean_absolute_error: 2.1574\nEpoch 10/50\n26/26 [==============================] - 2s 83ms/step - loss: 2.3011 - mean_absolute_error: 2.1687: 0s - loss: 2.3089 - mean_absolute_err\n106/106 [==============================] - 10s 90ms/step - loss: 2.3392 - mean_absolute_error: 2.1994 - val_loss: 2.3011 - val_mean_absolute_error: 2.1687\nEpoch 11/50\n26/26 [==============================] - 2s 84ms/step - loss: 2.2555 - mean_absolute_error: 2.1360\n106/106 [==============================] - 10s 91ms/step - loss: 2.3631 - mean_absolute_error: 2.2410 - val_loss: 2.2555 - val_mean_absolute_error: 2.1360\nEpoch 12/50\n26/26 [==============================] - 2s 84ms/step - loss: 2.2592 - mean_absolute_error: 2.1510\n106/106 [==============================] - 10s 90ms/step - loss: 2.3132 - mean_absolute_error: 2.1944 - val_loss: 2.2592 - val_mean_absolute_error: 2.1510\nEpoch 13/50\n26/26 [==============================] - 2s 85ms/step - loss: 2.2897 - mean_absolute_error: 2.1929\n106/106 [==============================] - 10s 91ms/step - loss: 2.4038 - mean_absolute_error: 2.2867 - val_loss: 2.2897 - val_mean_absolute_error: 2.1929\nEpoch 14/50\n26/26 [==============================] - 2s 84ms/step - loss: 2.2254 - mean_absolute_error: 2.1342\n106/106 [==============================] - 10s 92ms/step - loss: 2.3538 - mean_absolute_error: 2.2611 - val_loss: 2.2254 - val_mean_absolute_error: 2.1342\nEpoch 15/50\n26/26 [==============================] - 2s 83ms/step - loss: 2.2729 - mean_absolute_error: 2.1829\n106/106 [==============================] - 10s 90ms/step - loss: 2.3080 - mean_absolute_error: 2.2155 - val_loss: 2.2729 - val_mean_absolute_error: 2.1829\nEpoch 16/50\n26/26 [==============================] - 2s 84ms/step - loss: 2.2701 - mean_absolute_error: 2.1860\n106/106 [==============================] - 10s 91ms/step - loss: 2.2931 - mean_absolute_error: 2.2101 - val_loss: 2.2701 - val_mean_absolute_error: 2.1860\nEpoch 17/50\n26/26 [==============================] - 2s 82ms/step - loss: 2.3181 - mean_absolute_error: 2.2359\n106/106 [==============================] - 9s 89ms/step - loss: 2.2858 - mean_absolute_error: 2.1901 - val_loss: 2.3181 - val_mean_absolute_error: 2.2359\nEpoch 18/50\n26/26 [==============================] - 2s 84ms/step - loss: 2.2424 - mean_absolute_error: 2.1617: 0s - loss: 2.2684 - mean_absolute_error\n106/106 [==============================] - 10s 90ms/step - loss: 2.2768 - mean_absolute_error: 2.1915 - val_loss: 2.2424 - val_mean_absolute_error: 2.1617\nEpoch 19/50\n26/26 [==============================] - 2s 83ms/step - loss: 2.3621 - mean_absolute_error: 2.2785: 1s - loss: 2.2812 - mean_absolute_\n106/106 [==============================] - 10s 91ms/step - loss: 2.3387 - mean_absolute_error: 2.2603 - val_loss: 2.3621 - val_mean_absolute_error: 2.2785\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"HBox(children=(IntProgress(value=0, max=26), HTML(value='')))","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"58238b66a78549daa754782ff4f31ade"}},"metadata":{}},{"output_type":"stream","text":"Epoch 1/50\n26/26 [==============================] - 2s 87ms/step - loss: 3.5106 - mean_absolute_error: 2.3533\n106/106 [==============================] - 11s 102ms/step - loss: 4.8845 - mean_absolute_error: 2.6958 - val_loss: 3.5106 - val_mean_absolute_error: 2.3533\nEpoch 2/50\n26/26 [==============================] - 2s 83ms/step - loss: 3.0249 - mean_absolute_error: 2.3782\n106/106 [==============================] - 10s 91ms/step - loss: 3.1824 - mean_absolute_error: 2.3265 - val_loss: 3.0249 - val_mean_absolute_error: 2.3782\nEpoch 3/50\n26/26 [==============================] - 2s 84ms/step - loss: 2.6860 - mean_absolute_error: 2.2432\n106/106 [==============================] - 10s 92ms/step - loss: 2.8530 - mean_absolute_error: 2.3077 - val_loss: 2.6860 - val_mean_absolute_error: 2.2432\nEpoch 4/50\n26/26 [==============================] - 2s 83ms/step - loss: 2.4831 - mean_absolute_error: 2.1412\n106/106 [==============================] - 10s 91ms/step - loss: 2.6144 - mean_absolute_error: 2.2262 - val_loss: 2.4831 - val_mean_absolute_error: 2.1412\nEpoch 5/50\n26/26 [==============================] - 2s 83ms/step - loss: 2.7272 - mean_absolute_error: 2.4602: 0s - loss: 2.8186 - mean_absolute_error:\n106/106 [==============================] - 10s 90ms/step - loss: 2.5432 - mean_absolute_error: 2.2451 - val_loss: 2.7272 - val_mean_absolute_error: 2.4602\nEpoch 6/50\n26/26 [==============================] - 2s 83ms/step - loss: 2.4035 - mean_absolute_error: 2.1705\n106/106 [==============================] - 10s 90ms/step - loss: 2.4640 - mean_absolute_error: 2.2129 - val_loss: 2.4035 - val_mean_absolute_error: 2.1705\nEpoch 7/50\n26/26 [==============================] - 2s 82ms/step - loss: 2.6030 - mean_absolute_error: 2.4076\n106/106 [==============================] - 9s 89ms/step - loss: 2.4180 - mean_absolute_error: 2.2052 - val_loss: 2.6030 - val_mean_absolute_error: 2.4076\nEpoch 8/50\n 61/106 [================>.............] - ETA: 2s - loss: 2.4257 - mean_absolute_error: 2.2316","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm.autonotebook import tqdm\n\n# average the prediction from the multiple folds\n\nsubmission = pd.read_csv(os.path.join(DATADIR, 'sample_submission.csv'), index_col='seg_id', dtype={'time_to_failure': np.float32})\n\nfor fold in range(len(folder)):\n    model.load_weights('signal-conv.{0}.ckpt.hdf5'.format(fold))\n    for seg_id in tqdm(submission.index):\n        seg = pd.read_csv(os.path.join(DATADIR, 'test/' + seg_id + '.csv'))\n        X = SignalFeatures().generate(seg, predict=True)\n        y = model.predict(X[np.newaxis, :])\n        submission.loc[seg_id]['time_to_failure'] += y\n\nsubmission['time_to_failure'] /= len(folder)\nsubmission.to_csv('submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport matplotlib.pyplot as plt\n\ndef show_history(hist, fold):\n    fig, ax1 = plt.subplots()\n    ax1.plot(hist.history['loss'], label='loss')\n    ax1.plot(hist.history['val_loss'], c='g', label='val_loss')\n    ax1.set_title('Model loss - {0}'.format(fold))\n    ax1.set_ylabel('Loss')\n    ax1.set_xlabel('Epoch')\n    if 'mean_absolute_error' in hist.history:\n        ax2 = ax1.twinx()\n        ax2.plot(hist.history['mean_absolute_error'], c='y', label='mae')\n        ax2.plot(hist.history['val_mean_absolute_error'], c='r', label='val_mae')\n        ax2.set_ylabel('MAE')\n    fig.legend()\n    plt.show()\n\nfor i, hist in enumerate(cv_history):\n    show_history(hist, i)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for fold, measurements in enumerate(error_measurements):\n    serr, errhist, hdensity = measurements\n    print(fold)\n    print(' ', '[0, 1.0)', np.mean(errhist[:2]),\n          '[1.0, 8.0)', np.mean(errhist[2:16]),\n          '[8.0, inf)', np.mean(errhist[16:]))\n    print(' ', '[2.0, 4.0)', np.mean(errhist[4:8]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(len(error_measurements), 1, figsize=(8, 16),\n                        sharex=True, sharey=True)\n\ndef show_mae_distribution(ax1, classes, measurements, fold):\n    _, errhist, hdensity = measurements\n    ax1.set_title('MAE per ttf class - {0}'.format(fold))\n    ax1.plot(classes, errhist, label='mae', c='r')\n    ax2 = ax1.twinx()\n    ax2.plot(classes, hdensity, label='density', c='b')\n\nfor i, measurements in enumerate(error_measurements):\n    show_mae_distribution(axes[i], classes, measurements, fold)\n\nfig.legend()\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}