{"cells":[{"metadata":{"_uuid":"7e8bd7e097f4f8123631786e833b00b39b163d50"},"cell_type":"markdown","source":"Los Alamos National Laboratory - Earthquake analysis\n------------------------------------------------------------------------------"},{"metadata":{"_uuid":"5b691752a2a5fcd0131deb7d750bd940fe49d2ad","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sb\nimport os\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7f7faedc6f7962eef4bec04192e227de74ea6833","trusted":true},"cell_type":"code","source":"def extract_features(df):\n    # container\n    data = []\n    \n    # features from the acoustic data set only, since the segments contain nothing else\n    max_acoustic = df['acoustic_data'].max()\n    mean_acoustic = df['acoustic_data'].mean()\n    \n    data.append([mean_acoustic])\n    data.append([df['acoustic_data'].std()])\n    data.append([max_acoustic])\n    data.append([df['acoustic_data'].min()])\n    \n    #number of `peaks` -> above mean + (max-mean)/2 -> any value above (max + mean)/2\n    signal_values = df['acoustic_data'].loc[df['acoustic_data'] > (max_acoustic + mean_acoustic) / 2.]\n    \n    signal_values = np.array(signal_values)\n    \n    data.append([signal_values.shape[0]]) # number of peaks\n    \n    data.append(np.correlate(df['acoustic_data'].values[::1000],\n                       df['acoustic_data'].values[::1000], mode='same')) # auto-correlate 0.01 % of the data\n                                                                         # to see how self-similair it is\n                                             \n    acoustic_histo = np.histogram(df['acoustic_data'], bins=75)\n    data.append(acoustic_histo[0]) # bins\n    data.append(acoustic_histo[0]) # values\n    \n    data.append(np.abs(np.fft.fft(df['acoustic_data'].values[::1000], n=100)))\n    \n    # we must flatten out the features\n    return [item for sublist in data for item in sublist]","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"_uuid":"b3e7c91b976462d558f5908317a627a0d4bf4d11","trusted":true},"cell_type":"code","source":"TextFileReader = pd.read_csv('../input/train.csv', chunksize=150000) # the segment files contain 150000 lines each!\n\nreduced_data = dict()\ncounter = 0\n\nfor df in TextFileReader:\n    reduced_data[counter] = dict()\n    last_time_to_failure = df['time_to_failure'].values[::-1][0]\n    reduced_data[counter][last_time_to_failure] = extract_features(df)\n    counter += 1\n    if counter % 250 == 0: print('%d segments - done.' % counter)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"be0e6cde891ea6d6b38b2650fc0ab905cacc766d","trusted":true},"cell_type":"code","source":"TextFileReader = pd.read_csv('../input/train.csv', chunksize=150000, skiprows=25000)\n\nfor df in TextFileReader:\n    df.columns = ['acoustic_data', 'time_to_failure']\n    reduced_data[counter] = dict()\n    last_time_to_failure = df['time_to_failure'].values[::-1][0]\n    reduced_data[counter][last_time_to_failure] = extract_features(df)\n    counter += 1\n    if counter % 250 == 0: print('%d segments - done.' % counter)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"93e3833f70f226d26969ac54509ef55b5f51678f","trusted":true},"cell_type":"code","source":"TextFileReader = pd.read_csv('../input/train.csv', chunksize=150000, skiprows=75000)\n\nfor df in TextFileReader:\n    df.columns = ['acoustic_data', 'time_to_failure']\n    reduced_data[counter] = dict()\n    last_time_to_failure = df['time_to_failure'].values[::-1][0]\n    reduced_data[counter][last_time_to_failure] = extract_features(df)\n    counter += 1\n    if counter % 250 == 0: print('%d segments - done.' % counter)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"be9fff3942839c8b35f0fd5c2057f652597a5e71","trusted":true},"cell_type":"code","source":"len(reduced_data) # number of segments achieved that we could predict on!","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b20f8ad79ff02077785511d807e08be87393a009","trusted":true},"cell_type":"code","source":"dataframes = []\n\nfor index in range(len(reduced_data)):\n    df = pd.DataFrame.from_dict(reduced_data[index], orient='index')\n    df['_id'] = index\n    df['ttf'] = df.index\n    df.set_index('_id', inplace=True)\n    dataframes.append(df)\n    \ndel reduced_data","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"32c82a72f2bae65c7e3eab2112f8775d0d538de7","trusted":true},"cell_type":"code","source":"for df in dataframes:\n    df.to_csv('df_all.csv', mode='a', header=False, index=False)\n\ndel dataframes","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"336d223c551420341ad0b31392a3dcd6c9789688","trusted":true},"cell_type":"code","source":"train = pd.read_csv('df_all.csv', header=None)\nos.remove('df_all.csv')\n\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6b02941e41e61da653c425aece7aa8c3c086a3fd","trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import normalize","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"46932eecb7fd5c61f75ca423709bf554a59a6d0f","trusted":true},"cell_type":"code","source":"train = train.dropna()\ntrain.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ec7811b52f8fedd7c308a52aa3dca1af25e74e8a","trusted":true},"cell_type":"code","source":"X = normalize(train.values[:, :405])\ny = train.values[:, 405]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5383896a38c98f5c12fa221cb811720fce28819b"},"cell_type":"code","source":"del train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\ntf.enable_eager_execution()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_ = np.reshape(X, (X.shape[0], 1, X.shape[1]))\n\ndataset = tf.data.Dataset.from_tensor_slices((X_, y))\nsequences = dataset.batch(1, drop_remainder=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for seq, target in sequences.take(1):\n    print(seq.shape, target)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 6\n\nBUFFER_SIZE = 20000\n\ndataset = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE, drop_remainder=True)\n\ndataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if tf.test.is_gpu_available():\n    rnn = tf.keras.layers.CuDNNGRU\nelse:\n    import functools\n    rnn = functools.partial(\n        tf.keras.layers.GRU, recurrent_activation='sigmoid')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(rnn_units, batch_size):\n    model = tf.keras.models.Sequential([\n        tf.keras.layers.InputLayer(input_shape=(1, 405), batch_size=batch_size),\n        tf.keras.layers.Dense(1024, activation='relu'),\n        rnn(rnn_units),\n        tf.keras.layers.Dense(256, activation='relu'),\n        tf.keras.layers.Dense(1024, activation='relu'),\n        tf.keras.layers.Dense(1, activation='relu')\n    ])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = build_model(1024, BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def loss(labels, logits):\n    return tf.keras.losses.MSE(labels, logits)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(\n    optimizer = tf.train.AdamOptimizer(learning_rate=0.001),\n    loss = loss)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"EPOCHS = 35\n\nsamples_per_epoch = X.shape[0]\nsteps_per_epoch = samples_per_epoch // BATCH_SIZE\n\nhistory = model.fit(dataset.repeat(), epochs=EPOCHS, steps_per_epoch=steps_per_epoch)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_ = build_model(rnn_units=1024, batch_size=1)\n\nweights = model.get_weights()\n\nmodel_.set_weights(weights)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_files = os.listdir('../input/test/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result = dict()\nfor file in test_files:\n    \n    df = pd.read_csv('../input/test/' + file)\n    \n    data = np.array(extract_features(df))\n\n    X_test = normalize(data.reshape(1, -1))\n    X_test = X_test.reshape(1, 1, 405)\n    prediction = model_.predict(X_test)[0]\n    result[file[::-1][4:][::-1]] = prediction","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result_df = pd.DataFrame.from_dict(result, orient='index', columns=['time_to_failure'])\nresult_df.head(n=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result_df.to_csv('./submission.csv', columns=['time_to_failure'], index_label='seg_id')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}