{"cells":[{"metadata":{"_uuid":"4b031ef24805b262fc29e6378084d3d099744020"},"cell_type":"markdown","source":"let's build an [autoencoder](http://https://blog.keras.io/building-autoencoders-in-keras.html) on test data, shall we?"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pyarrow.parquet as pq\nimport os\n\nN_MEASUREMENTS = 800_000\nN_FISR_SIGNAL_INDEX = 8712\nN_LAST_SIGNAL_INDEX = 29048\nN_TOTAL_SIGNALS = N_LAST_SIGNAL_INDEX - N_FISR_SIGNAL_INDEX","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5c77fe30ed7a79ecd8c83c9844a79bc2f18a29d9"},"cell_type":"code","source":"idx = N_FISR_SIGNAL_INDEX + np.arange(N_TOTAL_SIGNALS)\nidx","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ae0da4a153e4fb7acdeaaedde00e947085778c42"},"cell_type":"code","source":"np.random.shuffle(idx) # more fun on each run!","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# the whole set is just too big to be read at once, will use data generator instead\ndef generator(indexes, batch_size):\n    # From the docs: The generator is expected to loop over its data indefinitely.\n    # An epoch finishes when steps_per_epoch (see fit_generator()) batches have been seen by the model.\n    while True:\n        # 3 phases per signal\n        for start_col in range(0, N_TOTAL_SIGNALS, batch_size * 3):\n            # uncomment to see what signals are being loaded right now\n            # print(f'\\nloading signals from {start_col} to {start_col + batch_size * 3}')\n            \n            cols = [str(c) for c in indexes[start_col:start_col + batch_size * 3]]\n            n_signals = len(cols) // 3   # could be less than batch_size!\n            signals = pq.read_pandas('../input/test.parquet', columns=cols).to_pandas().values\n            # transform data from column-wise to row-wise\n            signals = np.vstack(np.split(signals, n_signals, axis=1))\n            signals = signals.reshape(-1, N_MEASUREMENTS, 3).astype(np.float16) / 130. \n            \n            # X == y, cool\n            yield signals, signals # shape=(?, 800_000, 3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d1fe0094704adbe9a3e6d5c397d8b14c76163f80"},"cell_type":"code","source":"from keras.models import Sequential, Model\nfrom keras.layers import Input, Dense, Conv1D, ELU, BatchNormalization, GRU, Conv2DTranspose, Reshape, Activation","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9c3335d45862da125867784737d2870d81267d41"},"cell_type":"code","source":"# our toy model\ndef make_model(ts=N_MEASUREMENTS,):\n    #\n    encoder = Sequential(\n        [\n            # 400 samples per filter, @ 4MHz sampling rate that should detect 10KHz and below\n            Conv1D(8, 400, strides=80, padding='same', input_shape=(ts, 3)), # (10_000, 8)\n            ELU(),\n            Conv1D(16, 20, strides=2, padding='same'), # (5_000, 16)\n            ELU(),\n            Conv1D(32, 4, strides=2, padding='same'), # (2_500, 32)\n            BatchNormalization(),\n            ELU(),\n\n            # LSTM will hopefully detect some periods\n            GRU(128,  return_sequences=False, recurrent_dropout=0.3),\n            Dense(128),\n            BatchNormalization(),\n            ELU(),\n            \n            # finally, compress our signal to 9 dimentions (3 per phase? who knows..)\n            Dense(9, activation='tanh'),\n        ]\n    )\n    encoder.summary()\n\n    decoder = Sequential(\n        [\n            Dense(128, input_shape=(9,)),\n            ELU(),\n            Dense(ts // 2000),  # [?, 400]\n            BatchNormalization(),\n            ELU(),\n            Reshape((ts // 2000, 1, 1)),           # [?, rows, cols, ch]\n            Conv2DTranspose(16, kernel_size= 4, strides=(2,  5), padding='same'),\n            # [?,  800,   5, 16]\n            Conv2DTranspose( 8, kernel_size=16, strides=(2,  2), padding='same'),    \n            # [?, 1600,  10,  8]\n            Conv2DTranspose( 3, kernel_size=16, strides=(5, 10), padding='same'),    \n            # [?, 8000, 100,  3]\n            Reshape((ts, 3)),  # [?, ts, 3]\n            Activation('tanh')\n        ]\n    )\n    decoder.summary()\n\n    inp = Input(shape=(ts, 3))\n    output = decoder(encoder(inp))\n    ae = Model(inputs=inp, outputs=output)\n    ae.summary()\n\n    return encoder, decoder, ae","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"59721cd4b423335ca97c33a472c299f6058b6a3a"},"cell_type":"code","source":"encoder, decoder, ae = make_model()\n\nmetric = 'mae'\nloss = 'mse'\noptimizer = 'adam' # RMSprop()\nae.compile(optimizer=optimizer, loss=loss, metrics=[metric])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cd40c55886cdb8059dab00d3399869c4a68d9e55"},"cell_type":"code","source":"# keep this low, as each signal is 800K * 3 samples ( or batch_size * 800_000 * 3 * 2 in bytes )\nbatch_size = 12\n\n# keep this low to get nice history graph\nsteps_per_epoch = 10\n\n# may wary, depending on how logn you are willing to train\nn_epochs = 10\nprint(f'will train on {batch_size * steps_per_epoch * n_epochs} signals')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"548a33e86639d005fb8b3534169adbc576ab321a"},"cell_type":"code","source":"history = ae.fit_generator(\n            generator(idx, batch_size),\n            steps_per_epoch=steps_per_epoch,\n            epochs=n_epochs,\n            validation_data=None, # need another generator for this\n            shuffle=False,        # already shuffled\n        )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bc49b2bfa8b69280b4c776b58b7440ca1740a845"},"cell_type":"code","source":"import matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d155eacaa65df6e6b79f94362bcb3faa0b472cd2"},"cell_type":"code","source":" def plot_history(history):\n    loss = history.history['loss']\n    mae = history.history['mean_absolute_error']\n    #\n    epochs = range(len(loss))\n    #\n    plt.plot(epochs, loss, 'bo', label='loss')\n    plt.plot(epochs, mae, 'b', label='mae')\n    plt.legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"306cd8392d92958d41a710d92b098899e6366f58"},"cell_type":"code","source":"# training is hard\nplot_history(history)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2b87f672bdd882e85d1acd9812d89df91163aa25"},"cell_type":"markdown","source":"the fun part, let's plot our predicted signals!"},{"metadata":{"trusted":true,"_uuid":"c6cd7e8935f450aa3530ec1421d90c59e080a8c5"},"cell_type":"code","source":"def plot_signals(x, y_pred):\n    n_samples = N_MEASUREMENTS\n    fig, ax = plt.subplots(nrows=2, ncols=3, figsize=(40, 20), dpi=80, facecolor='w', edgecolor='k', sharey=False)\n    for ph in range(3):\n        ax[0][0].plot(x[:, ph])\n        ax[0][1].plot(x[:, ph][n_samples//2:n_samples//2 + n_samples//16])\n        ax[0][2].plot(x[:, ph][n_samples//2:n_samples//2 + n_samples//64])\n    for ph in range(3):\n        ax[1][0].plot(y_pred[0, :, ph])\n        ax[1][1].plot(y_pred[0, :, ph][n_samples//2:n_samples//2 + n_samples//16])\n        ax[1][2].plot(y_pred[0, :, ph][n_samples//2:n_samples//2 + n_samples//64])        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7403ffa9d817bb215c9e147887b9d6f3929b1b1c","scrolled":true},"cell_type":"code","source":"signals, _ = next(generator(idx, 2))\ny_pred = ae.predict(signals[0].reshape(1, -1, 3))\nplot_signals(signals[0], y_pred)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c809b75b2563fbb74edd3b322ac10b055dc99419"},"cell_type":"markdown","source":"nice, our model was able to encode the source signal into 9-dimentional vector (or just 18 bytes!), and train the decoder to restore a pretty complex shape from it\n\nhow to use this vector for something usefull is another kernel.."},{"metadata":{"trusted":true,"_uuid":"998c677726c9e1b41926e8f0c2c3e62388944003"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}