{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Imports"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport numpy as np \nimport pandas as pd\nimport tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Read the TFRecord Files Into tf.data.Dataset Objects"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def decode(serialized_example):\n    \"\"\" Parses a set of features and label from the given `serialized_example`.\n        \n        It is used as a map function for `dataset.map`\n\n    Args:\n        serialized_example (tf.Example): A serialized example containing the\n            following features:\n                – sensor_feature_0 – [int64]\n                – sensor_feature_1 – [int64]\n                – sensor_feature_2 – [int64]\n                – sensor_feature_3 – [int64]\n                – sensor_feature_4 – [int64]\n                – sensor_feature_5 – [int64]\n                – sensor_feature_6 – [int64]\n                – sensor_feature_7 – [int64]\n                – sensor_feature_8 – [int64]\n                – sensor_feature_9 – [int64]\n                – label_feature    – int64\n        \n    Returns:\n        A decoded tf.data.Dataset object representing the tfrecord dataset\n    \"\"\"\n    # Defaults are not specified since both keys are required.\n    feature_dict = {\n        'sensor_feature_0': tf.io.FixedLenSequenceFeature(shape=(), dtype=tf.int64, allow_missing=True),\n        'sensor_feature_1': tf.io.FixedLenSequenceFeature(shape=(), dtype=tf.int64, allow_missing=True),\n        'sensor_feature_2': tf.io.FixedLenSequenceFeature(shape=(), dtype=tf.int64, allow_missing=True),\n        'sensor_feature_3': tf.io.FixedLenSequenceFeature(shape=(), dtype=tf.int64, allow_missing=True),\n        'sensor_feature_4': tf.io.FixedLenSequenceFeature(shape=(), dtype=tf.int64, allow_missing=True),\n        'sensor_feature_5': tf.io.FixedLenSequenceFeature(shape=(), dtype=tf.int64, allow_missing=True),\n        'sensor_feature_6': tf.io.FixedLenSequenceFeature(shape=(), dtype=tf.int64, allow_missing=True),\n        'sensor_feature_7': tf.io.FixedLenSequenceFeature(shape=(), dtype=tf.int64, allow_missing=True),\n        'sensor_feature_8': tf.io.FixedLenSequenceFeature(shape=(), dtype=tf.int64, allow_missing=True),\n        'sensor_feature_9': tf.io.FixedLenSequenceFeature(shape=(), dtype=tf.int64, allow_missing=True),\n        'label_feature': tf.io.FixedLenFeature(shape=(), dtype=tf.int64),\n        }\n    \n  \n    # Define a parser\n    features = tf.io.parse_single_example(serialized_example, features=feature_dict)\n        \n    # Decode the data and capture the label feature\n    sensors = [tf.cast(features[f\"sensor_feature_{i}\"], tf.int16) for i in range(10)]\n    \n    label = tf.cast(features[\"label_feature\"], tf.int32)\n    return sensors, label\n\ndef get_tfrecord_ds(tfrecord_dir):\n    tfrecord_paths = [os.path.join(tfrecord_dir, f_name) \\\n                      for f_name in os.listdir(tfrecord_dir) \\\n                      if f_name.endswith('.tfrec')]\n    return tf.data.TFRecordDataset(tfrecord_paths)\n\ntrain_ds = get_tfrecord_ds(\"../input/ingv-volcanic-eruption-training-tfrecords/train\")\nval_ds = get_tfrecord_ds(\"../input/ingv-volcanic-eruption-training-tfrecords/val\")\ntest_ds = get_tfrecord_ds(\"../input/ingv-volcanic-eruption-testing-tfrecords/test\")\n\nprint(\"\\n... TFRECORD DATASETS ...\\n\\t–– TRAIN DS – \" \\\n      f\"{train_ds}\\n\\t–– VAL DS   – {val_ds}\\n\\t–– TEST DS  – {test_ds}\")\n\ntrain_ds = train_ds.map(decode)\nval_ds = val_ds.map(decode)\ntest_ds = test_ds.map(decode)\n\nprint(\"\\n... DECODED DATASETS ...\\n\\t–– TRAIN DS – \" \\\n      f\"{train_ds}\\n\\t–– VAL DS   – {val_ds}\\n\\t–– TEST DS  – {test_ds}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Set Hyperparameters for Preprocessing and Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTOTUNE = tf.data.experimental.AUTOTUNE\nBATCH_SIZE = 16\nSHUFFLE_BUFFER = 160 # (2 TFRecords in advance)\nN_EPOCHS = 50","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preprocessing\n\nInitial work will be done with no real preprocessing to set a dumb baseline"},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocessing_fn(x,y):\n    def _preprocess_y(_y, max_to_norm=50000000):\n        \"\"\" Approximate reduction from millions to integers \"\"\"\n        return tf.divide(tf.cast(_y, tf.float32), tf.cast(max_to_norm, tf.float32))\n    \n    def _preprocess_x(_x, max_to_norm=1000):\n        \"\"\" No active normalization \"\"\"\n        return tf.divide(tf.cast(tf.transpose(_x), tf.float32), tf.cast(max_to_norm, tf.float32))\n    \n    x = _preprocess_x(x)\n    y = _preprocess_y(y)\n    return x,y \n    \npp_train_ds = train_ds.shuffle(SHUFFLE_BUFFER)\npp_train_ds = pp_train_ds.map(lambda x,y: preprocessing_fn(x,y),\n                              num_parallel_calls=AUTOTUNE)\npp_train_ds = pp_train_ds.batch(BATCH_SIZE)\nfinal_train_ds = pp_train_ds.prefetch(AUTOTUNE)\n\npp_val_ds = val_ds.map(lambda x,y: preprocessing_fn(x,y),\n                       num_parallel_calls=AUTOTUNE)\npp_val_ds = pp_val_ds.batch(BATCH_SIZE)\nfinal_val_ds = pp_val_ds.prefetch(AUTOTUNE)\n\npp_test_ds = test_ds.map(lambda x,y: (tf.divide(tf.cast(tf.transpose(x), tf.float32), tf.cast(1000, tf.float32)), y))\npp_test_ds = pp_test_ds.batch(BATCH_SIZE)\nfinal_test_ds = pp_test_ds.prefetch(AUTOTUNE)\n\nprint(\"\\n... PREPROCESSED DATASETS ...\\n\\t–– TRAIN DS – \" \\\n      f\"{final_train_ds}\\n\\t–– VAL DS   – {final_val_ds}\\n\\t–– TEST DS  – {final_test_ds}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Basic SqueezeNet Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def sqz_exp(inputs, squeeze, expand, residual, kernel_width=10, batch_norm=True):\n    \n    SQZ_ARGS = dict(filters=squeeze, kernel_size=1, padding='same', use_bias=not batch_norm)\n    EXP_BR_1_ARGS = dict(filters=expand, kernel_size=1, padding='same', use_bias=not batch_norm)\n    EXP_BR_2_ARGS = dict(filters=expand, kernel_size=kernel_width, padding='same', use_bias=not batch_norm)\n    \n    # Squeeze Part\n    x = tf.keras.layers.Conv1D(**SQZ_ARGS)(inputs)\n    if batch_norm:\n        x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.Activation(\"relu\")(x)\n\n    # Branching Part (Expand Part)\n    br_1 = tf.keras.layers.Conv1D(**EXP_BR_1_ARGS)(x)\n    br_2 = tf.keras.layers.Conv1D(**EXP_BR_2_ARGS)(x)\n    if batch_norm:\n        br_1 = tf.keras.layers.BatchNormalization()(br_1)\n        br_2 = tf.keras.layers.BatchNormalization()(br_2)\n    combine = tf.keras.layers.concatenate([br_1, br_2])\n    combine = tf.keras.layers.Activation(\"relu\")(combine)\n    \n    # Residual Part\n\n    if residual:\n        return tf.keras.layers.Concatenate()([combine, inputs])\n    else:\n        return combine\n\n# this is to make it behave similarly to other Keras layers\ndef sqz_module(squeeze, expand, kernel_width, residual=False, bn=True):\n    return lambda x: sqz_exp(x, squeeze, expand, residual, kernel_width, bn)\n\ndef get_sqz_model(input_shape=(60001,10),\n                  n_blocks=3, \n                  init_kernel_width=200,\n                  init_sqz_size=32,\n                  init_exp_size=128,\n                  n_dense=1024,\n                  pool_size=10):\n\n    inputs = tf.keras.layers.Input(shape=input_shape)\n    x = inputs\n\n    for i in range(n_blocks):\n        if i==0:\n            x = sqz_module(squeeze=init_sqz_size, \n                           expand=init_exp_size, \n                           kernel_width=init_kernel_width,\n                           residual=False)(x)\n        else:\n            x = tf.keras.layers.MaxPooling1D(max(2, pool_size-(i-1)*2))(x)\n            x = sqz_module(squeeze=init_sqz_size*i, \n                expand=init_exp_size*i, \n                kernel_width=max(init_kernel_width//2**(i+1), 10),\n                residual=True)(x)\n\n    x = tf.keras.layers.Dense(n_dense, activation='relu')(x)\n    x = tf.keras.layers.GlobalAveragePooling1D()(x)\n    x = tf.keras.layers.Dropout(0.4)(x)\n    x = tf.keras.layers.Dense(n_dense//2, activation='relu')(x)\n    x = tf.keras.layers.Dropout(0.6)(x)\n    \n    # OUTPUT\n    outputs = tf.keras.layers.Dense(1, activation='linear')(x)\n    return tf.keras.Model(inputs=inputs, outputs=[outputs])\n\nsqznet = get_sqz_model()\nsqznet.summary()\ntf.keras.utils.plot_model(sqznet, show_shapes=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Compile the Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"sqznet.compile(optimizer=tf.keras.optimizers.Adam(lr=0.00225), loss=tf.keras.losses.Huber(), metrics=\"mae\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Callbacks"},{"metadata":{"trusted":true},"cell_type":"code","source":"CKPT_PATH = \"./ckpts/EPOCH__{epoch:04d}___MAE__{val_mae:.2f}.ckpt\"\nckpt_cb = tf.keras.callbacks.ModelCheckpoint(CKPT_PATH, monitor=\"val_mae\", verbose=1)\nlr_cb = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_mae', factor=0.5, patience=2, verbose=1, min_lr=0.00001)\nearly_cb = tf.keras.callbacks.EarlyStopping(monitor='val_mae', patience=6, verbose=1, restore_best_weights=True)\ncb_list = [ckpt_cb, lr_cb, early_cb]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Fitting"},{"metadata":{"trusted":true},"cell_type":"code","source":"sqznet.fit(final_train_ds, validation_data=final_val_ds, \n           epochs=N_EPOCHS, \n           callbacks=cb_list)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prediction"},{"metadata":{"trusted":true},"cell_type":"code","source":"# sqznet = tf.keras.models.load_model(\"/tmp/ckpts/EPOCH__0006___MAE__6.5477.ckpt\", compile=True)\n\ntime_to_eruption = np.zeros((4520,), dtype=np.int32)\nsegment_id = np.zeros((4520,), dtype=np.int32)\n\nfor i, (inputs, ids) in enumerate(final_test_ds):\n    print(f\"Infering on Batch {i+1}\")\n    segment_id[BATCH_SIZE*i:min(BATCH_SIZE*(i+1), 4520)] = ids.numpy()\n    time_to_eruption[BATCH_SIZE*i:min(BATCH_SIZE*(i+1), 4520)] = sqznet.predict(inputs)[:, 0]*50000000\n\npred_df = pd.DataFrame({\"segment_id\":segment_id, \"time_to_eruption\":time_to_eruption})\npred_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_df.describe().T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}