{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"# Importing the needed utilities for NN\nimport tensorflow as tf\nfrom tensorflow.keras import Model, Input, models\nfrom tensorflow.keras.utils import plot_model\nfrom tensorflow.keras.callbacks import EarlyStopping, LearningRateScheduler, ReduceLROnPlateau\nfrom tensorflow.keras.layers import BatchNormalization, Dropout, Flatten, Dense, InputLayer, Concatenate, Add, SeparableConv2D, Layer\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.initializers import TruncatedNormal\n\nfrom sklearn.model_selection import RepeatedKFold # KFold\n\nimport matplotlib.pyplot as plt # Plotting\n\n# Data Wrangling\nimport numpy as np\nimport pandas as pd ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Preprocessing\n- It might be misleading to create data for the missing signal values, hence they are imputed with zeros.\n- There are 6000 rows and 10 columns in each csv file within train and test folders. In order to extract features from all of these .csv files, I have stored the measures of central tendency for each .csv file. \n- Two type of data:\n    - Raw: Mean, std, etc.\n    - Rolling: Got the measures of central tendency for the mean and std of the rolling data.\n\n##### Data is imported from: <a href=\"https://www.kaggle.com/damoonshahhosseini/volcano-pca\">reduced-data</a>\n##### Neural Networks saved here: <a href=\"https://www.kaggle.com/damoonshahhosseini/volcanonn\">volcano-nn</a>"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/predict-volcanic-eruptions-ingv-oe/train.csv') # Training data\nsample_submission = pd.read_csv('/kaggle/input/predict-volcanic-eruptions-ingv-oe/sample_submission.csv') \n\n# Getting the ids\ntrain_ids, test_ids = train['segment_id'], sample_submission['segment_id']\n\n# Value to be predicted\ny = train['time_to_eruption']\n\n# Measures of Central Tendency for raw data\nX = np.load('/kaggle/input/volcano-pca/train.npy')\nX_test = np.load('/kaggle/input/volcano-pca/test.npy')\n\n# Measures of Central Tendency for rolling data\nX_rolling = np.load('/kaggle/input/volcano-pca/rolling_train.npy')\nX_rolling_test = np.load('/kaggle/input/volcano-pca/rolling_test.npy')\n\nX_concat = np.load('/kaggle/input/volcano-pca/train_concat.npy')\n# X_concat_test = np.load('/kaggle/input/volcano-pca/test_concat.npy')\n\n\nX_aug = np.load('/kaggle/input/volcano-pca/aug_train_concat_2.npy')\ny_aug = np.load('/kaggle/input/volcano-pca/y_aug2.npy')\n\nX = X_aug[:, :, :16]\nX_rolling = X_aug[:, :, 16:]\n\n# Model's name\nMODEL_NAME = 'res32_v3'\n\nIn1 = Input(shape=(10, 16, 1))\nIn2 = Input(shape=(10, 16, 1))\nInm = Input(shape=(10, 32, 1))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Neural Network Utilities\nSome of the fucntion and global entities used in the construction of the Neural Networks."},{"metadata":{"trusted":true},"cell_type":"code","source":"# The Global weight initializer\nWInit = TruncatedNormal(0, 1, 111)\n\ndef concat_inputs(Inp1, Inp2): \n    \"\"\" Concatenates the raw measures with rolling measures \"\"\"\n    return Concatenate(axis=2)([Inp1, Inp2])\n    \ndef fc(Inp, unit=100, n=3, activation='relu', pred=True, bn=False):\n    \"\"\" Fully Connected layer used on the flattened data \"\"\"\n    Inp = Flatten()(Inp)\n    out = Dense(unit, activation=activation ,kernel_initializer=WInit)(Inp)\n    for i in range(0, n-2): \n        out = Dense(unit, activation=activation ,kernel_initializer=WInit)(out)\n        if bn: out = BatchNormalization()(out)\n        \n        \n    if not pred: \n        out = Dense(unit, activation=activation ,kernel_initializer=WInit)(out)\n        return out \n        \n    out = Dense(1, kernel_initializer=TruncatedNormal(10, 1e-1, 11))(out)\n        \n    return out\n\ndef submit_prediction(pred, write=True, file_name=\"\", Return=True):\n    \"\"\"\n        Submits prediction and modifies the negative values.\n        \n        input:\n            pred: the predicted values\n            write: boolean value indicating if the predictions should be\n                written to a csv file.\n            file_name: name of the csv file to write\n            Return: if the edited predictions should be returned\n        \n        return:\n            returns the edited predictions if asked for (Return==1)\n    \"\"\"\n    # Replace the negative values with the mean of the data\n    pred = np.where(pred < 0, np.mean(pred), pred)\n    \n    sample_submission['time_to_eruption'] = pred  # Formatting\n    if write: sample_submission.to_csv(f'{file_name}.csv', index=False)  # Write the file to a csv file\n    \n    if Return: return pred\n    \n\ndef vertical_kernel(x,y, length): return [(y, x + i) for i in range(length)]\ndef horizontal_kernel(x,y, length): return [(y + i, x) for i in range(length)]\n\ndef module(inp, vertical_kernels, horizontal_kernels, unit=16, std=1e-1, seed=11):\n    vertical_convs = [SeparableConv2D(unit, \n                                        kernel_size, \n                                        kernel_initializer=TruncatedNormal(0, std, seed),\n                                        bias_initializer=TruncatedNormal(0, std, seed))(inp) for kernel_size in vertical_kernels]\n    \n    horizontal_convs = [SeparableConv2D(unit, \n                                        kernel_size, \n                                        kernel_initializer=TruncatedNormal(0, std, seed),\n                                        bias_initializer=TruncatedNormal(0, std, seed))(inp) for kernel_size in horizontal_kernels]\n\n    vertical_concats = [Concatenate(axis=2)(\n        [vertical_convs[i], vertical_convs[len(vertical_convs) - 1 - i]]\n    ) for i in range(len(vertical_convs) // 2)]\n\n    horizontal_concats = [Concatenate(axis=1)(\n        [horizontal_convs[i], horizontal_convs[len(horizontal_convs) - 1 - i]]\n    ) for i in range(len(horizontal_convs) // 2)]\n    \n    return vertical_concats, horizontal_concats","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Models"},{"metadata":{"trusted":true},"cell_type":"code","source":"def model2(inp):\n    vertical_kernels = vertical_kernel(6,10, 8)\n    horizontal_kernels = horizontal_kernel(32, 3, 8)\n\n    vert_concat1, hor_concat1 = module(inp, vertical_kernels, horizontal_kernels)\n\n    vert_preds = [fc(layer, unit=100, n=5, activation='relu', pred=True) for layer in vert_concat1]\n    hor_preds = [fc(layer, unit=100, n=5, activation='relu', pred=True) for layer in hor_concat1]\n    \n    vert_preds.extend(hor_preds)\n    \n    pred = fc(Concatenate()(vert_preds), unit=25, n=3, activation='relu', pred=True)\n    \n    return pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def resnet_module(inp, unit=32, stride1=(3,3), stride2=(5,5), stride3=(3,3), mid_activation=None, padding='same', bn=True, std=1e-2, seed=123):\n    \n    l1 = SeparableConv2D(filters=unit, \n                         kernel_size=stride1, \n                         kernel_initializer=TruncatedNormal(0, std, seed), \n                         padding=padding\n                        )(inp)\n    \n    l2 = SeparableConv2D(filters=unit, \n                         kernel_size=stride2, \n                         kernel_initializer=TruncatedNormal(0, std*1.1, seed),\n                         activation=mid_activation,\n                         padding='same'\n                        )(l1)\n    \n    l3 = SeparableConv2D(filters=unit, \n                         kernel_size=stride3, \n                         kernel_initializer=TruncatedNormal(0, std*5, seed), \n                         padding=padding\n                        )(l2)\n    \n    la = Add()([l1, l3])\n    \n    if bn: return BatchNormalization()(la)\n    \n    return la\n    \n\ndef mres(inp):\n    \n    # First three moduls: 10 x 16\n    b1 = resnet_module(inp, stride1=(1,3), stride2=(5,5), stride3=(3,1), bn=False, std=1)\n    b1 = resnet_module(b1, stride1=(3,1), stride2=(3,3), stride3=(1,3), mid_activation='relu',bn=True, std=1.5)\n    b1 = resnet_module(b1, stride1=(3,3), stride2=(3,3), stride3=(3,3), mid_activation='relu',bn=True, std=1)\n    \n    # Second three moduls: 10 x 16\n    b2 = resnet_module(b1, unit=64, stride1=(4,4), stride2=(1,1), stride3=(4,4), mid_activation='relu', bn=False, std=1e-2)\n    b2 = resnet_module(b2, unit=64, stride1=(6,1), stride2=(3,3), stride3=(1,6),bn=False, std=1.5e-3)\n    b2 = resnet_module(b2, unit=64, stride1=(4,4), stride2=(3,3), stride3=(4,4), mid_activation='relu',bn=False, std=3e-1)\n    \n     # Second three moduls: 10 x 16\n    b3 = resnet_module(b2, unit=128, stride1=(4,4), stride2=(1,1), stride3=(4,4), mid_activation='sigmoid', bn=False, std=1e-2)\n    b3 = resnet_module(b3, unit=128, stride1=(6,1), stride2=(3,3), stride3=(1,6),bn=False, std=1.5e-3)\n    b3 = resnet_module(b3, unit=128, stride1=(4,4), stride2=(1,1), stride3=(4,4), mid_activation='sigmoid',bn=True, std=3e-1)\n    \n    b3 = Dropout(0.5)(b3)\n    \n     # Second three moduls: 10 x 16\n    b4 = resnet_module(b3, unit=256, stride1=(3,3), stride2=(1,1), stride3=(3,3), mid_activation='sigmoid', bn=False, std=1e-2)\n    b4 = resnet_module(b4, unit=256, stride1=(6,1), stride2=(1,1), stride3=(1,6),bn=False, std=1.5e-3)\n    b4 = resnet_module(b4, unit=256, stride1=(2,3), stride2=(1,1), stride3=(2,1), mid_activation='relu',bn=True, std=3e-1)\n    \n    # Second three moduls: 10 x 16\n    b5 = resnet_module(b4, unit=256, stride1=(4,4), stride2=(1,1), stride3=(4,4), mid_activation='sigmoid', bn=True, std=1e-2)\n    b5 = resnet_module(b5, unit=256, stride1=(3, 3), stride2=(3,3), stride3=(3,3),bn=False, std=1.5e-3)\n    b5 = resnet_module(b5, unit=256, stride1=(4,4), stride2=(3,3), stride3=(4,4), mid_activation='relu',bn=True, std=3e-1)\n    \n    b5 = Dropout(0.5)(b5)\n    \n    # Second three moduls: 10 x 16\n    b6 = resnet_module(b5, unit=256, stride1=(3,3), stride2=(7,7), stride3=(2,3), mid_activation='relu', bn=True, std=1e-2)\n    b6 = resnet_module(b6, unit=256, stride1=(4,1), stride2=(3,3), stride3=(1,4),bn=False, std=1.5e-3)\n    b6 = resnet_module(b6, unit=256, stride1=(2,2), stride2=(1,1), stride3=(3,2), mid_activation='relu',bn=True, std=3e-1)\n    \n    p = fc(b6, unit=100, n=3, activation='relu')\n    \n    return p","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Input\n# In1 = Input(shape=(10,16,1))\n\n# m1 = model1(In1)\n# In = concat_inputs(In1, In2)\n# model = Model(inputs=[In1], outputs=[mres(In1)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model.count_params() / 1e6","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot_model(model, show_layer_names=0, show_shapes=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = models.load_model('/kaggle/input/volcanonn/res32_v2')\nfrom tensorflow.keras import backend as K\nK.set_value(model.optimizer.learning_rate, 0.0015)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.optimizer.learning_rate","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model.compile(optimizer=Adam(0.0015), loss='mae')\n\n# hist = model.fit([X, X_rolling], y_aug, epochs= 100, batch_size=32,\n#                  callbacks=[\n#                      ReduceLROnPlateau(monitor='loss', factor=0.9, patience=10, verbose=1, min_delta=0),\n#                      EarlyStopping(monitor='val_loss', min_delta=0, patience=300, verbose=1,\n#                                    mode='min', baseline=None, restore_best_weights=True)\n#                 ],\n    \n#                  validation_split=0.3,\n#                   verbose=True,\n#                   shuffle=True) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 0.0015\n# (pd.DataFrame(hist.history) / 1e6).plot(y=['loss', 'val_loss'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training with Repeated KFold"},{"metadata":{"trusted":true},"cell_type":"code","source":"def kfold_validation(model, X, y, X_test, n_rep=3, n_fold=5, batch_size=32, epochs=300, verbose=1):\n    \"\"\"\n        Runs a repeated KFold on a given model and data\n        \n        input:\n            model: model to traing the data\n            X, y, X_test: datasets needed for training and prediction\n            n_rep, n_fold: parameters of Repeated KFold\n            batch_size, epoch, verbose: info for the training data\n        \n        return:\n            model: trained model\n            preds: predictions at each fold\n            avg_preds: average of all predictions\n    \"\"\"\n    history, index = {}, 0  # Keep track of the loss and val_loss (history object)\n    prediction = np.zeros((X_test.shape[0])) # For every single prediction\n    preds = np.empty((n_rep * n_fold, X_test.shape[0])) # Saving all the predictions\n    \n    kf = RepeatedKFold(n_splits=n_fold, n_repeats=n_rep, random_state=288490)\n    \n    for train_indices, val_indices in kf.split(X, y):\n    \n        print(f'{index + 1}th fold, Validation Indices: ', val_indices[:5])\n\n        # Data divided into Train and Validation splits\n        X_train, X_val = X[train_indices], X[val_indices]\n        X_rolling_train, X_rolling_val = X_rolling[train_indices], X_rolling[val_indices]\n        y_train, y_val = y[train_indices], y[val_indices]\n\n        # Fitting\n        history[index] = model.fit(\n            x=[X_train, X_rolling_train], \n            y=y_train, \n            epochs=epochs,\n            batch_size=batch_size,\n            validation_data=([X_val, X_rolling_val], y_val),\n            verbose=verbose,\n            callbacks=[ \n                ReduceLROnPlateau(factor=0.96, patience=10, verbose=True, monitor='val_loss', min_lr=1e-45, min_delta=1e-4),\n                EarlyStopping(\n                    monitor='val_loss', \n                    patience=25, \n                    restore_best_weights=True, \n                    min_delta=1e4, \n                    verbose=True\n                )\n            ]\n        )\n        \n        \n        model_prediction = model.predict([X_test, X_rolling_test], \n                                         batch_size=batch_size, \n                                         verbose=False).reshape(X_test.shape[0])\n        \n        model_prediction = submit_prediction(model_prediction, \n                                             write=True, \n                                             file_name=f\"sub{index}\", \n                                             Return=True)\n        \n        # Saving the predictions for each fold\n        preds[index] = model_prediction\n        index += 1\n        \n        # Starting different fold or end of folding\n        print('#----------------#----------------#----------------#----------------#----------------#')\n        \n    # Averaging the predictions\n    p = pd.DataFrame(preds)\n    p = p.sum() / (n_fold * n_rep)\n        \n    avg_pred = submit_prediction(p, \n                      write=True, \n                      file_name=f\"S_avg\", \n                      Return=True)\n    \n    return model, preds, avg_pred, history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model, preds, avg_pred, history = kfold_validation(model, X, y_aug, X_test, 6, 3, 32, epochs=500, verbose=0)\n\n# Model and its predictions will be saved\nnp.save('preds_V16.npy', preds)\nmodel.save(f'/kaggle/working/{MODEL_NAME}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Plotting"},{"metadata":{"trusted":true},"cell_type":"code","source":"num_cols = 5\nfig, axes = plt.subplots(len(history), num_cols, figsize=(200,50))\n# fig.legend([\"blue\", \"orange\"])\nfig.tight_layout()\n\nfor i in range(len(history)):\n    d = pd.DataFrame(history[i].history) / 1e6\n    d['Epoch'] = range(0,d.shape[0])\n\n#     d.iloc[:,:].plot(x=\"Epoch\", y=[\"loss\",\"val_loss\"], ax=axes[i][0])\n    for j in range(num_cols):\n        d.iloc[d.shape[0]//num_cols*j:d.shape[0]//num_cols*(j+1),:].plot(\n            x=\"Epoch\", y=[\"loss\",\"val_loss\"], ax=axes[i][j], title=f'{i+1}th fold')\n\nplt.subplots_adjust(left=0, bottom=None, right=0.1, top=None, wspace=0.5, hspace=0.3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}