{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## When the earthquake will take place? \n\n### Predicting the time remaining before laboratory earthquakes occur from real-time seismic data.","metadata":{}},{"cell_type":"markdown","source":"The data is related to lab earthquake data.\nIn lab, 2 plates are put under high pressure, which results to shear stress.\n\nThe training set is a single sequence of seismic signals.\nThe test set is based on segments of 150k instances and comes from many lab earthquakes.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import GridSearchCV\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2023-03-28T00:02:16.445922Z","iopub.execute_input":"2023-03-28T00:02:16.446399Z","iopub.status.idle":"2023-03-28T00:02:16.452908Z","shell.execute_reply.started":"2023-03-28T00:02:16.446336Z","shell.execute_reply":"2023-03-28T00:02:16.451812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Exploratory Data Analysis\n\ntrain_df = pd.read_csv('/kaggle/input/LANL-Earthquake-Prediction/train.csv', nrows=60000000, \n                    dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64})","metadata":{"execution":{"iopub.status.busy":"2023-03-28T00:05:02.793262Z","iopub.execute_input":"2023-03-28T00:05:02.793979Z","iopub.status.idle":"2023-03-28T00:05:20.317731Z","shell.execute_reply.started":"2023-03-28T00:05:02.793941Z","shell.execute_reply":"2023-03-28T00:05:20.316632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-03-28T00:06:51.896778Z","iopub.execute_input":"2023-03-28T00:06:51.897391Z","iopub.status.idle":"2023-03-28T00:06:51.918570Z","shell.execute_reply.started":"2023-03-28T00:06:51.897351Z","shell.execute_reply":"2023-03-28T00:06:51.917635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#visualize 1% of samples data, first 100 datapoints\ntrain_ad_sample_df = train_df['acoustic_data'].values[::100]\ntrain_ttf_sample_df = train_df['time_to_failure'].values[::100]\n\n#function for plotting based on both features\ndef plot_acc_ttf_data(train_ad_sample_df, train_ttf_sample_df, title=\"Acoustic data and time to failure: 1% sampled data\"):\n    fig, ax1 = plt.subplots(figsize=(12, 8))\n    plt.title(title)\n    plt.plot(train_ad_sample_df, color='r')\n    ax1.set_ylabel('acoustic data', color='r')\n    plt.legend(['acoustic data'], loc=(0.01, 0.95))\n    ax2 = ax1.twinx()\n    plt.plot(train_ttf_sample_df, color='b')\n    ax2.set_ylabel('time to failure', color='b')\n    plt.legend(['time to failure'], loc=(0.01, 0.9))\n    plt.grid(True)\n\nplot_acc_ttf_data(train_ad_sample_df, train_ttf_sample_df)\ndel train_ad_sample_df\ndel train_ttf_sample_df","metadata":{"execution":{"iopub.status.busy":"2023-03-28T00:06:55.367041Z","iopub.execute_input":"2023-03-28T00:06:55.368036Z","iopub.status.idle":"2023-03-28T00:06:56.040195Z","shell.execute_reply.started":"2023-03-28T00:06:55.367982Z","shell.execute_reply":"2023-03-28T00:06:56.039482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import IPython\nurl = 'https://raw.githubusercontent.com/filipmu/Kaggle-LANL-Earthquake-Prediction/master/data.png'\nIPython.display.Image(url)","metadata":{"execution":{"iopub.status.busy":"2023-03-26T14:09:22.341215Z","iopub.execute_input":"2023-03-26T14:09:22.341893Z","iopub.status.idle":"2023-03-26T14:09:22.778915Z","shell.execute_reply.started":"2023-03-26T14:09:22.341850Z","shell.execute_reply":"2023-03-26T14:09:22.777739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a href=\"https://www.kaggle.com/code/avloss/audio-analysis-with-animation/notebook\">One Resource from Kaggle who turned the data into sound and compared it with dubstep</a>\n\nHypothesis: perhaps just like a dubstep song, we can hear the build-up of an earthquake before the drop.","metadata":{}},{"cell_type":"code","source":"# feature engineering, let's generate some statistical features for better predictions\n\ndef gen_features(X):\n  strain = []\n  strain.append(X.mean())\n  strain.append(X.std())\n  strain.append(X.min())\n  strain.append(X.kurtosis())\n  strain.append(X.skew())\n  strain.append(np.quantile(X, 0.01))\n  return pd.Series(strain)","metadata":{"execution":{"iopub.status.busy":"2023-03-26T14:09:22.780512Z","iopub.execute_input":"2023-03-26T14:09:22.781133Z","iopub.status.idle":"2023-03-26T14:09:22.788117Z","shell.execute_reply.started":"2023-03-26T14:09:22.781093Z","shell.execute_reply":"2023-03-26T14:09:22.786941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"train = pd.read_csv('train.csv', iterator=True, chunksize=150_000, dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64})\n\nX_train = pd.DataFrame()\ny_train = pd.Series()\nfor df in train:\n    ch = gen_features(df['acoustic_data'])\n    X_train = X_train.append(ch, ignore_index=True)\n    y_train = y_train.append(pd.Series(df['time_to_failure'].values[-1]))\"\"\"","metadata":{"execution":{"iopub.status.busy":"2023-03-26T14:09:22.790165Z","iopub.execute_input":"2023-03-26T14:09:22.790592Z","iopub.status.idle":"2023-03-26T14:09:22.801071Z","shell.execute_reply.started":"2023-03-26T14:09:22.790554Z","shell.execute_reply":"2023-03-26T14:09:22.799844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defining the number of timesteps and features\n# since our test data is based on 150k samples, so we divide our feature vector \n# in 150 steps x len(data)//150\nn_timesteps = 150\nn_features = 1\n\n#split the data length\ntrain_len = len(train_df)#int(len(train_df)*0.8)\n\nX_train = np.zeros((train_len//n_timesteps, n_timesteps, n_features))\ny_train = np.zeros((train_len//n_timesteps, 1))\nprint(X_train.shape, y_train.shape)","metadata":{"execution":{"iopub.status.busy":"2023-03-28T00:07:16.514872Z","iopub.execute_input":"2023-03-28T00:07:16.515235Z","iopub.status.idle":"2023-03-28T00:07:16.525480Z","shell.execute_reply.started":"2023-03-28T00:07:16.515202Z","shell.execute_reply":"2023-03-28T00:07:16.524375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# split the data\nfrom sklearn.model_selection import train_test_split\n#dfX_train, dfX_test, dfy_train, dfy_test = train_test_split(train_df[['acoustic_data']], \n   #                                                 train_df[['time_to_failure']], \n #                                                   test_size=0.2, \n  #                                                  random_state=42)\ndfX_train, dfy_train = train_df[['acoustic_data']], train_df[['time_to_failure']]","metadata":{"execution":{"iopub.status.busy":"2023-03-28T00:07:53.086897Z","iopub.execute_input":"2023-03-28T00:07:53.087616Z","iopub.status.idle":"2023-03-28T00:07:53.656229Z","shell.execute_reply.started":"2023-03-28T00:07:53.087580Z","shell.execute_reply":"2023-03-28T00:07:53.655181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# FOR ADDING FEATURE ENGINEERING \n# X_train[i, :, 0] = train_df.iloc[i*n_timesteps:(i+1)*n_timesteps, 0:n_features].values\nfor i in range(train_len//n_timesteps):\n    X_train[i, :, 0] = dfX_train.iloc[i*n_timesteps:(i+1)*n_timesteps,0].values\n    y_train[i, 0] = dfy_train.iloc[(i+1)*n_timesteps-1] \n    #to predict the time remaining until the next earthquake at the end of each segment\n    # becuz we are using LSTM and our data is now divided in segments","metadata":{"execution":{"iopub.status.busy":"2023-03-28T00:07:56.135065Z","iopub.execute_input":"2023-03-28T00:07:56.135442Z","iopub.status.idle":"2023-03-28T00:09:01.266480Z","shell.execute_reply.started":"2023-03-28T00:07:56.135393Z","shell.execute_reply":"2023-03-28T00:09:01.265451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Dense, LSTM, Dropout\nfrom keras.callbacks import EarlyStopping\n\ncallback = EarlyStopping(monitor='loss', patience=3)\n\ndef create_lstm():\n    # Define the LSTM model\n    lstm_model = Sequential()\n    lstm_model.add(LSTM(128, input_shape=(n_timesteps, n_features), return_sequences=True))\n    lstm_model.add(Dropout(0.5))\n    lstm_model.add(LSTM(64, return_sequences=True))\n    lstm_model.add(Dropout(0.5))\n    lstm_model.add(LSTM(32, return_sequences=True))\n    lstm_model.add(Dropout(0.5))\n    lstm_model.add(LSTM(16))\n    lstm_model.add(Dropout(0.5))\n    lstm_model.add(Dense(1))\n\n    # Compile the model\n    lstm_model.compile(optimizer='adam', loss='mae')\n    return lstm_model","metadata":{"execution":{"iopub.status.busy":"2023-03-28T00:31:51.562213Z","iopub.execute_input":"2023-03-28T00:31:51.562613Z","iopub.status.idle":"2023-03-28T00:31:51.570330Z","shell.execute_reply.started":"2023-03-28T00:31:51.562579Z","shell.execute_reply":"2023-03-28T00:31:51.569300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#history = lstm_model.fit(X_train, y_train, epochs=10, batch_size=32, validation_split=0.2,callbacks=[callback],)","metadata":{"execution":{"iopub.status.busy":"2023-03-26T14:09:29.031123Z","iopub.execute_input":"2023-03-26T14:09:29.032395Z","iopub.status.idle":"2023-03-26T14:09:29.044321Z","shell.execute_reply.started":"2023-03-26T14:09:29.032353Z","shell.execute_reply":"2023-03-26T14:09:29.043373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualize accuracies\n\n%matplotlib inline\ndef perf_plot(history, what = 'loss'):\n    x = history.history[what]\n    val_x = history.history['val_' + what]\n    epochs = np.asarray(history.epoch) + 1\n    \n    plt.plot(epochs, x, 'g', label = \"Training \" + what)\n    plt.plot(epochs, val_x, 'r', label = \"Validation \" + what)\n    plt.title(\"Training and validation \" + what)\n    plt.xlabel(\"Epochs\")\n    plt.ylabel(\"Loss\")\n    plt.legend()\n    plt.show()\n    return None\n    \n#perf_plot(history)","metadata":{"execution":{"iopub.status.busy":"2023-03-26T14:09:29.046055Z","iopub.execute_input":"2023-03-26T14:09:29.046544Z","iopub.status.idle":"2023-03-26T14:09:29.057634Z","shell.execute_reply.started":"2023-03-26T14:09:29.046503Z","shell.execute_reply":"2023-03-26T14:09:29.056592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_absolute_error\ndef mae(model, dfX_test, dfy_test):\n  test_len = len(dfX_test)\n\n  x_test = dfX_test['acoustic_data'].values[:]\n  #print(x_test.shape)\n  x_test = np.reshape(x_test, (len(x_test)//n_timesteps, n_timesteps, 1))\n  #print(x_test.shape)\n\n  y_pred = model.predict(x_test)\n\n\n  y_test = np.zeros((test_len//n_timesteps, 1))\n  for i in range(test_len//n_timesteps):\n      y_test[i, 0] = dfy_test.iloc[(i+1)*n_timesteps-1]\n\n  mae = mean_absolute_error(y_test, y_pred)\n  print('Mean absolute error is', round(mae,3))\n\n#mae(model, dfX_test, dfy_test)","metadata":{"execution":{"iopub.status.busy":"2023-03-26T14:09:29.059270Z","iopub.execute_input":"2023-03-26T14:09:29.059694Z","iopub.status.idle":"2023-03-26T14:09:29.069048Z","shell.execute_reply.started":"2023-03-26T14:09:29.059657Z","shell.execute_reply":"2023-03-26T14:09:29.068164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from os import setegid\nfrom tqdm import tqdm\n\ndef test_earthquakes_submission(model):\n  # Load submission file\n  submission = pd.read_csv('/kaggle/input/LANL-Earthquake-Prediction/sample_submission.csv', index_col='seg_id', dtype={\"time_to_failure\": np.float32})\n  x = None\n  # Load each test data, create the feature matrix, get numeric prediction\n  for i, seg_id in enumerate(tqdm(submission.index)):\n    #  print(i)\n      seg_acoustic_data = (pd.read_csv('test/' + seg_id + '.csv'))['acoustic_data'].values[:]\n      x = np.reshape(seg_acoustic_data, (len(seg_acoustic_data)//n_timesteps, n_timesteps, 1))\n      #x = np.reshape(seg_acoustic_data, (len(seg_acoustic_data)//n_timesteps, n_timesteps, 1))\n      #print(x.shape)\n      submission.time_to_failure[i] = model.predict(x)[-1]\n\n#test_earthquake_submission(model)","metadata":{"execution":{"iopub.status.busy":"2023-03-26T14:09:29.070868Z","iopub.execute_input":"2023-03-26T14:09:29.071220Z","iopub.status.idle":"2023-03-26T14:09:29.079864Z","shell.execute_reply.started":"2023-03-26T14:09:29.071183Z","shell.execute_reply":"2023-03-26T14:09:29.078912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.layers import GRU\n\ndef create_gru():\n    gru_model = Sequential()\n    gru_model.add(GRU(128, input_shape=(n_timesteps, n_features), return_sequences=True))\n    gru_model.add(Dropout(0.1))\n    gru_model.add(GRU(64, return_sequences=True))\n    gru_model.add(Dropout(0.1))\n    gru_model.add(GRU(32, return_sequences=False))\n    gru_model.add(Dropout(0.1))\n    gru_model.add(Dense(1))\n\n    gru_model.compile(optimizer='adam', loss='mae')\n    return gru_model","metadata":{"execution":{"iopub.status.busy":"2023-03-26T14:09:29.081877Z","iopub.execute_input":"2023-03-26T14:09:29.082368Z","iopub.status.idle":"2023-03-26T14:09:29.091627Z","shell.execute_reply.started":"2023-03-26T14:09:29.082332Z","shell.execute_reply":"2023-03-26T14:09:29.090702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#history = gru_model.fit(X_train, y_train, epochs=10, batch_size=32, validation_split=0.2,callbacks=[callback],)","metadata":{"execution":{"iopub.status.busy":"2023-03-26T14:09:29.092875Z","iopub.execute_input":"2023-03-26T14:09:29.093965Z","iopub.status.idle":"2023-03-26T14:09:29.100985Z","shell.execute_reply.started":"2023-03-26T14:09:29.093918Z","shell.execute_reply":"2023-03-26T14:09:29.099994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#perf_plot(history)","metadata":{"execution":{"iopub.status.busy":"2023-03-26T14:09:29.102362Z","iopub.execute_input":"2023-03-26T14:09:29.102827Z","iopub.status.idle":"2023-03-26T14:09:29.110720Z","shell.execute_reply.started":"2023-03-26T14:09:29.102789Z","shell.execute_reply":"2023-03-26T14:09:29.109713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#mae(gru_model, dfX_test, dfy_test)","metadata":{"execution":{"iopub.status.busy":"2023-03-26T14:09:29.111924Z","iopub.execute_input":"2023-03-26T14:09:29.112181Z","iopub.status.idle":"2023-03-26T14:09:29.120283Z","shell.execute_reply.started":"2023-03-26T14:09:29.112157Z","shell.execute_reply":"2023-03-26T14:09:29.119160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.layers import SimpleRNN\n\ndef create_simpleRNN():\n    rnn_model = Sequential()\n\n    rnn_model.add(SimpleRNN(units=64, activation='tanh', input_shape=(n_timesteps, n_features), return_sequences=True))\n    rnn_model.add(SimpleRNN(units=32, activation='tanh', return_sequences=True))\n    rnn_model.add(SimpleRNN(units=16, activation='tanh'))\n\n    rnn_model.add(Dense(units=1))\n\n    rnn_model.compile(optimizer='adam', loss='mae')\n    return rnn_model","metadata":{"execution":{"iopub.status.busy":"2023-03-26T14:09:29.121411Z","iopub.execute_input":"2023-03-26T14:09:29.121694Z","iopub.status.idle":"2023-03-26T14:09:29.131491Z","shell.execute_reply.started":"2023-03-26T14:09:29.121662Z","shell.execute_reply":"2023-03-26T14:09:29.130558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#history = rnn_model.fit(X_train, y_train, epochs=10, batch_size=32, validation_split=0.2,callbacks=[callback],)","metadata":{"execution":{"iopub.status.busy":"2023-03-26T14:09:29.133362Z","iopub.execute_input":"2023-03-26T14:09:29.133669Z","iopub.status.idle":"2023-03-26T14:09:29.141237Z","shell.execute_reply.started":"2023-03-26T14:09:29.133643Z","shell.execute_reply":"2023-03-26T14:09:29.140218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#perf_plot(history)","metadata":{"execution":{"iopub.status.busy":"2023-03-26T14:09:29.142637Z","iopub.execute_input":"2023-03-26T14:09:29.143384Z","iopub.status.idle":"2023-03-26T14:09:29.154321Z","shell.execute_reply.started":"2023-03-26T14:09:29.143346Z","shell.execute_reply":"2023-03-26T14:09:29.153418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#mae(rnn_model, dfX_test, dfy_test)","metadata":{"execution":{"iopub.status.busy":"2023-03-26T14:09:29.155596Z","iopub.execute_input":"2023-03-26T14:09:29.155982Z","iopub.status.idle":"2023-03-26T14:09:29.164726Z","shell.execute_reply.started":"2023-03-26T14:09:29.155944Z","shell.execute_reply":"2023-03-26T14:09:29.163817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nfrom sklearn.model_selection import KFold\n\n# Define the K-fold cross-validation split\nkfold = KFold(n_splits=5, shuffle=True, random_state=42)\n\nlstm_loss_mae, gru_loss_mae, rnn_loss_mae = [],[],[]\n\n# Loop over the K-fold splits and train/evaluate the models\nfor fold, (train_idx, val_idx) in enumerate(kfold.split(X_train)):\n    \n    # Print the fold number\n    print(f\"\\nFold {fold+1}\\n\")\n    \n    # Get the training and validation data for this fold\n    X_train_fold = X_train[train_idx]\n    y_train_fold = y_train[train_idx]\n    X_val_fold = X_train[val_idx]\n    y_val_fold = y_train[val_idx]\n    \n    # Train and evaluate the LSTM model for this fold\n    lstm_model = create_lstm()\n    history = lstm_model.fit(X_train, y_train, epochs=5, batch_size=32, validation_split=0.2,\n                    callbacks=[callback],)\n    lstm_score = lstm_model.evaluate(X_val_fold, y_val_fold, verbose=0)\n    lstm_loss_mae.append(lstm_score)\n    print(f\"LSTM Fold {fold+1} score: {lstm_score:.5f}\")\n    \n    # Train and evaluate the GRU model for this fold\n    gru_model = create_gru()\n    history = gru_model.fit(X_train, y_train, epochs=5, batch_size=32, validation_split=0.2,\n                    callbacks=[callback],)\n    gru_score = gru_model.evaluate(X_val_fold, y_val_fold, verbose=0)\n    gru_loss_mae.append(gru_score)\n    print(f\"GRU Fold {fold+1} score: {gru_score:.5f}\")\n    \n    # Train and evaluate the Simple RNN model for this fold\n    rnn_model = create_simpleRNN()\n    history = rnn_model.fit(X_train, y_train, epochs=5, batch_size=32, validation_split=0.2,\n                    callbacks=[callback],)\n    rnn_score = rnn_model.evaluate(X_val_fold, y_val_fold, verbose=0)\n    rnn_loss_mae.append(rnn_score)\n    print(f\"RNN Fold {fold+1} score: {rnn_score:.5f}\")\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2023-03-26T14:09:29.166495Z","iopub.execute_input":"2023-03-26T14:09:29.166902Z","iopub.status.idle":"2023-03-26T16:29:35.900524Z","shell.execute_reply.started":"2023-03-26T14:09:29.166844Z","shell.execute_reply":"2023-03-26T16:29:35.899454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nk = np.arange(1, 6, 1, dtype=int)\nk\nmaxi = []\nmodel_loss_mae = [lstm_loss_mae, gru_loss_mae, rnn_loss_mae]\nfor i in model_loss_mae:\n    maxi.append(max(i))\nmax_loss = max(maxi)\n\"\"\"\n","metadata":{"execution":{"iopub.status.busy":"2023-03-26T23:18:55.117882Z","iopub.execute_input":"2023-03-26T23:18:55.118839Z","iopub.status.idle":"2023-03-26T23:18:55.126798Z","shell.execute_reply.started":"2023-03-26T23:18:55.118800Z","shell.execute_reply":"2023-03-26T23:18:55.125685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\ndef kfold_plot():\n    k = np.arange(1, 6, 1, dtype=int)\n    plt.plot(k, lstm_loss_mae, 'g', label = \"LSTM MAE\")\n    plt.plot(k, gru_loss_mae, 'r', label = \"GRU MAE\")\n    plt.plot(k, rnn_loss_mae, 'b', label = \"Simple RNN MAE\")\n    plt.title(\"K-Fold Cross Validation Loss (MAE)\")\n    plt.xlabel(\"K-Fold\")\n    plt.ylabel(\"Loss (MAE)\")\n    plt.legend()\n    plt.xticks(k)\n    plt.ylim(0,math.ceil(max_loss))\n    plt.show()\n    return None\n    \n#kfold_plot()","metadata":{"execution":{"iopub.status.busy":"2023-03-26T23:20:17.955677Z","iopub.execute_input":"2023-03-26T23:20:17.956052Z","iopub.status.idle":"2023-03-26T23:20:18.158557Z","shell.execute_reply.started":"2023-03-26T23:20:17.956019Z","shell.execute_reply":"2023-03-26T23:20:18.157648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model selection\nModel selection was based on the best cross-validation scores. \n\n**GRU and LSTM almost performed same with around similar Mean Absolute Errors on 5-fold and RNN followed on 3rd**","metadata":{}},{"cell_type":"markdown","source":"* **Recurrent network especially GRU & LSTMs are designed for sequential data and can capture long-term dependencies than other neural networks like CNN.** \n* **They have a memory cell that can selectively forget noise and remember relevant patterns in the data.**","metadata":{}},{"cell_type":"code","source":"# Train and evaluate the LSTM model for this fold\nlstm_model = create_lstm()\nhistory = lstm_model.fit(X_train, y_train, epochs=20, batch_size=128, validation_split=0.2,\n                callbacks=[callback],)\nlstm_score = lstm_model.evaluate(X_val_fold, y_val_fold, verbose=0)\nprint(f\"LSTM Fold {fold+1} score: {lstm_score:.5f}\")\nperf_plot(history)","metadata":{"execution":{"iopub.status.busy":"2023-03-28T00:34:48.495711Z","iopub.execute_input":"2023-03-28T00:34:48.496632Z","iopub.status.idle":"2023-03-28T01:17:25.626143Z","shell.execute_reply.started":"2023-03-28T00:34:48.496594Z","shell.execute_reply":"2023-03-28T01:17:25.625225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n# Save\nsubmission.to_csv('submission.csv')\n\"\"\"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-26T16:29:36.135666Z","iopub.status.idle":"2023-03-26T16:29:36.136163Z","shell.execute_reply.started":"2023-03-26T16:29:36.135906Z","shell.execute_reply":"2023-03-26T16:29:36.135932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IPython.display.Image('https://raw.githubusercontent.com/filipmu/Kaggle-LANL-Earthquake-Prediction/master/preds.png')","metadata":{"execution":{"iopub.status.busy":"2023-03-26T20:00:12.028927Z","iopub.execute_input":"2023-03-26T20:00:12.029633Z","iopub.status.idle":"2023-03-26T20:00:12.439926Z","shell.execute_reply.started":"2023-03-26T20:00:12.029597Z","shell.execute_reply":"2023-03-26T20:00:12.438716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Future Directions\n\n* Ensemble Neural Network\n* Comprehensive Feature Engineering\n* Experiments with full data (600 million)","metadata":{}}]}