{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd \n\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport seaborn as sns\nsns.set()\n\nfrom os import listdir\n\nfrom sklearn.model_selection import TimeSeriesSplit\n\nimport warnings\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\nwarnings.filterwarnings(\"ignore\", category=UserWarning)\nwarnings.filterwarnings(\"ignore\", category=FutureWarning)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\", nrows=10000000,\n                    dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.options.display.precision = 15\ntrain.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.rename({\"acoustic_data\": \"signal\", \"time_to_failure\": \"earthquake_time\"}, axis=\"columns\", inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for n in range(10):\n    print(train.earthquake_time.values[n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(2,1, figsize=(20,12))\nax[0].plot(train.index.values, train.earthquake_time.values, c=\"darkred\")\nax[0].set_title(\"Earthquake remaining time of 10 Mio rows\")\nax[0].set_xlabel(\"Index\")\nax[0].set_ylabel(\"EarthQuake remaining time in ms\");\nax[1].plot(train.index.values, train.signal.values, c=\"mediumseagreen\")\nax[1].set_title(\"Signal of 10 Mio rows\")\nax[1].set_xlabel(\"Index\")\nax[1].set_ylabel(\"Acoustic Signal\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(3,1,figsize=(20,18))\nax[0].plot(train.index.values[0:50000], train.earthquake_time.values[0:50000], c=\"Blue\")\nax[0].set_xlabel(\"Index\")\nax[0].set_ylabel(\"Time to Earthquake\")\nax[0].set_title(\"How does the second Earthquaketime pattern look like?\")\nax[1].plot(train.index.values[0:49999], np.diff(train.earthquake_time.values[0:50000]))\nax[1].set_xlabel(\"Index\")\nax[1].set_ylabel(\"Difference between Earthquaketimes\")\nax[1].set_title(\"Are the jumps always the same?\")\nax[2].plot(train.index.values[0:4000], train.earthquake_time.values[0:4000])\nax[2].set_xlabel(\"Index from 0 to 4000\")\nax[2].set_ylabel(\"Earthquake Remaining time\")\nax[2].set_title(\"How does the Earthquaketime changes within the first block?\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from os import listdir","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_path = \"../input/test/\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_files = listdir(\"../input/test/\")\nprint(test_files[0:10])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"To check total segments in test file"},{"metadata":{"trusted":true},"cell_type":"code","source":"len(test_files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.set_option(\"display.precision\", 4)\ntest1 = pd.read_csv('../input/test/seg_37669c.csv', dtype='int16')\nprint(test1.describe())\nplt.figure(figsize=(10,5))\nplt.title(\"Acoustic data distribution\")\nax = sns.distplot(test1.acoustic_data, label='seg_37669c', kde=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nfrom random import shuffle","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_folder_files = os.listdir(\"../input/test/\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Showing distribution of 10 random files from test data"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axis = plt.subplots(5, 2, figsize=(12,14))\nshuffle(test_folder_files)\nxrow = xcol = 0\nfor f in test_folder_files[:10]:\n    tmp = pd.read_csv('../input/test/{}'.format(f), dtype='int16')\n    ax = sns.distplot(tmp.acoustic_data, label=f.replace('.csv',''), ax=axis[xrow][xcol], kde=False)\n    if xcol == 0:\n        xcol += 1\n    else:\n        xcol = 0\n        xrow += 1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Time series data for the same 10 files"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axis = plt.subplots(5, 2, figsize=(12,14))\nxrow = xcol = 0\nfor f in test_folder_files[:10]:\n    tmp = pd.read_csv('../input/test/{}'.format(f), dtype='int16')\n    ax = sns.lineplot(data=tmp.acoustic_data.values,\n                      label=f.replace('.csv',''),\n                      ax=axis[xrow][xcol],\n                      color='orange')\n    if xcol == 0:\n        xcol += 1\n    else:\n        xcol = 0\n        xrow += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission = pd.read_csv(\"../input/sample_submission.csv\")\nsample_submission.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(sample_submission)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Thus test file and submission have same number of segments ids"},{"metadata":{},"cell_type":"markdown","source":"Exploring how the signal of test file looks like:"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(5,1, figsize=(20,25))\n\nfor n in range(5):\n    seg = pd.read_csv(test_path  + test_files[n])\n    ax[n].plot(seg.acoustic_data.values, c=\"mediumseagreen\")\n    ax[n].set_xlabel(\"Index\")\n    ax[n].set_ylabel(\"Signal\")\n    ax[n].set_ylim([-300, 300])\n    ax[n].set_title(\"Test {}\".format(test_files[n]));","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,2, figsize=(20,5))\nsns.distplot(train.signal.values, ax=ax[0], color=\"Green\", bins=100, kde=False)\nax[0].set_xlabel(\"Signal\")\nax[0].set_ylabel(\"Density\")\nax[0].set_title(\"Signal distribution\")\n\nlow = train.signal.mean() - 3 * train.signal.std()\nhigh = train.signal.mean() + 3 * train.signal.std() \nsns.distplot(train.loc[(train.signal >= low) & (train.signal <= high), \"signal\"].values,\n             ax=ax[1],\n             color=\"Green\",\n             bins=150, kde=False)\nax[1].set_xlabel(\"Signal\")\nax[1].set_ylabel(\"Density\")\nax[1].set_title(\"Signal distribution without peaks\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"stepsize = np.diff(train.earthquake_time)\ntrain = train.drop(train.index[len(train)-1])\ntrain[\"stepsize\"] = stepsize\ntrain.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.stepsize = train.stepsize.apply(lambda l: np.round(l, 10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"stepsize_counts = train.stepsize.value_counts()\nstepsize_counts","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import TimeSeriesSplit\n\ncv = TimeSeriesSplit(n_splits=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"window_sizes = [10, 50, 100, 500, 1000, 2000]\nfor window in window_sizes:\n    train[\"rolling_mean_\" + str(window)] = train.signal.rolling(window=window).mean()\n    train[\"rolling_std_\" + str(window)] = train.signal.rolling(window=window).std()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(len(window_sizes),1,figsize=(20,6*len(window_sizes)))\n\nn = 0\nfor col in train.columns.values:\n    if \"rolling_\" in col:\n        if \"mean\" in col:\n            mean_df = train.iloc[4435000:4445000][col]\n            ax[n].plot(mean_df, label=col, color=\"green\")\n        if \"std\" in col:\n            std = train.iloc[4435000:4445000][col].values\n            ax[n].fill_between(mean_df.index.values,\n                               mean_df.values-std, mean_df.values+std,\n                               facecolor='lightblue',\n                               alpha = 0.5, label=col)\n            ax[n].legend()\n            n+=1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Thus it shows from the graph that a window size of 50 is enough for the given dataset"},{"metadata":{},"cell_type":"markdown","source":"Exploring some basic features like mean, std deviation, min, max etc"},{"metadata":{"trusted":true},"cell_type":"code","source":"train[\"rolling_q25\"] = train.signal.rolling(window=50).quantile(0.25)\ntrain[\"rolling_q75\"] = train.signal.rolling(window=50).quantile(0.75)\ntrain[\"rolling_q50\"] = train.signal.rolling(window=50).quantile(0.5)\ntrain[\"rolling_iqr\"] = train.rolling_q75 - train.rolling_q25\ntrain[\"rolling_min\"] = train.signal.rolling(window=50).min()\ntrain[\"rolling_max\"] = train.signal.rolling(window=50).max()\ntrain[\"rolling_skew\"] = train.signal.rolling(window=50).skew()\ntrain[\"rolling_kurt\"] = train.signal.rolling(window=50).kurt()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"peaks = train[train.signal.abs() > 500]\npeaks.earthquake_time.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10,5))\nplt.title(\"Cumulative distribution - time to failure with high signal\")\nax = sns.distplot(peaks.earthquake_time, hist_kws=dict(cumulative=True), kde_kws=dict(cumulative=True))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rolling_mean = []\nrolling_std = []\nlast_time = []","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"rolling_mean = []\nrolling_std = []\nlast_time = []\ninit_idx = 0\nfor _ in range(4194):  # 629M / 150k = 4194\n    x = train.iloc[init_idx:init_idx + 150000]\n   # last_time.append(x.earthquake_time.values[0])\n    rolling_mean.append(x.signal.abs().mean())\n    rolling_std.append(x.signal.abs().std())\n    init_idx += 150000\n    \nrolling_mean = np.array(rolling_mean)\nlast_time = np.array(last_time)\n\n# plot rolling mean\nfig, ax1 = plt.subplots(figsize=(10, 5))\nfig.suptitle('Mean for chunks with 150k samples of training data', fontsize=14)\n\nax2 = ax1.twinx()\nax1.set_xlabel('index')\nax1.set_ylabel('Acoustic data')\nax2.set_ylabel('Time to failure')\np1 = sns.lineplot(data=rolling_mean, ax=ax1, color='orange')\np2 = sns.lineplot(data=last_time, ax=ax2, color='gray')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Thus it is seen from the above graph that std deviation is higher for chunks that are closer to the earthquake time "},{"metadata":{},"cell_type":"markdown","source":"# RNN Model: "},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from numpy.random import seed\nseed(639)\nfrom tensorflow import set_random_seed\nset_random_seed(5944)\n\nfloat_data = pd.read_csv(\"../input/train.csv\", dtype={\"acoustic_data\": np.float32, \"time_to_failure\": np.float32}).values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def extract_features(z):\n     return np.c_[z.mean(axis=1), \n                  np.transpose(np.percentile(np.abs(z), q=[0, 50, 75, 100], axis=1)),\n                  z.std(axis=1)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_X(x, last_index=None, n_steps=150, step_length=1000):\n    if last_index == None:\n        last_index=len(x)\n       \n    assert last_index - n_steps * step_length >= 0\n\n    # Reshaping and approximate standardization with mean 5 and std 3.\n    #temp = (x[(last_index - n_steps * step_length):last_index].reshape(n_steps, -1) - 5 ) / 3\n    temp = (x[(last_index - n_steps * step_length):last_index].reshape(n_steps, -1).astype(np.float32) - 5 ) / 3\n    \n    # Extracts features of sequences of full length 1000, of the last 100 values and finally also \n    # of the last 10 observations. \n    return np.c_[extract_features(temp),\n                 extract_features(temp[:, -step_length // 10:]),\n                 extract_features(temp[:, -step_length // 100:])]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Generating features"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Query \"create_X\" to figure out the number of features\nn_features = create_X(float_data[0:150000]).shape[1]\nprint(\"RNN is based on %i features\"% n_features)\n    \n# The generator endlessly selects \"batch_size\" ending positions of sub-time series. For each ending position,\n# the \"time_to_failure\" serves as target, while the features are created by the function \"create_X\".\ndef generator(data, min_index=0, max_index=None, batch_size=16, n_steps=150, step_length=1000):\n    if max_index is None:\n        max_index = len(data) - 1\n     \n    while True:\n        # Pick indices of ending positions\n        rows = np.random.randint(min_index + n_steps * step_length, max_index, size=batch_size)\n         \n        # Initialize feature matrices and targets\n        samples = np.zeros((batch_size, n_steps, n_features))\n        targets = np.zeros(batch_size, )\n        \n        for j, row in enumerate(rows):\n            samples[j] = create_X(data[:, 0], last_index=row, n_steps=n_steps, step_length=step_length)\n            targets[j] = data[row - 1, 1]\n        yield samples, targets\n        \nbatch_size = 32\n\n# Position of second (of 16) earthquake. Used to have a clean split\n# between train and validation\nsecond_earthquake = 50085877\nfloat_data[second_earthquake, 1]\n\n# Initialize generators\n# train_gen = generator(float_data, batch_size=batch_size) # Use this for better score\ntrain_gen = generator(float_data, batch_size=batch_size, min_index=second_earthquake + 1)\nvalid_gen = generator(float_data, batch_size=batch_size, max_index=second_earthquake)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.contrib.rnn import *\nfrom keras.models import Sequential\nfrom keras.layers import Dense, CuDNNLSTM, LSTM \nfrom keras.optimizers import adam\nfrom keras.callbacks import ModelCheckpoint\nimport tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cb = [ModelCheckpoint(\"model.hdf5\", save_best_only=True, period=3)]\n\nmodel = Sequential()\nmodel.add(LSTM(48, input_shape=(None, n_features)))\nmodel.add(Dense(10, activation='relu'))\nmodel.add(Dense(1))\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"model.compile(optimizer=adam(lr=0.0005), loss=\"mae\")\n\nhistory = model.fit_generator(train_gen,\n                              steps_per_epoch=150,\n                              epochs=50,\n                              verbose=2,\n                              callbacks=cb,\n                              validation_data=valid_gen,\n                              validation_steps=300)"},{"metadata":{},"cell_type":"markdown","source":"import matplotlib.pyplot as plt\n\ndef perf_plot(history, what = 'loss'):\n    x = history.history[what]\n    val_x = history.history['val_' + what]\n    epochs = np.asarray(history.epoch) + 1\n    \n    plt.plot(epochs, x, 'bo', label = \"Training \" + what)\n    plt.plot(epochs, val_x, 'b', label = \"Validation \" + what)\n    plt.title(\"Training and validation \" + what)\n    plt.xlabel(\"Epochs\")\n    plt.legend()\n    plt.show()\n    return None\n\nperf_plot(history)"},{"metadata":{},"cell_type":"markdown","source":"submission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id', dtype={\"time_to_failure\": np.float32})\n\n# Load each test data, create the feature matrix, get numeric prediction\nfor i, seg_id in enumerate(tqdm(submission.index)):\n  #  print(i)\n    seg = pd.read_csv('../input/test/' + seg_id + '.csv')\n    x = seg['acoustic_data'].values\n    submission.time_to_failure[i] = model.predict(np.expand_dims(create_X(x), 0))\n\nsubmission.head()\n\n# Save\nsubmission.to_csv('submission_rnn.csv')"},{"metadata":{},"cell_type":"markdown","source":"### RNN with Optimizer as \"SGD\" and Cost function as \"Hinge\""},{"metadata":{"trusted":true},"cell_type":"code","source":"import keras\nimport keras.utils\nfrom keras import utils as np_utils\nfrom keras import optimizers\nfrom keras.optimizers import sgd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(optimizer=sgd(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True), loss=\"hinge\")\n\nhistory = model.fit_generator(train_gen,\n                              steps_per_epoch=150,\n                              epochs=20,\n                              verbose=2,\n                              callbacks=cb,\n                              validation_data=valid_gen,\n                              validation_steps=300)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id', dtype={\"time_to_failure\": np.float32})\n\n# Load each test data, create the feature matrix, get numeric prediction\nfor i, seg_id in enumerate(tqdm(submission.index)):\n  #  print(i)\n    seg = pd.read_csv('../input/test/' + seg_id + '.csv')\n    x = seg['acoustic_data'].values\n    submission.time_to_failure[i] = model.predict(np.expand_dims(create_X(x), 0))\n\nsubmission.head()\n\n# Save\nsubmission.to_csv('submission_rnn1.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# NN Model:"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nimport gc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rows = 150000\nsegments = int(np.floor(train.shape[0] / rows))\n\nX_train = pd.DataFrame(index = range(segments),dtype = np.float32,columns = ['mean','std','99quat','50quat','25quat','1quat'])\ny_train = pd.DataFrame(index = range(segments),dtype = np.float32,columns = ['time_to_failure'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for segment in tqdm(range(segments)):\n    x = train.iloc[segment*rows:segment*rows+rows]\n    y = x['earthquake_time'].values[-1]\n    x = x['signal'].values\n    X_train.loc[segment,'mean'] = np.mean(x)\n    X_train.loc[segment,'std']  = np.std(x)\n    X_train.loc[segment,'99quat'] = np.quantile(x,0.99)\n    X_train.loc[segment,'50quat'] = np.quantile(x,0.5)\n    X_train.loc[segment,'25quat'] = np.quantile(x,0.25)\n    X_train.loc[segment,'1quat'] =  np.quantile(x,0.01)\n    y_train.loc[segment,'time_to_failure'] = y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scaler = StandardScaler()\nX_scaler = scaler.fit_transform(X_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"model = Sequential()\nmodel.add(Dense(32,input_shape = (6,),activation = 'relu'))\nmodel.add(Dense(32,activation = 'relu'))\nmodel.add(Dense(32,activation = 'relu'))\nmodel.add(Dense(32,activation = 'relu'))\nmodel.add(Dense(1))\nmodel.compile(loss = 'mae',optimizer = 'adam')"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"model.fit(X_scaler,y_train.values.flatten(), epochs = 100)"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"sub_data = pd.read_csv('../input/sample_submission.csv',index_col = 'seg_id')"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"X_test = pd.DataFrame(columns = X_train.columns,dtype = np.float32,index = sub_data.index)"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"for seq in tqdm(X_test.index):\n    test_data = pd.read_csv('../input/test/'+seq+'.csv')\n    x = test_data['acoustic_data'].values\n    X_test.loc[seq,'mean'] = np.mean(x)\n    X_test.loc[seq,'std']  = np.std(x)\n    X_test.loc[seq,'99quat'] = np.quantile(x,0.99)\n    X_test.loc[seq,'50quat'] = np.quantile(x,0.5)\n    X_test.loc[seq,'25quat'] = np.quantile(x,0.25)\n    X_test.loc[seq,'1quat'] =  np.quantile(x,0.01)"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"X_test_scaler = scaler.transform(X_test)"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"pred = model.predict(X_test_scaler)"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"sub_data.head()"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"sub_data['time_to_failure'] = pred\nsub_data['seg_id'] = sub_data.index"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"sub_data.to_csv('sub_earthquake.csv',index = False)"},{"metadata":{},"cell_type":"markdown","source":"### NN with Hinge as Cost function and Adadelta as Optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Sequential()\nmodel.add(Dense(32,input_shape = (6,),activation = 'relu'))\nmodel.add(Dense(32,activation = 'relu'))\nmodel.add(Dense(32,activation = 'relu'))\nmodel.add(Dense(1))\nmodel.compile(loss = 'hinge',optimizer = 'adadelta')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(X_scaler,y_train.values.flatten(), epochs = 100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_data = pd.read_csv('../input/sample_submission.csv',index_col = 'seg_id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test = pd.DataFrame(columns = X_train.columns,dtype = np.float32,index = sub_data.index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for seq in tqdm(X_test.index):\n    test_data = pd.read_csv('../input/test/'+seq+'.csv')\n    x = test_data['acoustic_data'].values\n    X_test.loc[seq,'mean'] = np.mean(x)\n    X_test.loc[seq,'std']  = np.std(x)\n    X_test.loc[seq,'99quat'] = np.quantile(x,0.99)\n    X_test.loc[seq,'50quat'] = np.quantile(x,0.5)\n    X_test.loc[seq,'25quat'] = np.quantile(x,0.25)\n    X_test.loc[seq,'1quat'] =  np.quantile(x,0.01)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test_scaler = scaler.transform(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred = model.predict(X_test_scaler)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_data['time_to_failure'] = pred\nsub_data['seg_id'] = sub_data.index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_data.to_csv('sub_earthquake_nn1.csv',index = False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CatBoostRegressor"},{"metadata":{"trusted":true},"cell_type":"code","source":"from multiprocessing import Pool\nfrom catboost import CatBoostRegressor","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X=X_train.copy()\ny=y_train.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train_pool = Pool(X,y)\ncat_model = CatBoostRegressor(\n                               iterations=10000, \n                               learning_rate=0.03,\n                               eval_metric='MAE',\n                               verbose=1\n                              )\ncat_model.fit(X,y,silent=False)\ny_pred_cat = cat_model.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission['time_to_failure'] = y_pred_cat\nsubmission.to_csv('submission_cat.csv')","execution_count":84,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# SVM"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test = pd.DataFrame(columns=X_train.columns, dtype=np.float64, index=submission.index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scaler = StandardScaler()\nscaler.fit(X_train)\nX_train_scaled = scaler.transform(X_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test_scaled = scaler.transform(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X=X_train.copy()\ny=y_train.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.svm import NuSVR\nfrom sklearn.metrics import mean_absolute_error","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"svm = NuSVR()\nsvm.fit(X_train_scaled, y_train.values.flatten())\ny_pred_svm = svm.predict(X_train_scaled)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"score = mean_absolute_error(y_train.values.flatten(), y_pred_svm)\nprint(f'Score: {score:0.3f}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# LGBM"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import KFold\nimport time\nimport lightgbm as lgb\nfrom sklearn import svm, datasets\nfrom sklearn.model_selection import GridSearchCV\n#set_params(**params)\n\nfrom sklearn.model_selection import ParameterGrid\nfrom sklearn.svm import SVC\nparam_grid = ParameterGrid({'C': [.1, 1, 10], 'gamma':[\"auto\", 0.01]})\n\nfor params in param_grid:\n    svc_clf = SVC(**params)\n    print (svc_clf)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"folds = KFold(n_splits=20, shuffle=True, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ny_pred_lgb = np.zeros(len(X_test_scaled))\nfor fold_n, (train_index, valid_index) in tqdm(enumerate(folds.split(X))):\n    print('Fold', fold_n, 'started at', time.ctime())\n    X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]\n    y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]\n        \n    model = lgb.LGBMRegressor(**params, n_estimators = 22000, n_jobs = -1)\n    model.fit(X_train, y_train, \n                    eval_set=[(X_train, y_train), (X_valid, y_valid)], eval_metric='mae',\n                    verbose=1000, early_stopping_rounds=200)\n            \n    y_pred_valid = model.predict(X_valid)\n    y_pred_lgb += model.predict(X_test_scaled, num_iteration=model.best_iteration_) / folds.n_splits","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}