{"cells":[{"metadata":{},"cell_type":"markdown","source":"# CNN-LSTM without features"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.svm import NuSVR\nfrom sklearn.metrics import mean_absolute_error","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv', dtype={'acoustic_data': np.float32, 'time_to_failure': np.float32}).values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train [0:150000, 0 ] .mean(axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# pandas doesn't show us all the decimals\npd.options.display.precision = 15","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rows = 150_000\nsegments = int(np.floor(train.shape[0] / rows))\nprint('train.shape',train.shape)\nsegments\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# for data set without features"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_X(x, last_index=None, n_steps=150, step_length=1000):\n    if last_index == None:\n        last_index=len(x)\n       \n    assert last_index - n_steps * step_length >= 0\n\n    \n    # Reshaping and approximate standardization with mean 5 and std 3.\n    temp = (x[(int(last_index) - n_steps * step_length):int(last_index)].reshape(n_steps,step_length ).astype(np.float32) - 5 ) / 3   \n    # convert (150000) to [150 1000 ]\n    # then extract feature from each row of length 1000. so total 150 \n    \n    # Extracts features of sequences of full length 1000, of the last 100 values and finally also \n    # of the last 10 observations. \n    \n    return temp\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"[](http://)"},{"metadata":{},"cell_type":"markdown","source":"# Creating Training Data without features"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Query \"create_X\" to figure out the number of features\nn_features = create_X(train [0:150000,0]).shape\nprint(\"Output segment shape\", n_features)     # 18 features each row of segment ie 150x18 features of 150000 chunk input\n\nn_steps=150\nstep_length=1000\nmaxsize=train .shape[0]\nseg = int(np.floor(maxsize / (n_steps*step_length))) \nbatch_size = seg-1   # (4193,) \nxx=350\n\n\n##############################################################################################\nrows_initialize = np.zeros((seg), dtype=float)\nprint(rows_initialize.shape)\n\nfor seg1 in tqdm(range(1,seg)) :      # for loop from 1 to 4194 segment value\n    rows_initialize [seg1] = seg1 * (n_steps*step_length) \n\nrows=np.delete(rows_initialize,0)    # (4193,)\n\nprint(rows.shape)\n\n########################################################################################\nbatch_size=batch_size-xx    # training data\n#batch_size=xx              # validation data\nsplit_point=xx\nsecond_earthquake = rows[xx]\n\n\n\n##########################################################################################\n\nif batch_size < 1000  :    # validation set \n               rows_1 = rows[:split_point+1]    #  0:350 \n        \nif batch_size > 1000 :   # training set\n               rows_1 = rows[split_point+1 :]    # (351,) ie 351:4193    \n            \n\n       \n    # Initialize feature matrices and targets\nsamples_tr= np.zeros((rows_1.shape[0], n_features[0], n_features[1]), dtype=float)   #  for validation (350,150000)  for training ( 3842, 150000) \ntargets_tr = np.zeros(rows_1.shape[0], )    # (16,)  for validation (350)    for training ( 3843)\n        \nfor j, row in enumerate(rows_1):             # 16 for validation (350)    for training ( 3843)\n    samples_tr[j] = create_X(train[:, 0], last_index=row, n_steps=n_steps, step_length=step_length)\n    targets_tr[j] = train[int(row - 1), 1]         \n    \n    \n################################################################################################\n\nprint('samples_tr shape', samples_tr.shape)\nprint('targets_tr shape', targets_tr.shape)\n\nsamples_tr.shape\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Creating Validation data without features"},{"metadata":{"trusted":true},"cell_type":"code","source":"#batch_size=batch_size-xx    # training data\nbatch_size=xx              # validation data\nsplit_point=xx\nsecond_earthquake = rows[xx]\n\n##########################################################################################\n\nif batch_size < 1000  :    # validation set \n               rows_1 = rows[:split_point+1]    #  0:350 \n        \nif batch_size > 1000 :   # training set\n               rows_1 = rows[split_point+1 :]    # (351,) ie 351:4193    \n            \n\n       \n    # Initialize feature matrices and targets\nsamples_vd= np.zeros((rows_1.shape[0], n_features[0], n_features[1]), dtype=float)    #  for validation (350,150000)  for training ( 3842, 150000) \ntargets_vd = np.zeros(rows_1.shape[0], )    # (16,)  for validation (350)    for training ( 3843)\n        \nfor j, row in enumerate(rows_1):             # 16 for validation (350)    for training ( 3843)\n    samples_vd[j] = create_X(train[:, 0], last_index=row, n_steps=n_steps, step_length=step_length)\n    targets_vd[j] = train[int(row - 1), 1]         \n    \n    \n################################################################################################\n\n    \nprint('samples_tr shape', samples_tr.shape)\nprint('targets_tr shape',targets_tr.shape) \n    \nprint('samples_vd shape', samples_vd.shape)\nprint('targets_vd shape',targets_vd.shape)  \n#print('rows_1 shape',rows_1.shape[0])\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Define the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Dense, CuDNNGRU, SimpleRNN, LSTM ,  Dropout, Activation, Flatten, Input, Conv1D, MaxPooling1D\nfrom keras.optimizers import adam\nfrom keras.callbacks import ModelCheckpoint\nfrom keras.optimizers import RMSprop\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## CNN combined with LSTM Model \ni = (n_features[0],n_features[1])\nmodel = Sequential ()\n\nmodel.add(Conv1D (kernel_size = (3), filters = 32, strides=2, input_shape=i, kernel_initializer='he_normal', activation='relu')) \n#model.add(BatchNormalization())\nmodel.add(Dropout(0.2))\n\nmodel.add(Conv1D (kernel_size = (3), filters = 16, strides=2, kernel_initializer='he_normal', activation='relu')) \n#model.add(BatchNormalization())\nmodel.add(Dropout(0.2))\nmodel.add(MaxPooling1D())\n\nmodel.add(Conv1D (kernel_size = (3), filters = 8, strides=2, kernel_initializer='he_normal', activation='relu')) \n#model.add(BatchNormalization())\nmodel.add(Dropout(0.2))\nmodel.add(MaxPooling1D())\n\n\n#model.add(Flatten())\n#model.add(Dense (250, activation='relu', kernel_initializer='he_normal'))\n#model.add(BatchNormalization())\n#model.add(Dropout(0.5))\n    \nmodel.add(LSTM(256,  return_sequences=True))\nmodel.add(Dropout(0.2))\nmodel.add(LSTM(128))\nmodel.add(Dropout(0.2))\n\n\n\n\n\nmodel.add(Dense(256))\nmodel.add(Dense(128))\nmodel.add(Dense(64))\nmodel.add(Dense(32))\nmodel.add(Dense(16))\nmodel.add(Dense(8))\nmodel.add(Dense(4))\nmodel.add(Dense(1))\n\n\n\n##1st model\n#model.add(Conv1D(5, 3, activation='relu', input_shape= i))\n#model.add(MaxPooling1D(2))\n#model.add(LSTM(50,  return_sequences=True))\n#model.add(LSTM(10))\n#model.add(Dense(240))\n#model.add(Dense(120))\n#model.add(Dense(60))\n#model.add(Dense(30))\n#model.add(Dense(1))\n\n##2nd model \n\n#model.add(Conv1D(16, 3, activation='relu', input_shape= i))\n#model.add(MaxPooling1D(2))\n#model.add(Conv1D(128, 3, activation='relu'))\n#model.add(MaxPooling1D(2))\n#model.add(Conv1D(16, 3, activation='relu'))\n#model.add(MaxPooling1D(2))\n#model.add(Dropout(0.1))\n#model.add(LSTM(48,dropout=0.2, recurrent_dropout=0.2, return_sequences=True))\n#model.add(LSTM(20,dropout=0.2, recurrent_dropout=0.2, return_sequences=False))\n#model.add(Dense(1, activation='linear'))\n ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Compile and fit model"},{"metadata":{"trusted":true},"cell_type":"code","source":"import keras\nfrom keras.optimizers import RMSprop\nopt = keras.optimizers.adam(lr=.005)\n\nmodel.compile(loss=\"mae\",\n              optimizer=opt, metrics=['mean_absolute_error'])\n             # metrics=['accuracy'])\n\n\nbatch_size = 128 # mini-batch with 32 examples\nepochs = 50\nhistory = model.fit(\n    samples_tr, targets_tr,\n    batch_size=batch_size,\n    epochs=epochs,\n    verbose=1)\n   # validation_data=(samples_vd  ,targets_vd ))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# Load submission file\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id', dtype={\"time_to_failure\": np.float32})","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prepare submission data\nLoad each test data, create the feature matrix, get numeric prediction"},{"metadata":{"trusted":true},"cell_type":"code","source":"for i, seg_id in enumerate(tqdm(submission.index)):\n  #  print(i)\n    seg = pd.read_csv('../input/test/' + seg_id + '.csv')\n    x = seg['acoustic_data'].values\n    submission.time_to_failure[i] = model.predict(np.expand_dims(create_X(x), 0))\n\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Save submission file"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}