{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport psutil\nimport math\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom tqdm import tqdm_notebook\nfrom sklearn.metrics import mean_absolute_error\npd.options.display.precision = 15\n\nimport time\nimport datetime\n\nimport gc\nimport seaborn as sns\nimport tensorflow as tf\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":1,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"%%time\ntrain = pd.read_csv('../input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32})","execution_count":2,"outputs":[{"output_type":"stream","text":"CPU times: user 2min 4s, sys: 11.5 s, total: 2min 15s\nWall time: 2min 16s\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_mean = train['acoustic_data'].mean()\nX_std = train['acoustic_data'].std()\ny_mean = train['time_to_failure'].mean()\ny_std = train['time_to_failure'].std()","execution_count":3,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cut_time = []\nfor i in range(0,len(train)-10000,10000):\n if train['time_to_failure'][i] < train['time_to_failure'][i + 10000]:\n cut_time.append(i)\naccurate_cut_time = [0]\nfor t in cut_time:\n for i in range(t,t+10000):\n if train['time_to_failure'][i] < train['time_to_failure'][i + 1]:\n accurate_cut_time.append(i + 1)\n break\naccurate_cut_time.append(len(train))","execution_count":4,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_all = []\nfor i in range(len(accurate_cut_time)-1):\n df_all.append(train.iloc[accurate_cut_time[i]:accurate_cut_time[i] + 150000 * ((accurate_cut_time[i+1] - accurate_cut_time[i]) // 150000)])","execution_count":5,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train\ndel cut_time\ndel accurate_cut_time\ngc.collect()","execution_count":6,"outputs":[{"output_type":"execute_result","execution_count":6,"data":{"text/plain":"18"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_df = []\nval_target = []\nfor df in df_all[:3]:\n segments = int(np.floor(df.shape[0] / 150000))\n for segment in tqdm_notebook(range(segments)):\n seg = df.iloc[segment*150000:segment*150000+150000]\n x_raw = seg['acoustic_data']\n val_df.append((x_raw.values - X_mean) / X_std)\n val_target.append(seg['time_to_failure'].values[-1])\nval_df = np.array(val_df).reshape((-1, 150000,1))\nval_target = np.array(val_target).reshape((-1, 1))","execution_count":7,"outputs":[{"output_type":"display_data","data":{"text/plain":"HBox(children=(IntProgress(value=0, max=37), HTML(value='')))","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8d4117764cbb4f789219487c9d48c35b"}},"metadata":{}},{"output_type":"stream","text":"\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"HBox(children=(IntProgress(value=0, max=296), HTML(value='')))","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8311895cd1e7454f881a281a6640feed"}},"metadata":{}},{"output_type":"stream","text":"\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"HBox(children=(IntProgress(value=0, max=363), HTML(value='')))","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b7a4ce60f62a424786a428f2a5028e0d"}},"metadata":{}},{"output_type":"stream","text":"\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"import keras\nfrom keras.models import Model\nfrom keras.layers import *\nfrom keras.optimizers import Adam,SGD\nfrom keras.backend import clear_session\nimport tensorflow.keras.backend as K\nimport tensorflow as tf\n\nclass Attention(Layer):\n def __init__(self, step_dim,\n W_regularizer=None, b_regularizer=None,\n W_constraint=None, b_constraint=None,\n bias=True, **kwargs):\n self.supports_masking = True\n self.init = initializers.get('glorot_uniform')\n\n self.W_regularizer = regularizers.get(W_regularizer)\n self.b_regularizer = regularizers.get(b_regularizer)\n\n self.W_constraint = constraints.get(W_constraint)\n self.b_constraint = constraints.get(b_constraint)\n\n self.bias = bias\n self.step_dim = step_dim\n self.features_dim = 0\n super(Attention, self).__init__(**kwargs)\n\n def build(self, input_shape):\n assert len(input_shape) == 3\n\n self.W = self.add_weight((input_shape[-1],),\n initializer=self.init,\n name='{}_W'.format(self.name),\n regularizer=self.W_regularizer,\n constraint=self.W_constraint)\n self.features_dim = input_shape[-1]\n\n if self.bias:\n self.b = self.add_weight((input_shape[1],),\n initializer='zero',\n name='{}_b'.format(self.name),\n regularizer=self.b_regularizer,\n constraint=self.b_constraint)\n else:\n self.b = None\n\n self.built = True\n\n def compute_mask(self, input, input_mask=None):\n return None\n\n def call(self, x, mask=None):\n features_dim = self.features_dim\n step_dim = self.step_dim\n\n eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n\n if self.bias:\n eij += self.b\n\n eij = K.tanh(eij)\n\n a = K.exp(eij)\n\n if mask is not None:\n a *= K.cast(mask, K.floatx())\n\n a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n\n a = K.expand_dims(a)\n weighted_input = x * a\n return K.sum(weighted_input, axis=1)\n\n def compute_output_shape(self, input_shape):\n return input_shape[0], self.features_dim","execution_count":8,"outputs":[{"output_type":"stream","text":"Using TensorFlow backend.\n","name":"stderr"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"clear_session()\ninp = Input(shape=(150000,1))\n\n# First_LSTM_Cell = Bidirectional(CuDNNLSTM(32, return_sequences=True))\n# First_Attention = Attention(400)\n# x = []\n# for i in range(375):\n# temp = First_LSTM_Cell(Lambda(lambda x: keras.backend.slice(x, (0,i*400,0), (-1,400,-1)))(inp))\n# x.append(First_Attention(temp))\n# x = Concatenate()(x)\n# x = Reshape((375,128))(x)\n\nx = inp\ndilation_rates = [2**i for i in range(10)] \nfor dilation_rate in dilation_rates:\n tanh_out = Conv1D(filters=16,\n kernel_size=15, \n padding='same',\n activation = 'tanh',\n dilation_rate=dilation_rate)(x)\n sigm_out = Conv1D(filters=16,\n kernel_size=15, \n padding='same',\n activation = 'sigmoid',\n dilation_rate=dilation_rate)(x)\n x = Multiply()([tanh_out,sigm_out])\n x = Conv1D(filters = 16,\n kernel_size = 1,\n padding='same',\n activation = 'relu',)(x)\n x = BatchNormalization()(x)\n x = SpatialDropout1D(0.2)(x)\n\nx = Conv1D(filters=16,\n kernel_size=60, \n strides = 20,\n# activation='relu',\n padding='same')(inp)\n\ndilation_rates = [2**i for i in range(5)] \nfor dilation_rate in dilation_rates:\n tanh_out = Conv1D(filters=32,\n kernel_size=15, \n padding='same',\n activation = 'tanh',\n dilation_rate=dilation_rate)(x)\n sigm_out = Conv1D(filters=32,\n kernel_size=15, \n padding='same',\n activation = 'sigmoid',\n dilation_rate=dilation_rate)(x)\n x = Multiply()([tanh_out,sigm_out])\n x = Conv1D(filters = 32,\n kernel_size = 1,\n padding='same',\n activation = 'relu',)(x)\n x = BatchNormalization()(x)\n x = SpatialDropout1D(0.2)(x)\n \nx = Conv1D(filters=32,\n kernel_size=60, \n strides = 20,\n# activation='relu',\n padding='same')(x)\n\nx = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)\nx = Attention(375)(x)\nx = Dropout(0.2)(x)\nx = Dense(64, activation=\"relu\")(x)\nx = Dense(1)(x)\n\nmodel = Model(inputs = inp, outputs=x)\nmodel.summary()","execution_count":20,"outputs":[{"output_type":"stream","text":"__________________________________________________________________________________________________\nLayer (type) Output Shape Param # Connected to \n==================================================================================================\ninput_1 (InputLayer) (None, 150000, 1) 0 \n__________________________________________________________________________________________________\nconv1d_31 (Conv1D) (None, 7500, 16) 976 input_1[0][0] \n__________________________________________________________________________________________________\nconv1d_32 (Conv1D) (None, 7500, 32) 7712 conv1d_31[0][0] \n__________________________________________________________________________________________________\nconv1d_33 (Conv1D) (None, 7500, 32) 7712 conv1d_31[0][0] \n__________________________________________________________________________________________________\nmultiply_11 (Multiply) (None, 7500, 32) 0 conv1d_32[0][0] \n conv1d_33[0][0] \n__________________________________________________________________________________________________\nconv1d_34 (Conv1D) (None, 7500, 32) 1056 multiply_11[0][0] \n__________________________________________________________________________________________________\nbatch_normalization_11 (BatchNo (None, 7500, 32) 128 conv1d_34[0][0] \n__________________________________________________________________________________________________\nspatial_dropout1d_11 (SpatialDr (None, 7500, 32) 0 batch_normalization_11[0][0] \n__________________________________________________________________________________________________\nconv1d_35 (Conv1D) (None, 7500, 32) 15392 spatial_dropout1d_11[0][0] \n__________________________________________________________________________________________________\nconv1d_36 (Conv1D) (None, 7500, 32) 15392 spatial_dropout1d_11[0][0] \n__________________________________________________________________________________________________\nmultiply_12 (Multiply) (None, 7500, 32) 0 conv1d_35[0][0] \n conv1d_36[0][0] \n__________________________________________________________________________________________________\nconv1d_37 (Conv1D) (None, 7500, 32) 1056 multiply_12[0][0] \n__________________________________________________________________________________________________\nbatch_normalization_12 (BatchNo (None, 7500, 32) 128 conv1d_37[0][0] \n__________________________________________________________________________________________________\nspatial_dropout1d_12 (SpatialDr (None, 7500, 32) 0 batch_normalization_12[0][0] \n__________________________________________________________________________________________________\nconv1d_38 (Conv1D) (None, 7500, 32) 15392 spatial_dropout1d_12[0][0] \n__________________________________________________________________________________________________\nconv1d_39 (Conv1D) (None, 7500, 32) 15392 spatial_dropout1d_12[0][0] \n__________________________________________________________________________________________________\nmultiply_13 (Multiply) (None, 7500, 32) 0 conv1d_38[0][0] \n conv1d_39[0][0] \n__________________________________________________________________________________________________\nconv1d_40 (Conv1D) (None, 7500, 32) 1056 multiply_13[0][0] \n__________________________________________________________________________________________________\nbatch_normalization_13 (BatchNo (None, 7500, 32) 128 conv1d_40[0][0] \n__________________________________________________________________________________________________\nspatial_dropout1d_13 (SpatialDr (None, 7500, 32) 0 batch_normalization_13[0][0] \n__________________________________________________________________________________________________\nconv1d_41 (Conv1D) (None, 7500, 32) 15392 spatial_dropout1d_13[0][0] \n__________________________________________________________________________________________________\nconv1d_42 (Conv1D) (None, 7500, 32) 15392 spatial_dropout1d_13[0][0] \n__________________________________________________________________________________________________\nmultiply_14 (Multiply) (None, 7500, 32) 0 conv1d_41[0][0] \n conv1d_42[0][0] \n__________________________________________________________________________________________________\nconv1d_43 (Conv1D) (None, 7500, 32) 1056 multiply_14[0][0] \n__________________________________________________________________________________________________\nbatch_normalization_14 (BatchNo (None, 7500, 32) 128 conv1d_43[0][0] \n__________________________________________________________________________________________________\nspatial_dropout1d_14 (SpatialDr (None, 7500, 32) 0 batch_normalization_14[0][0] \n__________________________________________________________________________________________________\nconv1d_44 (Conv1D) (None, 7500, 32) 15392 spatial_dropout1d_14[0][0] \n__________________________________________________________________________________________________\nconv1d_45 (Conv1D) (None, 7500, 32) 15392 spatial_dropout1d_14[0][0] \n__________________________________________________________________________________________________\nmultiply_15 (Multiply) (None, 7500, 32) 0 conv1d_44[0][0] \n conv1d_45[0][0] \n__________________________________________________________________________________________________\nconv1d_46 (Conv1D) (None, 7500, 32) 1056 multiply_15[0][0] \n__________________________________________________________________________________________________\nbatch_normalization_15 (BatchNo (None, 7500, 32) 128 conv1d_46[0][0] \n__________________________________________________________________________________________________\nspatial_dropout1d_15 (SpatialDr (None, 7500, 32) 0 batch_normalization_15[0][0] \n__________________________________________________________________________________________________\nconv1d_47 (Conv1D) (None, 375, 32) 61472 spatial_dropout1d_15[0][0] \n__________________________________________________________________________________________________\nbidirectional_1 (Bidirectional) (None, 375, 128) 50176 conv1d_47[0][0] \n__________________________________________________________________________________________________\nattention_1 (Attention) (None, 128) 503 bidirectional_1[0][0] \n__________________________________________________________________________________________________\ndropout_1 (Dropout) (None, 128) 0 attention_1[0][0] \n__________________________________________________________________________________________________\ndense_1 (Dense) (None, 64) 8256 dropout_1[0][0] \n__________________________________________________________________________________________________\ndense_2 (Dense) (None, 1) 65 dense_1[0][0] \n==================================================================================================\nTotal params: 265,928\nTrainable params: 265,608\nNon-trainable params: 320\n__________________________________________________________________________________________________\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"def generator():\n batch_size = 16\n while True:\n X = []\n Y = []\n for i in range(batch_size):\n while True:\n land = 3 + np.random.randint(len(df_all)-3)\n point = np.random.randint(int((len(df_all[land])-150000)/100))\n seg = df_all[land].iloc[point*100:point*100+150000]\n if seg['time_to_failure'].values[-1] < seg['time_to_failure'].values[0]:\n break\n x_raw = seg['acoustic_data']\n x = (x_raw.values - X_mean) / X_std\n y = seg['time_to_failure'].values[-1]\n# y = (y - y_mean) / y_std\n X.append(x)\n Y.append(y)\n yield np.array(X).reshape((-1, 150000,1)),np.array(Y).reshape((-1, 1))","execution_count":21,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import LearningRateScheduler,ModelCheckpoint,EarlyStopping\ndef step_decay(epoch):\n x = 0.0001\n if epoch >= 200: x = 0.00001\n if epoch >= 500: x = 0.000001\n return x\nlr_decay = LearningRateScheduler(step_decay)","execution_count":22,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(Adam(lr=0.0001), loss='mae')\nhistory = model.fit_generator(generator(),\n steps_per_epoch=400,\n epochs=1000,\n verbose=2,\n callbacks = [ModelCheckpoint(\"model.hdf5\", save_best_only=True,save_weights_only = True, period=1), lr_decay, EarlyStopping(patience = 50)],\n validation_data = (val_df,val_target)\n )","execution_count":null,"outputs":[{"output_type":"stream","text":"Epoch 1/1000\n - 47s - loss: 2.5283 - val_loss: 2.4016\nEpoch 2/1000\n - 42s - loss: 1.8912 - val_loss: 2.4754\nEpoch 3/1000\n - 42s - loss: 1.9501 - val_loss: 2.4220\nEpoch 4/1000\n - 42s - loss: 1.9104 - val_loss: 2.4752\nEpoch 5/1000\n - 41s - loss: 1.8659 - val_loss: 2.5236\nEpoch 6/1000\n - 42s - loss: 1.8829 - val_loss: 2.5199\nEpoch 7/1000\n - 41s - loss: 1.8851 - val_loss: 2.3959\nEpoch 8/1000\n - 42s - loss: 1.8770 - val_loss: 2.3685\nEpoch 9/1000\n - 41s - loss: 1.8634 - val_loss: 2.4483\nEpoch 10/1000\n - 42s - loss: 1.8461 - val_loss: 2.3494\nEpoch 11/1000\n - 41s - loss: 1.8572 - val_loss: 2.3476\nEpoch 12/1000\n - 41s - loss: 1.8552 - val_loss: 2.3832\nEpoch 13/1000\n - 41s - loss: 1.8124 - val_loss: 2.1923\nEpoch 14/1000\n - 42s - loss: 1.8088 - val_loss: 2.4483\nEpoch 15/1000\n - 42s - loss: 1.7870 - val_loss: 2.2507\nEpoch 16/1000\n - 41s - loss: 1.8074 - val_loss: 2.2863\nEpoch 17/1000\n - 41s - loss: 1.7807 - val_loss: 2.2355\nEpoch 18/1000\n - 42s - loss: 1.8235 - val_loss: 2.3735\nEpoch 19/1000\n - 41s - loss: 1.7939 - val_loss: 2.2968\nEpoch 20/1000\n - 41s - loss: 1.7825 - val_loss: 2.2311\nEpoch 21/1000\n - 41s - loss: 1.6960 - val_loss: 2.2526\nEpoch 22/1000\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_weights(\"model.hdf5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id')\npred_test = []\nfor i, seg_id in enumerate(tqdm_notebook(submission.index)):\n seg = pd.read_csv('../input/test/' + seg_id + '.csv')\n x_raw = seg['acoustic_data']\n pred_test.append(model.predict(((x_raw.values - X_mean) / X_std).reshape((-1, 150000,1))).reshape((1)))\nsubmission['time_to_failure'] = np.array(pred_test)\nprint(submission.head())\nsubmission.to_csv('submission.csv')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}