{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport psutil\nimport math\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom tqdm import tqdm_notebook\nfrom sklearn.metrics import mean_absolute_error\npd.options.display.precision = 15\n\nimport time\nimport datetime\n\nimport gc\nimport seaborn as sns\nimport tensorflow as tf\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport scipy.signal as sg\nfrom scipy.signal import hann\nfrom scipy.signal import hilbert\nfrom scipy.signal import convolve","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"%%time\ntrain = pd.read_csv('../input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"etq_meta = [\n{\"start\":0,         \"end\":5656574},\n{\"start\":5656574,   \"end\":50085878},\n{\"start\":50085878,  \"end\":104677356},\n{\"start\":104677356, \"end\":138772453},\n{\"start\":138772453, \"end\":187641820},\n{\"start\":187641820, \"end\":218652630},\n{\"start\":218652630, \"end\":245829585},\n{\"start\":245829585, \"end\":307838917},\n{\"start\":307838917, \"end\":338276287},\n{\"start\":338276287, \"end\":375377848},\n{\"start\":375377848, \"end\":419368880},\n{\"start\":419368880, \"end\":461811623},\n{\"start\":461811623, \"end\":495800225},\n{\"start\":495800225, \"end\":528777115},\n{\"start\":528777115, \"end\":585568144},\n{\"start\":585568144, \"end\":621985673},\n{\"start\":621985673, \"end\":629145480},\n]\n\ndf = []\nfor i in [2, 7, 0, 4, 11, 13, 9, 1, 14, 10]:\n    df.append(train[etq_meta[i]['start']:etq_meta[i]['start']+150000*((etq_meta[i]['end'] - etq_meta[i]['start'])//150000)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.concat(df)\ndel df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# NY_FREQ_IDX = 75000  # the test signals are 150k samples long, Nyquist is thus 75k.\n# CUTOFF = 18000\n# MAX_FREQ_IDX = 100000\n# FREQ_STEP = 2500\n\n# def des_bw_filter_lp(cutoff=CUTOFF):  # low pass filter\n#     b, a = sg.butter(4, Wn=cutoff/NY_FREQ_IDX)\n#     return b, a\n\n# def fft_process(x):\n#     xc = x.values\n#     xcdm = xc - np.mean(xc)\n    \n# #     b, a = des_bw_filter_lp(cutoff=18000)\n# #     xcz = sg.lfilter(b, a, xcdm)\n    \n#     zc = np.fft.fft(xcdm)\n#     zc = zc[:MAX_FREQ_IDX]\n\n#     # FFT transform values\n#     realFFT = np.real(zc)\n#     imagFFT = np.imag(zc)\n\n#     return np.swapaxes([realFFT,imagFFT],0,1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_seg = len(train)//150000\ntrain_X = []\ntrain_y = []\nfor i in tqdm_notebook(range(num_seg)):\n#     train_X.append(fft_process(train['acoustic_data'].iloc[150000 * i:150000 * i + 150000]))\n    if 100000 * i + 150000 < len(train):\n        train_X.append(train['acoustic_data'].iloc[150000 * i:150000 * i + 150000])\n        train_y.append(train['time_to_failure'].iloc[150000 * i + 149999])\ndel train\ngc.collect()\ntrain_X = np.array(train_X,dtype = np.float32)\ntrain_y = np.array(train_y,dtype = np.float32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_mean = train_X.mean(0)\nX_std = train_X.std(0)\ntrain_X -= X_mean\ntrain_X /= X_std\ny_mean = train_y.mean()\ny_std = train_y.std()\ntrain_y -= y_mean\ntrain_y /= y_std","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# X_mean = train['acoustic_data'].mean()\n# X_std = train['acoustic_data'].std()\n\n# y_mean = train['time_to_failure'].mean()\n# y_std = train['time_to_failure'].std()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id')\ntest_X = []\nfor i, seg_id in enumerate(tqdm_notebook(submission.index)):\n    seg = pd.read_csv('../input/test/' + seg_id + '.csv')\n    test_X.append(seg['acoustic_data'])\ntest_X = np.array(test_X,dtype = np.float32)\ntest_X -= X_mean\ntest_X /= X_std","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_X = np.expand_dims(train_X,-1)\ntest_X = np.expand_dims(test_X,-1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import keras\nfrom keras.models import Model\nfrom keras.layers import *\nfrom keras.optimizers import Adam,SGD\nfrom keras.backend import clear_session\nimport tensorflow.keras.backend as K\nimport tensorflow as tf\n\nclass Attention(Layer):\n    def __init__(self, step_dim,\n                 W_regularizer=None, b_regularizer=None,\n                 W_constraint=None, b_constraint=None,\n                 bias=True, **kwargs):\n        self.supports_masking = True\n        self.init = initializers.get('glorot_uniform')\n\n        self.W_regularizer = regularizers.get(W_regularizer)\n        self.b_regularizer = regularizers.get(b_regularizer)\n\n        self.W_constraint = constraints.get(W_constraint)\n        self.b_constraint = constraints.get(b_constraint)\n\n        self.bias = bias\n        self.step_dim = step_dim\n        self.features_dim = 0\n        super(Attention, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight((input_shape[-1],),\n                                 initializer=self.init,\n                                 name='{}_W'.format(self.name),\n                                 regularizer=self.W_regularizer,\n                                 constraint=self.W_constraint)\n        self.features_dim = input_shape[-1]\n\n        if self.bias:\n            self.b = self.add_weight((input_shape[1],),\n                                     initializer='zero',\n                                     name='{}_b'.format(self.name),\n                                     regularizer=self.b_regularizer,\n                                     constraint=self.b_constraint)\n        else:\n            self.b = None\n\n        self.built = True\n\n    def compute_mask(self, input, input_mask=None):\n        return None\n\n    def call(self, x, mask=None):\n        features_dim = self.features_dim\n        step_dim = self.step_dim\n\n        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n\n        if self.bias:\n            eij += self.b\n\n        eij = K.tanh(eij)\n\n        a = K.exp(eij)\n\n        if mask is not None:\n            a *= K.cast(mask, K.floatx())\n\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n\n        a = K.expand_dims(a)\n        weighted_input = x * a\n        return K.sum(weighted_input, axis=1)\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[0],  self.features_dim","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def keras_model():\n    def wave_block(x,filters,kernel_size,n):\n        dilation_rates = [2**i for i in range(n)]\n        x = Conv1D(filters=filters,\n                    kernel_size=1, \n                    padding='same')(x)\n        res_x = x\n#         x_ = []\n        for dilation_rate in dilation_rates:\n            tanh_out = Conv1D(filters=filters,\n                    kernel_size=kernel_size, \n                    padding='same',\n                    activation = 'tanh',\n                    dilation_rate=dilation_rate)(x)\n            sigm_out = Conv1D(filters=filters,\n                    kernel_size=kernel_size, \n                    padding='same',\n                    activation = 'sigmoid',\n                    dilation_rate=dilation_rate)(x)\n            x = Multiply()([tanh_out,sigm_out])\n            x = Conv1D(filters = filters,\n                           kernel_size = 1,\n                           padding='same')(x)\n#             x = BatchNormalization()(x)\n#             x = SpatialDropout1D(0.2)(x)\n            res_x = Add()([res_x,x])\n#             x_.append(x)\n        return res_x\n        \n    \n    clear_session()\n    inp = Input(shape=(150000,1))\n    Collect = []\n    x = wave_block(inp,16,3,8)\n    x = AveragePooling1D(10)(x)\n#     x = BatchNormalization()(x)\n#     x = SpatialDropout1D(0.05)(x)\n    x = wave_block(x,32,3,5)\n    x = AveragePooling1D(10)(x)\n#     x = BatchNormalization()(x)\n#     x = SpatialDropout1D(0.05)(x)\n    x = wave_block(x,64,3,3)\n    x = AveragePooling1D(10)(x)\n#     x = BatchNormalization()(x)\n#     x = SpatialDropout1D(0.05)(x)\n    x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)\n    x = Attention(150)(x)\n    x = Dropout(0.2)(x)\n    x = Dense(128, activation=\"relu\")(x)\n    x = Dense(1)(x)\n    model = Model(inputs = inp, outputs=x)\n    return model\nmodel = keras_model()\nmodel.summary()\n# SVG(model_to_dot(model).create(prog='dot', format='svg'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import LearningRateScheduler,ModelCheckpoint,EarlyStopping\ndef step_decay(epoch):\n    x = 0.0001\n    if epoch >= 40: x = 0.00001\n    return x\nlr_decay = LearningRateScheduler(step_decay)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from sklearn.model_selection import KFold\n# n_fold = 5\n# folds = KFold(n_splits=n_fold, shuffle=True, random_state=42)\n# oof = np.zeros(len(train_X))\n# prediction = np.zeros(len(test_X))\n# for fold_, (trn_idx, val_idx) in enumerate(folds.split(train_X, train_y)):\n#     print('working fold %d' % fold_)\n#     strLog = \"fold {}\".format(fold_)\n#     print(strLog)\n    \n#     model = keras_model()\n\n#     X_tr, X_val = train_X[trn_idx], train_X[val_idx]\n#     y_tr, y_val = train_y[trn_idx], train_y[val_idx]\n\n#     model.compile(Adam(lr=0.0005), loss='mse', metrics=['mae'])\n#     model.fit(X_tr,y_tr,\n#              batch_size = 8,\n#              epochs=20,\n#              verbose=2,\n#              callbacks = [ModelCheckpoint(\"model.hdf5\", save_best_only=True,save_weights_only = True, period=1), lr_decay, EarlyStopping(patience = 50)],\n#              validation_data = (X_val,y_val)\n#              )\n#     model.load_weights(\"model.hdf5\")\n#     oof[val_idx] = model.predict(X_val)[:,0]\n#     prediction += model.predict(test_X)[:,0] / n_fold\n# print(\"CV:\",mean_absolute_error(oof,train_y))\n# submission['time_to_failure'] = prediction * y_std + y_mean\n# print(submission.head())\n# submission.to_csv('submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction = np.zeros(len(test_X))\n\nmodel = keras_model()\n\nmodel.compile(Adam(lr=0.00001), loss='mse', metrics=['mae'])\nmodel.fit(train_X,train_y,\n         batch_size = 16,\n         epochs=100,\n         verbose=2\n         )\nprediction = model.predict(test_X)[:,0]\nsubmission['time_to_failure'] = prediction * y_std + y_mean\nprint(submission.head())\nsubmission.to_csv('submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def Generator():\n#     batch_size = 32\n#     while True:\n#         X = []\n#         Y = []\n#         for i in range(batch_size):\n#             while True:\n#                 point = np.random.randint((len(train)-150000))\n#                 seg = train.iloc[point:point+150000]\n#                 if seg['time_to_failure'].values[-1] < seg['time_to_failure'].values[0]:\n#                     break\n#             x_raw = seg['acoustic_data']\n#             x = (x_raw.values - X_mean) / X_std\n#             y = seg['time_to_failure'].values[-1]\n#             y = (y - y_mean) / y_std\n#             X.append(x)\n#             Y.append(y)\n#         yield np.array(X).reshape((-1, 150000,1)),np.array(Y).reshape((-1, 1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# prediction = np.zeros(len(test_X))\n# model = keras_model()\n# model.compile(Adam(lr=0.001), loss='mse', metrics=['mae'])\n# model.fit_generator(Generator(),\n#                      steps_per_epoch=1000,\n#                      epochs=30,\n#                      verbose=2\n#                      )\n# prediction = model.predict(test_X)[:,0]\n# submission['time_to_failure'] = prediction * y_std + y_mean\n# print(submission.head())\n# submission.to_csv('submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}