{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport pyarrow.parquet as pq # Used to read the data\nimport os \nimport numpy as np\nfrom keras.layers import * # Keras is the most friendly Neural Network library, this Kernel use a lot of layers classes\nfrom keras.models import Model\nfrom tqdm import tqdm # Processing time measurement\nfrom sklearn.model_selection import train_test_split \nfrom keras import backend as K # The backend give us access to tensorflow operations and allow us to create the Attention class\nfrom keras import optimizers # Allow us to access the Adam class to modify some parameters\nfrom sklearn.model_selection import GridSearchCV, StratifiedKFold # Used to use Kfold to train our model\nfrom keras.callbacks import * # This object helps the model to train in a smarter way, avoiding overfitting","execution_count":1,"outputs":[{"output_type":"stream","text":"Using TensorFlow backend.\n","name":"stderr"}]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"df_meta_train = pd.read_csv('../input/metadata_train.csv')\ndf_meta_test = pd.read_csv('../input/metadata_test.csv')","execution_count":2,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def obter_sinais_treino(num_amostra):\n    signals = [str(i) for i in df_meta_train[df_meta_train['id_measurement'] == num_amostra]['signal_id']]\n    return pd.read_parquet('../input/train.parquet', columns=signals)","execution_count":3,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def obter_sinais_teste(num_amostra):\n    signals = [str(i) for i in df_meta_test[df_meta_test['id_measurement'] == num_amostra]['signal_id']]\n    return pd.read_parquet('../input/test.parquet', columns=signals)","execution_count":4,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"STEP = 160 #Constante para definir quantas amostras do sinal serão ignoradas\n\ndef preparar_dados_treino(num_amostra):\n    X = np.array([])\n    signals = obter_sinais_treino(num_amostra)\n    for signal in signals:\n        X = np.append(X, signals[signal].values[::STEP])\n    y = np.asscalar(df_meta_train[(df_meta_train['id_measurement'] == num_amostra) & (df_meta_train['phase']==0)].head()['target'])\n    return X, y\n\ndef preparar_dados_teste(num_amostra):\n    X = np.array([])\n    signals = obter_sinais_teste(num_amostra)\n    for signal in signals:\n        X = np.append(X, signals[signal].values[::STEP])\n    return X","execution_count":7,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"amostra_min = df_meta_train['id_measurement'].min()\namostra_max = df_meta_train['id_measurement'].max() + 1","execution_count":5,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = []\ny = []\nfor i in df_meta_train[df_meta_train['phase']==0]['id_measurement']:\n    X_temp, y_temp = preparar_dados_treino(i)\n    X.append(X_temp)\n    y.append(y_temp)\n    \nprint('100%')\n\nX = np.asarray(X)\ny = np.asarray(y)","execution_count":8,"outputs":[{"output_type":"stream","text":"100%\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"sh1 = X.shape[0]\nsh2 = int(X.shape[1] / 3)\nsh3 = 3\nX = X.reshape(sh1, sh2, sh3)\nX.shape","execution_count":9,"outputs":[{"output_type":"execute_result","execution_count":9,"data":{"text/plain":"(6, 5000, 3)"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nprint('Tamanho em memória: %d MB' % ((sys.getsizeof(X)/1024)/1024))","execution_count":10,"outputs":[{"output_type":"stream","text":"Tamanho em memória: 0 MB\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.neural_network import MLPClassifier\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import confusion_matrix","execution_count":11,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# It is the official metric used in this competition\n# below is the declaration of a function used inside the keras model, calculation with K (keras backend / thensorflow)\ndef matthews_correlation(y_true, y_pred):\n    '''Calculates the Matthews correlation coefficient measure for quality\n    of binary classification problems.\n    '''\n    y_pred_pos = K.round(K.clip(y_pred, 0, 1))\n    y_pred_neg = 1 - y_pred_pos\n\n    y_pos = K.round(K.clip(y_true, 0, 1))\n    y_neg = 1 - y_pos\n\n    tp = K.sum(y_pos * y_pred_pos)\n    tn = K.sum(y_neg * y_pred_neg)\n\n    fp = K.sum(y_neg * y_pred_pos)\n    fn = K.sum(y_pos * y_pred_neg)\n\n    numerator = (tp * tn - fp * fn)\n    denominator = K.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn))\n\n    return numerator / (denominator + K.epsilon())","execution_count":12,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Attention(Layer):\n    def __init__(self, step_dim,\n                 W_regularizer=None, b_regularizer=None,\n                 W_constraint=None, b_constraint=None,\n                 bias=True, **kwargs):\n        self.supports_masking = True\n        self.init = initializers.get('glorot_uniform')\n\n        self.W_regularizer = regularizers.get(W_regularizer)\n        self.b_regularizer = regularizers.get(b_regularizer)\n\n        self.W_constraint = constraints.get(W_constraint)\n        self.b_constraint = constraints.get(b_constraint)\n\n        self.bias = bias\n        self.step_dim = step_dim\n        self.features_dim = 0\n        super(Attention, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight((input_shape[-1],),\n                                 initializer=self.init,\n                                 name='{}_W'.format(self.name),\n                                 regularizer=self.W_regularizer,\n                                 constraint=self.W_constraint)\n        self.features_dim = input_shape[-1]\n\n        if self.bias:\n            self.b = self.add_weight((input_shape[1],),\n                                     initializer='zero',\n                                     name='{}_b'.format(self.name),\n                                     regularizer=self.b_regularizer,\n                                     constraint=self.b_constraint)\n        else:\n            self.b = None\n\n        self.built = True\n\n    def compute_mask(self, input, input_mask=None):\n        return None\n\n    def call(self, x, mask=None):\n        features_dim = self.features_dim\n        step_dim = self.step_dim\n\n        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n\n        if self.bias:\n            eij += self.b\n\n        eij = K.tanh(eij)\n\n        a = K.exp(eij)\n\n        if mask is not None:\n            a *= K.cast(mask, K.floatx())\n\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n\n        a = K.expand_dims(a)\n        weighted_input = x * a\n        return K.sum(weighted_input, axis=1)\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[0],  self.features_dim","execution_count":13,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#clf = MLPClassifier(hidden_layer_sizes=(200, 100, 50), max_iter=50, alpha=20, solver='adam', verbose=True, random_state=42, tol=0.5)\n\n# This is NN LSTM Model creation\ndef model_lstm(input_shape):\n    # The shape was explained above, must have this order\n    inp = Input(shape=(input_shape[1], input_shape[2]))\n    # This is the LSTM layer\n    # Bidirecional implies that the 160 chunks are calculated in both ways, 0 to 159 and 159 to zero\n    # although it appear that just 0 to 159 way matter, I have tested with and without, and tha later worked best\n    # 128 and 64 are the number of cells used, too many can overfit and too few can underfit\n    x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(inp)\n    # The second LSTM can give more fire power to the model, but can overfit it too\n    x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)\n    # Attention is a new tecnology that can be applyed to a Recurrent NN to give more meanings to a signal found in the middle\n    # of the data, it helps more in longs chains of data. A normal RNN give all the responsibility of detect the signal\n    # to the last cell. Google RNN Attention for more information :)\n    #x = Flatten()(x)\n    x = Attention(input_shape[1])(x)\n    x = Dense(64, activation=\"relu\")(x)\n    # A binnary classification as this must finish with shape (1,)\n    x = Dense(1, activation=\"sigmoid\")(x)\n    model = Model(inputs=inp, outputs=x)\n    # Pay attention in the addition of matthews_correlation metric in the compilation, it is a success factor key\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[matthews_correlation])\n    \n\n    return model","execution_count":14,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"splits = list(StratifiedKFold(n_splits=5, shuffle=True, random_state=42).split(X, y))\n\npreds_val = []\ny_val = []\n\nfor idx, (train_idx, val_idx) in enumerate(splits):\n    train_X, train_y, val_X, val_y = X[train_idx], y[train_idx], X[val_idx], y[val_idx]\n    \n    model = model_lstm(train_X.shape)\n\n    ckpt = ModelCheckpoint('weights_{}.h5'.format(idx), save_best_only=True, save_weights_only=True, verbose=1, monitor='val_matthews_correlation', mode='max')\n    \n    model.fit(train_X, train_y, batch_size=128, epochs=40, validation_data=[val_X, val_y], callbacks=[ckpt])\n    model.load_weights('weights_{}.h5'.format(idx))\n              \n     # Add the predictions of the validation to the list preds_val\n    preds_val.append(model.predict(val_X, batch_size=512))\n    # and the val true y\n    y_val.append(val_y)\n              \npreds_val = np.concatenate(preds_val)[...,0]\ny_val = np.concatenate(y_val)\npreds_val.shape, y_val.shape","execution_count":16,"outputs":[{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/sklearn/model_selection/_split.py:652: Warning: The least populated class in y has only 1 members, which is too few. The minimum number of members in any class cannot be less than n_splits=5.\n  % (min_groups, self.n_splits)), Warning)\n","name":"stderr"},{"output_type":"stream","text":"Train on 5 samples, validate on 1 samples\nEpoch 1/2\n5/5 [==============================] - 2s 415ms/step - loss: 0.6024 - matthews_correlation: 0.0000e+00 - val_loss: 0.2630 - val_matthews_correlation: 0.0000e+00\n\nEpoch 00001: val_matthews_correlation improved from -inf to 0.00000, saving model to weights_0.h5\nEpoch 2/2\n5/5 [==============================] - 1s 168ms/step - loss: 0.5024 - matthews_correlation: 0.0000e+00 - val_loss: 0.1730 - val_matthews_correlation: 0.0000e+00\n\nEpoch 00002: val_matthews_correlation did not improve from 0.00000\nTrain on 5 samples, validate on 1 samples\nEpoch 1/2\n5/5 [==============================] - 2s 454ms/step - loss: 0.7027 - matthews_correlation: 0.0000e+00 - val_loss: 0.4795 - val_matthews_correlation: 0.0000e+00\n\nEpoch 00001: val_matthews_correlation improved from -inf to 0.00000, saving model to weights_1.h5\nEpoch 2/2\n5/5 [==============================] - 1s 167ms/step - loss: 0.5739 - matthews_correlation: 0.0000e+00 - val_loss: 0.3210 - val_matthews_correlation: 0.0000e+00\n\nEpoch 00002: val_matthews_correlation did not improve from 0.00000\nTrain on 5 samples, validate on 1 samples\nEpoch 1/2\n5/5 [==============================] - 3s 502ms/step - loss: 0.7048 - matthews_correlation: 0.0000e+00 - val_loss: 0.4055 - val_matthews_correlation: 0.0000e+00\n\nEpoch 00001: val_matthews_correlation improved from -inf to 0.00000, saving model to weights_2.h5\nEpoch 2/2\n5/5 [==============================] - 1s 167ms/step - loss: 0.5457 - matthews_correlation: 0.0000e+00 - val_loss: 0.2757 - val_matthews_correlation: 0.0000e+00\n\nEpoch 00002: val_matthews_correlation did not improve from 0.00000\nTrain on 4 samples, validate on 2 samples\nEpoch 1/2\n4/4 [==============================] - 3s 718ms/step - loss: 0.8524 - matthews_correlation: 0.0000e+00 - val_loss: 0.6926 - val_matthews_correlation: 0.0000e+00\n\nEpoch 00001: val_matthews_correlation improved from -inf to 0.00000, saving model to weights_3.h5\nEpoch 2/2\n4/4 [==============================] - 1s 230ms/step - loss: 0.6621 - matthews_correlation: 0.0000e+00 - val_loss: 0.7468 - val_matthews_correlation: 0.0000e+00\n\nEpoch 00002: val_matthews_correlation did not improve from 0.00000\nTrain on 5 samples, validate on 1 samples\nEpoch 1/2\n5/5 [==============================] - 3s 620ms/step - loss: 0.7091 - matthews_correlation: 0.0000e+00 - val_loss: 0.4979 - val_matthews_correlation: 0.0000e+00\n\nEpoch 00001: val_matthews_correlation improved from -inf to 0.00000, saving model to weights_4.h5\nEpoch 2/2\n5/5 [==============================] - 1s 167ms/step - loss: 0.5847 - matthews_correlation: 0.0000e+00 - val_loss: 0.3354 - val_matthews_correlation: 0.0000e+00\n\nEpoch 00002: val_matthews_correlation did not improve from 0.00000\n","name":"stdout"},{"output_type":"execute_result","execution_count":16,"data":{"text/plain":"((6,), (6,))"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\ndef threshold_search(y_true, y_proba):\n    best_threshold = 0\n    best_score = 0\n    for threshold in tqdm([i * 0.05 for i in range(20)]):\n        score = K.eval(matthews_correlation(tf.convert_to_tensor(y_true, np.float64), tf.convert_to_tensor(y_proba > threshold,np.float64)))\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n    search_result = {'threshold': best_threshold, 'matthews_correlation': best_score}\n    return search_result","execution_count":17,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_threshold = threshold_search(y_val, preds_val)['threshold']\nbest_threshold","execution_count":18,"outputs":[{"output_type":"stream","text":"100%|██████████| 20/20 [00:12<00:00,  1.59it/s]\n","name":"stderr"},{"output_type":"execute_result","execution_count":18,"data":{"text/plain":"0.4"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"amostra_min = df_meta_test['id_measurement'].min()\namostra_max = df_meta_test['id_measurement'].max() + 1\nX = []\ny = []\nfor i in range(amostra_min, amostra_max):\n    X_temp = preparar_dados_teste(i)\n    X.append(X_temp)\nprint('100%')\nX = np.asarray(X)","execution_count":19,"outputs":[{"output_type":"stream","text":"100%\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"sh1 = X.shape[0]\nsh2 = int(X.shape[1] / 3)\nsh3 = 3\nX = X.reshape(sh1, sh2, sh3)\nX.shape","execution_count":20,"outputs":[{"output_type":"execute_result","execution_count":20,"data":{"text/plain":"(5, 5000, 3)"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds_test = []\nfor i in range(5):\n    model.load_weights('weights_{}.h5'.format(i))\n    pred = model.predict(X, batch_size=300, verbose=1)\n    preds_test.append(pred)\n\npreds_test = (np.squeeze(np.mean(preds_test, axis=0)) > best_threshold).astype(np.int)","execution_count":21,"outputs":[{"output_type":"stream","text":"5/5 [==============================] - 0s 55ms/step\n5/5 [==============================] - 0s 54ms/step\n5/5 [==============================] - 0s 54ms/step\n5/5 [==============================] - 0s 53ms/step\n5/5 [==============================] - 0s 54ms/step\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_predicted = pd.DataFrame()\ntest_predicted['id_measurement'] = [i for i in range(amostra_min, amostra_max)]\ntest_predicted['target'] = preds_test","execution_count":24,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"output = pd.merge(test_predicted, df_meta_test, on='id_measurement')\noutput.to_csv('submission-lstm-5fold.csv', index=False, columns=['signal_id', 'target'])","execution_count":25,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}