{"cells":[{"metadata":{"_uuid":"89a9787b8041b9b8537ff383f08543da0ce6c2c7"},"cell_type":"markdown","source":"Base on: https://www.kaggle.com/braquino/5-fold-lstm-attention-fully-commented-0-694\n\nI just changed the model."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\n\nimport pandas as pd\nimport numpy as np\n\nimport pyarrow.parquet as pq\n\nfrom keras.layers import *\nfrom keras.callbacks import *\nfrom keras.initializers import *\nfrom keras import optimizers\nfrom keras import backend as K\nfrom keras.models import Model\n\nimport tensorflow as tf\n\nfrom sklearn.model_selection import StratifiedKFold\n\nfrom tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1ad8b5f4f4745057282f162d60bf2e3fc4c9b398"},"cell_type":"code","source":"sample_size = 800000","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('../input/metadata_train.csv')\ndf_train = df_train.set_index(['id_measurement', 'phase'])\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"79f6d08fbe1ea2ce2e29f461ee161694205fca67"},"cell_type":"code","source":"max_num = 127\nmin_num = -128\n\ndef min_max_transf(ts, min_data, max_data, range_needed=(-1,1)):\n    if min_data < 0:\n        ts_std = (ts + abs(min_data)) / (max_data + abs(min_data))\n    else:\n        ts_std = (ts - min_data) / (max_data - min_data)\n    if range_needed[0] < 0:    \n        return ts_std * (range_needed[1] + abs(range_needed[0])) + range_needed[0]\n    else:\n        return ts_std * (range_needed[1] - range_needed[0]) + range_needed[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"560a6b4f4035afd8531df23fe18d64b72c2759dc"},"cell_type":"code","source":"def transform_ts(ts, n_dim=160, min_max=(-1,1)):\n    ts_std = min_max_transf(ts, min_data=min_num, max_data=max_num)\n    bucket_size = int(sample_size / n_dim)\n    new_ts = []\n    for i in range(0, sample_size, bucket_size):\n        ts_range = ts_std[i:i + bucket_size]\n        mean = ts_range.mean()\n        std = ts_range.std()\n        std_top = mean + std\n        std_bot = mean - std\n        percentil_calc = np.percentile(ts_range, [0, 1, 25, 50, 75, 99, 100]) \n        max_range = percentil_calc[-1] - percentil_calc[0]\n        relative_percentile = percentil_calc - mean\n        new_ts.append(np.concatenate([np.asarray([mean, std, std_top, std_bot, max_range]),percentil_calc, relative_percentile]))\n    return np.asarray(new_ts)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4b840e1c138536a4652f56d346c95e259881f868"},"cell_type":"code","source":"def prep_data(start, end):\n    praq_train = pq.read_pandas('../input/train.parquet', columns=[str(i) for i in range(start, end)]).to_pandas()\n    X = []\n    y = []\n    for id_measurement in tqdm(df_train.index.levels[0].unique()[int(start/3):int(end/3)]):\n        X_signal = []\n        for phase in [0,1,2]:\n            signal_id, target = df_train.loc[id_measurement].loc[phase]\n            if phase == 0:\n                y.append(target)\n            X_signal.append(transform_ts(praq_train[str(signal_id)]))\n        X_signal = np.concatenate(X_signal, axis=1)\n        X.append(X_signal)\n    X = np.asarray(X)\n    y = np.asarray(y)\n    return X, y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e6f6097c38861999314659fdc584ee971398cd8b"},"cell_type":"code","source":"X = []\ny = []\ndef load_all():\n    total_size = len(df_train)\n    for ini, end in [(0, int(total_size/2)), (int(total_size/2), total_size)]:\n        X_temp, y_temp = prep_data(ini, end)\n        X.append(X_temp)\n        y.append(y_temp)\nload_all()\nX = np.concatenate(X)\ny = np.concatenate(y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"71582faae33fe03c49a8420c372536fcd0ca12db"},"cell_type":"code","source":"def matthews_correlation(y_true, y_pred):\n    y_pred = tf.convert_to_tensor(y_pred, np.float32)\n    y_true = tf.convert_to_tensor(y_true, np.float32)\n    y_pred_pos = K.round(K.clip(y_pred, 0, 1))\n    y_pred_neg = 1 - y_pred_pos\n    y_pos = K.round(K.clip(y_true, 0, 1))\n    y_neg = 1 - y_pos\n    tp = K.sum(y_pos * y_pred_pos)\n    tn = K.sum(y_neg * y_pred_neg)\n    fp = K.sum(y_neg * y_pred_pos)\n    fn = K.sum(y_pos * y_pred_neg)\n    numerator = (tp * tn - fp * fn)\n    denominator = K.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn))\n    return numerator / (denominator + K.epsilon())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cd38f6bf59f7471c2fbebb8a2209d116082e38d9"},"cell_type":"code","source":"class Attention(Layer):\n    def __init__(self, step_dim,\n                 W_regularizer=None, b_regularizer=None,\n                 W_constraint=None, b_constraint=None,\n                 bias=True, **kwargs):\n        self.supports_masking = True\n        self.init = initializers.get('glorot_uniform')\n        self.W_regularizer = regularizers.get(W_regularizer)\n        self.b_regularizer = regularizers.get(b_regularizer)\n        self.W_constraint = constraints.get(W_constraint)\n        self.b_constraint = constraints.get(b_constraint)\n        self.bias = bias\n        self.step_dim = step_dim\n        self.features_dim = 0\n        super(Attention, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight((input_shape[-1],),\n                                 initializer=self.init,\n                                 name='{}_W'.format(self.name),\n                                 regularizer=self.W_regularizer,\n                                 constraint=self.W_constraint)\n        self.features_dim = input_shape[-1]\n        if self.bias:\n            self.b = self.add_weight((input_shape[1],),\n                                     initializer='zero',\n                                     name='{}_b'.format(self.name),\n                                     regularizer=self.b_regularizer,\n                                     constraint=self.b_constraint)\n        else:\n            self.b = None\n        self.built = True\n\n    def compute_mask(self, input, input_mask=None):\n        return None\n\n    def call(self, x, mask=None):\n        features_dim = self.features_dim\n        step_dim = self.step_dim\n\n        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n\n        if self.bias:\n            eij += self.b\n        eij = K.tanh(eij)\n        a = K.exp(eij)\n        if mask is not None:\n            a *= K.cast(mask, K.floatx())\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n        a = K.expand_dims(a)\n        weighted_input = x * a\n        return K.sum(weighted_input, axis=1)\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[0], self.features_dim","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"913d9afc4474af3ef9ea5f20940c3b78a370fa6c"},"cell_type":"code","source":"def squash(x, axis=-1):\n    s_squared_norm = K.sum(K.square(x), axis, keepdims=True)\n    scale = K.sqrt(s_squared_norm + K.epsilon())\n    return x / scale\n\nclass Capsule(Layer):\n    def __init__(self, num_capsule, dim_capsule, routings=3, kernel_size=(9, 1), share_weights=True,\n                 activation='default', **kwargs):\n        super(Capsule, self).__init__(**kwargs)\n        self.num_capsule = num_capsule\n        self.dim_capsule = dim_capsule\n        self.routings = routings\n        self.kernel_size = kernel_size\n        self.share_weights = share_weights\n        if activation == 'default':\n            self.activation = squash\n        else:\n            self.activation = Activation(activation)\n\n    def build(self, input_shape):\n        super(Capsule, self).build(input_shape)\n        input_dim_capsule = input_shape[-1]\n        if self.share_weights:\n            self.W = self.add_weight(name='capsule_kernel',\n                                     shape=(1, input_dim_capsule,\n                                            self.num_capsule * self.dim_capsule),\n                                     # shape=self.kernel_size,\n                                     initializer='glorot_uniform',\n                                     trainable=True)\n        else:\n            input_num_capsule = input_shape[-2]\n            self.W = self.add_weight(name='capsule_kernel',\n                                     shape=(input_num_capsule,\n                                            input_dim_capsule,\n                                            self.num_capsule * self.dim_capsule),\n                                     initializer='glorot_uniform',\n                                     trainable=True)\n\n    def call(self, u_vecs):\n        if self.share_weights:\n            u_hat_vecs = K.conv1d(u_vecs, self.W)\n        else:\n            u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])\n\n        batch_size = K.shape(u_vecs)[0]\n        input_num_capsule = K.shape(u_vecs)[1]\n        u_hat_vecs = K.reshape(u_hat_vecs, (batch_size, input_num_capsule,\n                                            self.num_capsule, self.dim_capsule))\n        u_hat_vecs = K.permute_dimensions(u_hat_vecs, (0, 2, 1, 3))\n        # final u_hat_vecs.shape = [None, num_capsule, input_num_capsule, dim_capsule]\n\n        b = K.zeros_like(u_hat_vecs[:, :, :, 0])  # shape = [None, num_capsule, input_num_capsule]\n        for i in range(self.routings):\n            b = K.permute_dimensions(b, (0, 2, 1))  # shape = [None, input_num_capsule, num_capsule]\n            c = K.softmax(b)\n            c = K.permute_dimensions(c, (0, 2, 1))\n            b = K.permute_dimensions(b, (0, 2, 1))\n            outputs = self.activation(tf.keras.backend.batch_dot(c, u_hat_vecs, [2, 2]))\n            if i < self.routings - 1:\n                b = tf.keras.backend.batch_dot(outputs, u_hat_vecs, [2, 3])\n\n        return outputs\n\n    def compute_output_shape(self, input_shape):\n        return (None, self.num_capsule, self.dim_capsule)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3aa5fa1c733349b96c74c77b1f8911eb67b6983"},"cell_type":"code","source":"def model_lstm(input_shape):\n    inp = Input(shape=(input_shape[1], input_shape[2],))\n    x = Bidirectional(CuDNNLSTM(128, return_sequences=True,\n                               kernel_initializer=glorot_normal(seed=1029),\n                               recurrent_initializer=orthogonal(gain=1.0, seed=1029)))(inp)\n    x = Bidirectional(CuDNNLSTM(128, return_sequences=True,\n                               kernel_initializer=glorot_normal(seed=1029),\n                               recurrent_initializer=orthogonal(gain=1.0, seed=1029)))(x)\n    x_1 = Attention(input_shape[1])(x)\n    x_1 = Dropout(0.5)(x_1)\n    \n    x_2 = Capsule(num_capsule=8, dim_capsule=8, routings=4, share_weights=True)(x)\n    x_2 = Flatten()(x_2)\n    x_2 = Dropout(0.5)(x_2)\n    \n    x_rcnn = Conv1D(filters=128, \n                    kernel_size=1, \n                    kernel_initializer='he_uniform')(inp)\n    x_rcnn = Activation('relu')(x_rcnn)\n    x_rcnn_atten = Attention(input_shape[1])(x_rcnn)\n    x_rcnn_capsule = Capsule(num_capsule=8, dim_capsule=8, routings=4, share_weights=True)(x_rcnn)\n    x_rcnn_capsule = Flatten()(x_rcnn_capsule)\n    \n    conc = concatenate([x_1, x_2, x_rcnn_atten, x_rcnn_capsule])\n    conc = Dense(512, activation=\"relu\")(conc)\n    conc = Dropout(0.5)(conc)\n    outp = Dense(1, activation=\"sigmoid\")(conc)\n    model = Model(inputs=inp, outputs=outp)\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[matthews_correlation])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8fcfac5dae2ac2f43eb2226cb88f79e55a62dc08","scrolled":true},"cell_type":"code","source":"N_SPLITS = 5\n\nsplits = list(StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=1029).split(X, y))\npreds_val = []\ny_val = []\nbest_scores = []\n\nfor idx, (train_idx, val_idx) in enumerate(splits):\n    K.clear_session()\n    print(\"Beginning fold {}\".format(idx+1))\n    train_X, train_y, val_X, val_y = X[train_idx], y[train_idx], X[val_idx], y[val_idx]\n    model = model_lstm(train_X.shape)\n    ckpt = ModelCheckpoint('weights_{}.h5'.format(idx), save_best_only=True, save_weights_only=True, verbose=1, monitor='val_matthews_correlation', mode='max')\n    history = model.fit(train_X, train_y, batch_size=128, epochs=50, validation_data=[val_X, val_y], callbacks=[ckpt])\n    best_scores.append(np.max(history.history['val_matthews_correlation']))\n    model.load_weights('weights_{}.h5'.format(idx))\n    preds_val.append(model.predict(val_X, batch_size=512))\n    y_val.append(val_y)\n    \nprint(\"\\n\" + str(best_scores))\nprint(\"\\n\" + str(np.mean(best_scores)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fb9a0dd8045a9cf4208a8e055fa4212a77394159"},"cell_type":"code","source":"preds_val = np.concatenate(preds_val)[...,0]\ny_val = np.concatenate(y_val)\nprint(preds_val.shape, y_val.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"31f88862ddc83b96e3be6fa94f5e6cb030fcfd2b"},"cell_type":"code","source":"def threshold_search(y_true, y_proba):\n    best_threshold = 0\n    best_score = 0\n    for threshold in tqdm([i * 0.01 for i in range(100)]):\n        score = K.eval(matthews_correlation(y_true.astype(np.float64), (y_proba > threshold).astype(np.float64)))\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n    search_result = {'threshold': best_threshold, 'matthews_correlation': best_score}\n    return search_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"044da6a8df5ec835efe0fdd882fac47da9e6dff7"},"cell_type":"code","source":"best_threshold = threshold_search(y_val, preds_val)['threshold']\nprint(best_threshold)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8f9412bcdaecf379a517f45df0e039bc17010b46"},"cell_type":"code","source":"meta_test = pd.read_csv('../input/metadata_test.csv')\nmeta_test = meta_test.set_index(['signal_id'])\nmeta_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"53db0a06193043091785f9d0eaea2cbb8c4f0aa5"},"cell_type":"code","source":"%%time\n\nfirst_sig = meta_test.index[0]\nn_parts = 10\nmax_line = len(meta_test)\npart_size = int(max_line / n_parts)\nlast_part = max_line % n_parts\nstart_end = [[x, x+part_size] for x in range(first_sig, max_line + first_sig, part_size)]\nstart_end = start_end[:-1] + [[start_end[-1][0], start_end[-1][0] + last_part]]\nX_test = []\nfor start, end in start_end:\n    subset_test = pq.read_pandas('../input/test.parquet', columns=[str(i) for i in range(start, end)]).to_pandas()\n    for i in tqdm(subset_test.columns):\n        id_measurement, phase = meta_test.loc[int(i)]\n        subset_test_col = subset_test[i]\n        subset_trans = transform_ts(subset_test_col)\n        X_test.append([i, id_measurement, phase, subset_trans])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f1368cba5ed4eb473b87b2ec31411cb8a1bea4a"},"cell_type":"code","source":"submission = pd.read_csv('../input/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eec8f69a87e89800fbc704b781b22f531691ba9c"},"cell_type":"code","source":"X_test_input = np.asarray([np.concatenate([X_test[i][3],X_test[i+1][3], X_test[i+2][3]], axis=1) for i in range(0,len(X_test), 3)])\nX_test_input.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6cb529e68974e17ca6b7bab5c229f05026aae92b"},"cell_type":"code","source":"preds_test = []\nfor i in range(N_SPLITS):\n    model.load_weights('weights_{}.h5'.format(i))\n    pred = model.predict(X_test_input, batch_size=300, verbose=1)\n    pred_3 = []\n    for pred_scalar in pred:\n        for i in range(3):\n            pred_3.append(pred_scalar)\n    preds_test.append(pred_3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1eff2205bc1aed5de63a09d08ce27110e8512d46"},"cell_type":"code","source":"preds_test = (np.squeeze(np.mean(preds_test, axis=0)) > best_threshold).astype(np.int)\npreds_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"69af441f1d6e1f202fe18cbf116f47d1a825ccad"},"cell_type":"code","source":"submission['target'] = preds_test\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"20bc9b178013cb4e1db08ce5a49078ab3d62e8d7"},"cell_type":"code","source":"submission.target.value_counts().sort_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3c3f058b0d0fb2ed8d23c41151de5d24ca99a769"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}