{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport warnings\nimport gc\n\nimport pyarrow.parquet as pq\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom tqdm import tqdm_notebook as tqdm\nfrom sklearn.model_selection import train_test_split\n\n%matplotlib inline\nwarnings.filterwarnings('ignore')\nprint(os.listdir(\"../input\"))\nprint(os.listdir(\"./\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a79d2495b269a201958b7fddce368bf149bc3d1a"},"cell_type":"code","source":"BATCH_SIZE = 64\nMERGE_SIZE = 400","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"metadata_train = pd.read_csv('../input/metadata_train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7022aec7b0546cceae68785e2bbb235dad2dd97a"},"cell_type":"code","source":"def read_wave_data(parquet_path,col_nums,end_col_num, merge_size=800):\n    df_diff = None\n    for i, col_num in tqdm(enumerate(col_nums)):\n        start = col_num\n        if i == len(col_nums) - 1:\n            end = end_col_num\n        else:\n            end = col_nums[i + 1]\n        columns = [str(j) for j in range(start,end)]\n        tmp_df = pq.read_pandas(parquet_path, columns=columns).to_pandas()\n        group_id = np.repeat(range(len(tmp_df) // merge_size), merge_size)\n        tmp_df['group_id'] = pd.Series(group_id)\n        tmp_diff = (tmp_df.groupby('group_id').max() - tmp_df.groupby('group_id').min()) / 256\n        if df_diff is None:\n            df_diff = tmp_diff\n        else:\n            df_diff = pd.concat([df_diff, tmp_diff], axis=1)\n    df_diff = df_diff.astype('float16')\n    return df_diff","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"01d5ca1e912544a0d59e8441e5af374ffe63dc0a"},"cell_type":"code","source":"train_parquet_path = '../input/train.parquet'\nend_col_num = metadata_train['signal_id'].values[-1] + 1\ncol_nums = metadata_train['signal_id'].values[::500].tolist()\ntrain_diff = read_wave_data(train_parquet_path,col_nums,end_col_num,merge_size=MERGE_SIZE)\nprint(train_diff.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"07ffc46537deb92ff1749e4b4d6aea1548607f1f"},"cell_type":"code","source":"def train_data_gen(metadata_train, train_diff, batch_size=128, is_reverse=False):\n    np.random.seed(1)\n    while True:\n        x_train = []\n        y_train = []\n        true_sample = metadata_train[metadata_train['target']==1].sample(batch_size // 2)\n        neg_sample = metadata_train[metadata_train['target']==0].sample(batch_size // 2)\n    \n        sample_signal_id = np.concatenate([true_sample['signal_id'].values,neg_sample['signal_id'].values])\n        np.random.shuffle(sample_signal_id)\n        for signal_id in sample_signal_id:\n            diffs = train_diff[str(signal_id)].values.T\n            \n            if is_reverse:\n                diffs = diffs[::-1]\n            \n            data = diffs[:, np.newaxis]\n            x_train.append(data)\n            y_train.append(metadata_train[metadata_train['signal_id']==signal_id]['target'].values[0])\n            \n        x_train = np.array(x_train)\n        y_train = np.array(y_train)\n        yield x_train, y_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3d62370032f9dce8e491e60b95b493fa51640c95"},"cell_type":"code","source":"metadata_train, metadata_val = train_test_split(metadata_train, test_size=0.2, random_state=42)\nprint(metadata_train.shape)\nprint(metadata_val.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2ce3d20f975a802e7cdca337db317531adb4f164"},"cell_type":"code","source":"x_val = []\ny_val = []\nfor signal_id in metadata_val['signal_id'].values:\n    diffs = train_diff[str(signal_id)].values.T\n    data = diffs[:, np.newaxis]\n    x_val.append(data)\n    y_val.append(metadata_val[metadata_val['signal_id']==signal_id]['target'].values[0])\nx_val = np.array(x_val)\ny_val = np.array(y_val)    \nprint(x_val.shape)\nprint(y_val.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"434b85fcbddefc7e7852020909bf2a98bdd6c58a"},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras import layers\n\nfrom keras.optimizers import Adam\nfrom keras.callbacks import ModelCheckpoint, LearningRateScheduler, EarlyStopping, ReduceLROnPlateau\nimport keras.models as models\nimport keras.backend as K","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6279b47e4dc18482e4ebc26aefd115c21477473f"},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\n\ndef mcc(y_true, y_pred):\n    cm = confusion_matrix(y_true, y_pred)\n    TP = cm[0][0]\n    FP = cm[0][1]\n    FN = cm[1][0]\n    TN = cm[1][1]\n    val = ((TP * TN) - (FP * FN)) / ((TP + FP)*(TP + FN)*(TN + FP)*(TN + FN))**0.5\n    return val","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3b93dbf9fed506113f89e7a779677b6e3c7d30b3"},"cell_type":"code","source":"def matthews_corr_coeff(y_true, y_pred):\n    y_pos_pred = K.round(K.clip(y_pred, 0, 1))\n    y_pos_true = K.round(K.clip(y_true, 0, 1))\n    \n    y_neg_pred = 1 - y_pos_pred\n    y_neg_true = 1 - y_pos_true\n\n    tp = K.sum(y_pos_true * y_pos_pred)\n    tn = K.sum(y_neg_true * y_neg_pred)\n    fp = K.sum(y_neg_true * y_pos_pred)\n    fn = K.sum(y_pos_true * y_neg_pred)\n    return (tp * tn - fp * fn) / (K.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn)) + K.epsilon())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"88a9a1390bce9836953811e3d8d7835b3b9b2c8a"},"cell_type":"code","source":"length_of_sequence = train_diff.shape[0]\ndrop_out_rate = 0.2\nrecurrent_dropout = 0.5\nSTEPS_PER_EPOCH = 100\nEPOCHS = 50","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"835bf0f11e59dee510dd036621785c2362384c81"},"cell_type":"code","source":"# Create Model\n\nmodel = Sequential()\nmodel.add(layers.Conv1D(32, 8, \n                 padding='same',\n                 input_shape=(length_of_sequence, 1),\n                 activation='relu'))\nmodel.add(layers.MaxPooling1D(2, padding='same'))\nmodel.add(layers.Conv1D(64, 8, padding='same', activation='relu'))\nmodel.add(layers.MaxPooling1D(2, padding='same'))\nmodel.add(layers.Conv1D(128, 8, padding='same', activation='relu'))\nmodel.add(layers.MaxPooling1D(2, padding='same'))\nmodel.add(layers.Conv1D(256, 8, padding='same', activation='relu'))\nmodel.add(layers.LSTM(64, \n#               return_sequences=True,\n               dropout = drop_out_rate,\n               recurrent_dropout = recurrent_dropout\n              ))\n               #batch_input_shape=(None, 2, length_of_sequence)))\n#model.add(layers.LSTM(128, \n#               dropout = drop_out_rate,\n#               recurrent_dropout = recurrent_dropout\n#              ))\n#model.add(layers.Dense(100,activation='relu'))\nmodel.add(layers.Dense(1,activation='sigmoid'))\n\nmodel.compile(loss='binary_crossentropy',\n              optimizer='rmsprop',\n              metrics=['accuracy',matthews_corr_coeff])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc954861abfe7878cc56909e8ba01ed28b99bc39"},"cell_type":"code","source":"weight_path=\"{}_weights.best.hdf5\".format('lstm_model')\nearly = EarlyStopping(monitor=\"val_loss\", \n                      mode=\"min\", \n                      patience=10) # probably needs to be more patient, but kaggle time is limited\nlr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=2, min_lr=0.001)\ncheckpoint = ModelCheckpoint(weight_path, monitor='val_loss', verbose=1, \n                             save_best_only=True, mode='min', save_weights_only = True)\n\ncallbacks_list = [checkpoint, early, lr]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"08bbb66a0085bece3047ed54f93ae555acb92cc4"},"cell_type":"code","source":"train_gen = train_data_gen(metadata_train, train_diff, batch_size=BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c54dda52eb00978d42af6e4aaf7d4e837e2c9ab0"},"cell_type":"code","source":"history = model.fit_generator(\n                train_gen,\n                steps_per_epoch=STEPS_PER_EPOCH,\n                epochs=EPOCHS,\n                validation_data=(x_val,y_val),\n                callbacks=callbacks_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fba17e77694a35fe3fdddc631fced82357bd7d06"},"cell_type":"code","source":"model.load_weights('lstm_model_weights.best.hdf5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f0bab392881faea748f0d615e184dd15266f2eec"},"cell_type":"code","source":"y_val_pred = model.predict(x_val)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e8863147279006caffcd3bf0333c07ce7d3b106d"},"cell_type":"code","source":"y_val_pred = y_val_pred.flatten()\ny_val_pred[y_val_pred >= 0.5] = 1\ny_val_pred[y_val_pred < 0.5] = 0\ny_val_pred.sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5bc5441bdf51378e4d9a98de3fb66c3fe01fda09"},"cell_type":"code","source":"y_val.sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cdd3d11f6c8a6a2224d9c296286d695449e00054"},"cell_type":"code","source":"mcc(y_val,y_val_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e1f1407e65f0cbd5696f64e869d8c0023469c45"},"cell_type":"markdown","source":"Add predict label to test data and train again."},{"metadata":{"trusted":true,"_uuid":"5d4aa4dc8ccbfc82c3ee6485114ef8febf471ff4"},"cell_type":"code","source":"metadata_test = pd.read_csv('../input/metadata_test.csv')\nmetadata_train = pd.read_csv('../input/metadata_train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"893a4abb834fcc4fb0ca7fa76cacb065ba834622"},"cell_type":"code","source":"test_parquet_path = '../input/test.parquet'\nend_col_num = metadata_test['signal_id'].values[-1] + 1\ncol_nums = metadata_test['signal_id'].values[::500].tolist()\ntest_diff = read_wave_data(test_parquet_path,col_nums,end_col_num,merge_size=MERGE_SIZE)\nprint(test_diff.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6ab5f0c0950a3bef9753763c1584b55dd479aa44"},"cell_type":"code","source":"x_test = []\nfor c in test_diff.columns:\n    diffs = test_diff[c].values.T\n    data = diffs[:, np.newaxis]\n    x_test.append(data)\n    \nx_test = np.array(x_test)\nprint(x_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"88aeba3db7f3267720881ef11499e1114b68976e"},"cell_type":"code","source":"y_test = model.predict(x_test)\ny_test = y_test.flatten()\ny_test[y_test >= 0.5] = 1\ny_test[y_test < 0.5] = 0\nprint(y_test.sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"07f9a81818a5eb11bc7a949bf5ee0213e9414ec3"},"cell_type":"code","source":"#y_test = np.array(y_test,dtype='bool')\nmetadata_test['target'] = pd.Series(y_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0cc492b24ca4fcd091d640e854c487fc859fbf9e"},"cell_type":"code","source":"metadata_all = pd.concat([metadata_train, metadata_test])\nprint(metadata_all.shape)\nmetadata_all.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"59c13c493e64bfe1305ee6d090dcfca8d2754908"},"cell_type":"code","source":"all_diff = pd.concat([train_diff,test_diff],axis=1)\nprint(all_diff.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ec7c5d40fa4875e0e27e17774600c287def0e5c3"},"cell_type":"code","source":"metadata_train, metadata_val = train_test_split(metadata_all, test_size=0.2, random_state=42)\nprint(metadata_train.shape)\nprint(metadata_val.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a2cd20b55af52ebe35a8bc208842d983df62696e"},"cell_type":"code","source":"x_val = []\ny_val = []\nfor signal_id in metadata_val['signal_id'].values:\n    diffs = all_diff[str(signal_id)].values.T\n    data = diffs[:, np.newaxis]\n    x_val.append(data)\n    y_val.append(metadata_val[metadata_val['signal_id']==signal_id]['target'].values[0])\nx_val = np.array(x_val)\ny_val = np.array(y_val)    \nprint(x_val.shape)\nprint(y_val.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea9e346efcdeafbec75e658c63e72ef87780b501"},"cell_type":"code","source":"# Create Model\n\nmodel_2 = Sequential()\nmodel_2.add(layers.Conv1D(32, 8, \n                 padding='same',\n                 input_shape=(length_of_sequence, 1),\n                 activation='relu'))\nmodel_2.add(layers.MaxPooling1D(2, padding='same'))\nmodel_2.add(layers.Conv1D(64, 8, padding='same', activation='relu'))\nmodel_2.add(layers.MaxPooling1D(2, padding='same'))\nmodel_2.add(layers.Conv1D(128, 8, padding='same', activation='relu'))\nmodel_2.add(layers.MaxPooling1D(2, padding='same'))\nmodel_2.add(layers.Conv1D(256, 8, padding='same', activation='relu'))\nmodel_2.add(layers.LSTM(64, \n               return_sequences=True,\n               dropout = drop_out_rate,\n               recurrent_dropout = recurrent_dropout\n              ))\n               #batch_input_shape=(None, 2, length_of_sequence)))\nmodel_2.add(layers.LSTM(128, \n               dropout = drop_out_rate,\n               recurrent_dropout = recurrent_dropout\n              ))\n#model.add(layers.Dense(100,activation='relu'))\nmodel_2.add(layers.Dense(1,activation='sigmoid'))\n\nmodel_2.compile(loss='binary_crossentropy',\n              optimizer='rmsprop',\n              metrics=['accuracy',matthews_corr_coeff])\nmodel_2.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6b3d4e64cd307d3c8b6ffe20dca142503e98ad45"},"cell_type":"code","source":"weight_path=\"{}_weights.best.hdf5\".format('lstm_model_2')\nlr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=2, min_lr=0.001)\ncheckpoint = ModelCheckpoint(weight_path, monitor='val_loss', verbose=1, \n                             save_best_only=True, mode='min', save_weights_only = True)\n\ncallbacks_list = [checkpoint, lr]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"70cea8047bcb0faa69a76aac3c31feaf21ae223c"},"cell_type":"code","source":"train_gen = train_data_gen(metadata_all, all_diff, batch_size=BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d93b3c126628be27aea56d0946acb1e66d915e37"},"cell_type":"code","source":"history = model_2.fit_generator(\n                train_gen,\n                steps_per_epoch=STEPS_PER_EPOCH,\n                epochs=EPOCHS,\n                validation_data=(x_val,y_val),\n                callbacks=callbacks_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"101f11a06754c15957de378b998d74f937aae37a"},"cell_type":"code","source":"model_2.load_weights('lstm_model_2_weights.best.hdf5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f202fd2788e0e6e1531c8ffaa1ed05d13b225c94"},"cell_type":"code","source":"y_val_pred = model_2.predict(x_val)\ny_val_pred = y_val_pred.flatten()\ny_val_pred[y_val_pred >= 0.5] = 1\ny_val_pred[y_val_pred < 0.5] = 0\nprint(y_val_pred.sum())\nprint(y_val.sum())\nprint(mcc(y_val,y_val_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce2150e36249cb0d15c07257e80de361204fb257"},"cell_type":"code","source":"y_test_1 = model.predict(x_test)\ny_test_2 = model_2.predict(x_test)\n\ny_test = (y_test_1.flatten()) * 0.5 + (y_test_2.flatten()) * 0.5\n#y_test = y_test_2.flatten()\ny_test[y_test >= 0.5] = 1\ny_test[y_test < 0.5] = 0\nprint(y_test.sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"249acfd5d44a580461b30926bf7303fdd8af8af8"},"cell_type":"code","source":"submit_df = pd.read_csv('../input/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"09915f111694a65d8e15374d37a7d044e23276a5"},"cell_type":"code","source":"y_test = np.array(y_test,dtype='bool')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9d06032459f93231604f0b4d795bbb8663623ff1"},"cell_type":"code","source":"submit_df['target'] = pd.Series(y_test)\nsubmit_df['target'].astype('bool')\nsubmit_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"798b4f5ebf848f20bca79fd9d90315d7445bca85"},"cell_type":"code","source":"submit_df.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a60a526f4f2bc86cb16aa24900ab0ae7362aee67"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}