{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport warnings\nimport gc\n\nimport pyarrow.parquet as pq\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom tqdm import tqdm_notebook as tqdm\nfrom sklearn.model_selection import train_test_split\n\n%matplotlib inline\nwarnings.filterwarnings('ignore')\nprint(os.listdir(\"../input\"))\nprint(os.listdir(\"./\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\n\nimport pandas as pd\nimport numpy as np\n\nimport pyarrow.parquet as pq\n\nfrom keras.layers import *\nfrom keras.callbacks import *\nfrom keras.initializers import *\nfrom keras import optimizers\nfrom keras import backend as K\nfrom keras.models import Model\nimport keras\n\nimport tensorflow as tf\n\nfrom sklearn.model_selection import StratifiedKFold\n\nfrom tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a79d2495b269a201958b7fddce368bf149bc3d1a"},"cell_type":"code","source":"BATCH_SIZE = 64\nMERGE_SIZE = 400","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv('../input/wind-gogogo/allData.csv')\n\nprint(\"The shape of allData: \", df.shape)\n\ndata = df.drop(columns=['group'])\nprint(\"The shape of part data: \", data.shape)\n\nprint(data['frozen'].value_counts(normalize=True))\ndataset = data.drop(data.columns[[0]], axis = 1).values\n\ntrain_dataset = dataset[0:-150000]\nval_dataset = dataset[-150000:-80000]\ntest_dataset = dataset[-80000:]\n\nprint(\"train_dataset.shape:\", train_dataset.shape,\n      \"\\nval_dataset.shape: \", val_dataset.shape,\n      \"\\ntest_dataset.shape: \", test_dataset.shape)\n\nepochs = 20\nbatch_size = 64\nverbose = True\n\n# ######################slide windows####################\ndef split_sequences(sequences, n_steps):\n\tX, y = list(), list()\n\tfor i in range(len(sequences)):\n\t\t# find the end of this pattern\n\t\tend_ix = i + n_steps\n\t\t# check if we are beyond the dataset\n\t\tif end_ix > len(sequences):\n\t\t\tbreak\n\t\t# gather input and output parts of the pattern\n\t\tseq_x, seq_y = sequences[i:end_ix, :-1], sequences[end_ix-1, -1]\n\t\tX.append(seq_x)\n\t\ty.append(seq_y)\n\treturn np.array(X), np.array(y)\n# ######################################################","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train=train_dataset[:,:-1]\n# train_label = train_dataset[:,-1]\n# val = val_dataset[:,:-1]\n# val_label = val_dataset[:,-1]\n# test = test_dataset[:,:-1]\n# test_label = test_dataset[:,-1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_steps = 64\ntrain_X, train_y = split_sequences(train_dataset, n_steps)\nval_X, val_y = split_sequences(val_dataset, n_steps)\ntest_X, test_y = split_sequences(test_dataset, n_steps)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_X.shape, train_y.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # data prepare\n# def multivariate_data(dataset, target, start_index, end_index, history_size=32,\n#                       target_size=1, step=1, single_step=False):  \n#     ##history_size:步长，step：滑动间隔，target_size：标签规模，\n#     #start_index：开始索引，end_index:结束索引\n#     data = []\n#     labels = []\n\n#     start_index = start_index + history_size\n#     if end_index is None:\n#         end_index = len(dataset) - target_size\n\n#     for i in range(start_index, end_index):\n#         indices = range(i-history_size, i, step)\n#         data.append(dataset[indices])\n\n#         if single_step:\n#             labels.append(target[i-1+target_size])\n#         else:\n#             labels.append(target[i+1:i+1+target_size])\n\n#     return np.array(data), np.array(labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_X, train_y = multivariate_data(train, train_label, 0, end_index=None)\n# test_X, test_y = multivariate_data(test, test_label, 0, end_index=None)\n# val_X, val_y = multivariate_data(val, val_label, 0, end_index=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = train_X\ny_train = train_y\nx_validate = val_X\ny_validate = val_y\nx_test = test_X\ny_test = test_y\nx_val = val_X\ny_val = val_y\n\n##输入\nx_train[0].shape\ninput_shape = x_train.shape\nnb_classes = 2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"434b85fcbddefc7e7852020909bf2a98bdd6c58a"},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras import layers\n\nfrom keras.optimizers import Adam\nfrom keras.callbacks import ModelCheckpoint, LearningRateScheduler, EarlyStopping, ReduceLROnPlateau\nimport keras.models as models\nimport keras.backend as K","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6279b47e4dc18482e4ebc26aefd115c21477473f"},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\n\ndef mcc(y_true, y_pred):\n    cm = confusion_matrix(y_true, y_pred)\n    TP = cm[0][0]\n    FP = cm[0][1]\n    FN = cm[1][0]\n    TN = cm[1][1]\n    val = ((TP * TN) - (FP * FN)) / ((TP + FP)*(TP + FN)*(TN + FP)*(TN + FN))**0.5\n    return val","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3b93dbf9fed506113f89e7a779677b6e3c7d30b3"},"cell_type":"code","source":"def matthews_corr_coeff(y_true, y_pred):\n    y_pos_pred = K.round(K.clip(y_pred, 0, 1))\n    y_pos_true = K.round(K.clip(y_true, 0, 1))\n    \n    y_neg_pred = 1 - y_pos_pred\n    y_neg_true = 1 - y_pos_true\n\n    tp = K.sum(y_pos_true * y_pos_pred)\n    tn = K.sum(y_neg_true * y_neg_pred)\n    fp = K.sum(y_neg_true * y_pos_pred)\n    fn = K.sum(y_pos_true * y_neg_pred)\n    return (tp * tn - fp * fn) / (K.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn)) + K.epsilon())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"88a9a1390bce9836953811e3d8d7835b3b9b2c8a"},"cell_type":"code","source":"length_of_sequence = x_train[0].shape\ndrop_out_rate = 0.5\nrecurrent_dropout = 0.5\nSTEPS_PER_EPOCH = 50\nEPOCHS = 100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_X.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"length_of_sequence","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"835bf0f11e59dee510dd036621785c2362384c81"},"cell_type":"code","source":"# Create Model\n\nmodel = Sequential()\nmodel.add(layers.Conv1D(32, 8, \n                 padding='same',\n                 input_shape=(length_of_sequence),\n                 activation='relu'))\nmodel.add(layers.MaxPooling1D(2, padding='same'))\nmodel.add(Dropout(0.5))\n# model.add(layers.BatchNormalization())\nmodel.add(layers.Conv1D(64, 8, padding='same', activation='relu'))\nmodel.add(layers.MaxPooling1D(2, padding='same'))\nmodel.add(Dropout(0.5))\nmodel.add(layers.Conv1D(128, 8, padding='same', activation='relu'))\n# model.add(layers.BatchNormalization())\n# model.add(layers.MaxPooling1D(2, padding='same'))\n# model.add(layers.Conv1D(256, 8, padding='same', activation='relu'))\nmodel.add(Dropout(0.5))\nmodel.add(layers.BatchNormalization())\nmodel.add(layers.LSTM(64, \n#               return_sequences=True,\n               dropout = drop_out_rate,\n               recurrent_dropout = recurrent_dropout\n              ))\n               #batch_input_shape=(None, 2, length_of_sequence)))\n#model.add(layers.LSTM(128, \n#               dropout = drop_out_rate,\n#               recurrent_dropout = recurrent_dropout\n#              ))\n#model.add(layers.Dense(100,activation='relu'))\nmodel.add(layers.Dense(1,activation='sigmoid'))\n\nmodel.compile(loss='binary_crossentropy',\n              optimizer='Adam',\n              metrics=['accuracy',matthews_corr_coeff])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc954861abfe7878cc56909e8ba01ed28b99bc39"},"cell_type":"code","source":"weight_path=\"{}_weights.best.hdf5\".format('lstm_model')\nearly = EarlyStopping(monitor=\"val_loss\", \n                      mode=\"min\", \n                      patience=10) # probably needs to be more patient, but kaggle time is limited\n# lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=2, min_lr=0.01)\ncheckpoint = ModelCheckpoint(weight_path, monitor='val_loss', verbose=1, \n                             save_best_only=True, mode='min', save_weights_only = True)\n\ncallbacks_list = [checkpoint]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(train_X, train_y, batch_size=128, \n                epochs=EPOCHS,validation_data=[val_X, val_y], callbacks=callbacks_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#####自定义打分函数\n\nfrom sklearn.metrics import accuracy_score, confusion_matrix, precision_score, recall_score, f1_score, matthews_corrcoef\n\ndef print_score(label, prediction, train=True):\n    if train:\n        print(\"Train Result:\\n================================================\")\n        print(f\"Accuracy Score: {accuracy_score(label, prediction) * 100:.2f}%\")\n        print(\"_______________________________________________\")\n        print(\"Classification Report:\", end='')\n        print(f\"\\tPrecision Score: {precision_score(label, prediction) * 100:.2f}%\")\n        print(f\"\\t\\t\\tRecall Score: {recall_score(label, prediction) * 100:.2f}%\")\n        print(f\"\\t\\t\\tF1 score: {f1_score(label, prediction) * 100:.2f}%\")\n        print(f\"\\t\\t\\tMCC score: {matthews_corrcoef(label, prediction) * 100:.2f}%\")\n        print(\"_______________________________________________\")\n        print(f\"Confusion Matrix: \\n {confusion_matrix(y_train, prediction)}\\n\")\n        \n    elif train==False:\n        print(\"Test Result:\\n================================================\")        \n        print(f\"Accuracy Score: {accuracy_score(label, prediction) * 100:.2f}%\")\n        print(\"_______________________________________________\")\n        print(\"Classification Report:\", end='')\n        print(f\"\\tPrecision Score: {precision_score(label, prediction) * 100:.2f}%\")\n        print(f\"\\t\\t\\tRecall Score: {recall_score(label, prediction) * 100:.2f}%\")\n        print(f\"\\t\\t\\tF1 score: {f1_score(label, prediction) * 100:.2f}%\")\n        print(f\"\\t\\t\\tMCC score: {matthews_corrcoef(label, prediction) * 100:.2f}%\")\n        print(\"_______________________________________________\")\n        print(f\"Confusion Matrix: \\n {confusion_matrix(label, prediction)}\\n\") ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fba17e77694a35fe3fdddc631fced82357bd7d06"},"cell_type":"code","source":"model.load_weights('lstm_model_weights.best.hdf5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_test_pred = model.predict(test_X)\n\nprint_score(test_y, y_test_pred.round(), train=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e1f1407e65f0cbd5696f64e869d8c0023469c45"},"cell_type":"markdown","source":"Add predict label to test data and train again."},{"metadata":{"trusted":true,"_uuid":"ea9e346efcdeafbec75e658c63e72ef87780b501"},"cell_type":"code","source":"# # Create Model\n\n# model_2 = Sequential()\n# model_2.add(layers.Conv1D(32, 8, \n#                  padding='same',\n#                  input_shape=(length_of_sequence),\n#                  activation='relu'))\n# model_2.add(layers.MaxPooling1D(2, padding='same'))\n# model_2.add(layers.Conv1D(64, 8, padding='same', activation='relu'))\n# model_2.add(layers.MaxPooling1D(2, padding='same'))\n# model_2.add(layers.Conv1D(128, 8, padding='same', activation='relu'))\n# model_2.add(layers.MaxPooling1D(2, padding='same'))\n# model_2.add(layers.Conv1D(256, 8, padding='same', activation='relu'))\n# model_2.add(layers.LSTM(64, \n#                return_sequences=True,\n#                dropout = drop_out_rate,\n#                recurrent_dropout = recurrent_dropout\n#               ))\n#                #batch_input_shape=(None, 2, length_of_sequence)))\n# model_2.add(layers.LSTM(128, \n#                dropout = drop_out_rate,\n#                recurrent_dropout = recurrent_dropout\n#               ))\n# #model.add(layers.Dense(100,activation='relu'))\n# model_2.add(layers.Dense(1,activation='sigmoid'))\n\n# model_2.compile(loss='binary_crossentropy',\n#               optimizer='rmsprop',\n#               metrics=['accuracy',matthews_corr_coeff])\n# model_2.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6b3d4e64cd307d3c8b6ffe20dca142503e98ad45"},"cell_type":"code","source":"# weight_path=\"{}_weights.best.hdf5\".format('lstm_model_2')\n# lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=3, min_lr=0.001)\n# checkpoint = ModelCheckpoint(weight_path, monitor='val_loss', verbose=1, \n#                              save_best_only=True, mode='min', save_weights_only = True)\n\n# callbacks_list = [checkpoint, lr]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d93b3c126628be27aea56d0946acb1e66d915e37"},"cell_type":"code","source":"# history = model.fit(train_X, train_y, batch_size=128, \n#                 epochs=EPOCHS,validation_data=[val_X, val_y], callbacks=callbacks_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"101f11a06754c15957de378b998d74f937aae37a"},"cell_type":"code","source":"# model_2.load_weights('lstm_model_2_weights.best.hdf5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f202fd2788e0e6e1531c8ffaa1ed05d13b225c94"},"cell_type":"code","source":"# y_val_pred = model_2.predict(x_val)\n# y_val_pred = y_val_pred.flatten()\n# y_val_pred[y_val_pred >= 0.5] = 1\n# y_val_pred[y_val_pred < 0.5] = 0\n# print(y_val_pred.sum())\n# print(y_val.sum())\n# print(mcc(y_val,y_val_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce2150e36249cb0d15c07257e80de361204fb257"},"cell_type":"code","source":"# y_test_1 = model.predict(x_test)\n# y_test_2 = model_2.predict(x_test)\n\n# y_test = (y_test_1.flatten()) * 0.5 + (y_test_2.flatten()) * 0.5\n# #y_test = y_test_2.flatten()\n# y_test[y_test >= 0.5] = 1\n# y_test[y_test < 0.5] = 0\n# print(y_test.sum())","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}