{"cells":[{"metadata":{"_uuid":"b193840ad69a12482ad3aea4e6a74c02ea13f9a8"},"cell_type":"markdown","source":"# Import Libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from numpy.random import seed\nseed(1)\nfrom tensorflow import set_random_seed\nset_random_seed(2)\nimport pandas as pd\nimport pyarrow.parquet as pq\nfrom tqdm import trange,tqdm\nimport os\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport tensorflow as tf\nimport keras\nimport keras.backend as K\nfrom keras.layers import LSTM,Dropout,Dense,TimeDistributed,Conv1D,MaxPooling1D,Flatten,GlobalAveragePooling1D,AveragePooling1D,GlobalMaxPooling1D,BatchNormalization,Activation,Bidirectional\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\nfrom keras.models import Sequential\nimport tensorflow as tf\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nfrom scipy import signal\nimport gc","execution_count":1,"outputs":[{"output_type":"stream","text":"Using TensorFlow backend.\n","name":"stderr"}]},{"metadata":{"_uuid":"531d88f76247b06226679a2cea9ff772aa91bf59"},"cell_type":"markdown","source":"[PHD Thesis](http://dspace.vsb.cz/bitstream/handle/10084/133114/VAN431_FEI_P1807_1801V001_2018.pdf) \n[Preprint](https://www.dropbox.com/s/2ltuvpw1b1ms2uu/A%20Complex%20Classification%20Approach%20of%20Partial%20Discharges%20from%20Covered%20Conductors%20in%20Real%20Environment%20%28preprint%29.pdf?dl=0) \n[Advance Pandas Tricks and Techniques](https://www.kaggle.com/ashishpatel26/advance-pandas-tricks-and-techniques) \n[CNN + LSTM for Signal Classification LB 0.513](https://www.kaggle.com/afajohn/cnn-lstm-for-signal-classification-lb-0-513)"},{"metadata":{"trusted":true,"_uuid":"d2f833929daba6f86efc1f1e44c60f6d7c5fb2be"},"cell_type":"code","source":"compression_bucket_size = 300","execution_count":2,"outputs":[]},{"metadata":{"_uuid":"f7f902ba485e849f4f91b8eac2663190e42ea727"},"cell_type":"markdown","source":"# Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load Data\ndef load_data(parquet_data, csv_metadata):\n pq_data = np.array(pq.read_pandas(('../input/'+parquet_data)).to_pandas().T)\n metadata = pd.read_csv('../input/'+csv_metadata) \n target = metadata['target'][:len(pq_data)].values\n return pq_data,target\n\nfrom numpy.fft import *\ndef filter_signal(signal, threshold=1e8):\n fourier = rfft(signal)\n frequencies = rfftfreq(signal.size, d=20e-3/signal.size)\n fourier[frequencies > threshold] = 0\n return irfft(fourier)\n\n# Subtract de-noised data from the raw signal data to process, or normalize(?), data. \ndef signal_processing(data):\n return abs(np.round((data-filter_signal(data,threshold=1e3)),2))\n\n\ndef train_validate_split(data,data_target,validate_size):\n metadata = pd.read_csv('../input/metadata_train.csv') \n signal_id_1 = list(metadata[metadata['target']==1]['signal_id'])\n signal_id_0 = list(metadata[metadata['target']==0]['signal_id'])\n train_1 = signal_id_1[0:int(len(signal_id_1)*(1-validate_size))]\n validate_1 = signal_id_1[int(len(signal_id_1)*(1-validate_size)):]\n train_0 = signal_id_0[0:int(len(signal_id_0)*(1-validate_size))]\n validate_0 = signal_id_0[int(len(signal_id_0)*(1-validate_size)):]\n \n data_train = data[sorted(np.concatenate((train_0,train_1)))]\n data_train_target = data_target[sorted(np.concatenate((train_0,train_1)))]\n data_validate = data[sorted(np.concatenate((validate_0,validate_1)))]\n data_validate_target = data_target[sorted(np.concatenate((validate_0,validate_1)))] \n \n return data_train, data_train_target, data_validate, data_validate_target\n\n# Reduce sample size from 800000 to 800000/bucket_size while not losing information by extracting features : std, mean, max, min\ndef compress_data_and_extract_features(data,bucket_size):\n data_bucket_std, data_bucket_mean, data_bucket_percentile_0, data_bucket_percentile_1, data_bucket_percentile_25, data_bucket_percentile_50, data_bucket_percentile_75, data_bucket_percentile_99, data_bucket_percentile_100 = [],[],[],[],[],[],[],[],[]\n \n \n for i in trange(data.shape[0]):\n holder_std, holder_mean, holder_percentile,holder_0,holder_1,holder_25,holder_50,holder_75,holder_99,holder_100 = [],[],[],[],[],[],[],[],[],[]\n #percentile_threshhold = np.percentile(abs(data[i]),99.97)\n for j in range(0,data.shape[1],bucket_size):\n holder_std.append(abs(data[i][j:(j+bucket_size)]).std())\n holder_mean.append(abs(data[i][j:(j+bucket_size)]).mean())\n holder_percentile=np.percentile(abs(data[i][j:(j+bucket_size)]),[0, 1, 25, 50, 75, 99, 100])\n holder_0.append(holder_percentile[0])\n holder_1.append(holder_percentile[1])\n holder_25.append(holder_percentile[2])\n holder_50.append(holder_percentile[3])\n holder_75.append(holder_percentile[4])\n holder_99.append(holder_percentile[5])\n holder_100.append(holder_percentile[6])\n #holder_peaks.append(sum(abs(data[i][j:(j+bucket_size)])>percentile_threshhold)) \n \n data_bucket_std.append(holder_std)\n data_bucket_mean.append(holder_mean)\n data_bucket_percentile_0.append(holder_0)\n data_bucket_percentile_1.append(holder_1)\n data_bucket_percentile_25.append(holder_25)\n data_bucket_percentile_50.append(holder_50)\n data_bucket_percentile_75.append(holder_75)\n data_bucket_percentile_99.append(holder_99)\n data_bucket_percentile_100.append(holder_100)\n #data_bucket_peaks.append(holder_peaks) \n return np.asarray(data_bucket_std), np.asarray(data_bucket_mean), np.asarray(data_bucket_percentile_0), np.asarray(data_bucket_percentile_1), np.asarray(data_bucket_percentile_25), np.asarray(data_bucket_percentile_50), np.asarray(data_bucket_percentile_75), np.asarray(data_bucket_percentile_99), np.asarray(data_bucket_percentile_100)\n\n# Reshape Input Data of multiple features into single input for LSTM Input\ndef LSTM_reshape_dstack(combined_data_list): \n for i in range(len(combined_data_list)):\n combined_data_list[i]=combined_data_list[i].reshape(combined_data_list[i].shape[0],combined_data_list[i].shape[1],1) \n return np.dstack(combined_data_list)","execution_count":3,"outputs":[]},{"metadata":{"_uuid":"f967aa1d39e321dffa43ddf9f822dff59bdcbaa4"},"cell_type":"markdown","source":"# Train Validate Split"},{"metadata":{"trusted":true},"cell_type":"code","source":"# singal = 6\n# # 3, 4, 5, 201, 202, 228, 229, 230, 270, 271\n\n# plt.figure(figsize=(10,3))\n# t = np.arange(0,len(train[0]))\n# plt.plot(t,train[singal])\n# plt.show()","execution_count":4,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a6b8ac1aa858e18ccb390d37af0f0f48f762d147"},"cell_type":"code","source":"train,target_train = load_data('train.parquet','metadata_train.csv')\nfor i in trange(len(train)): \n train[i]=signal_processing(train[i])\n# train_min = min(train[i])\n# train_max = max(train[i])\n# train[i]= (train[i]-train_min)/(train_max-train_min)*2-1\ndata_train, data_train_target, data_validate, data_validate_target = train_validate_split(train,target_train,0.3)\ndel train,target_train\ngc.collect()\n","execution_count":5,"outputs":[{"output_type":"stream","text":"100%|██████████| 8712/8712 [09:27<00:00, 14.79it/s]\n","name":"stderr"},{"output_type":"execute_result","execution_count":5,"data":{"text/plain":"26194"},"metadata":{}}]},{"metadata":{"_uuid":"7f02b6a7be035ecce61a7b9dd36607a7f90a588c"},"cell_type":"markdown","source":"# Execute Data Preparation for LSTM Input"},{"metadata":{"_uuid":"e2a8a4cbcf1933a5038ce0f2262903d6fcd8c0bf"},"cell_type":"markdown","source":"Explanation on prepararing data for LSTM training. \nWe have four features, or dimensions, of 1D signals. In order to stack them, you need to use [numpy.dstack](https://docs.scipy.org/doc/numpy-1.15.0/reference/generated/numpy.dstack.html). Per dstack's document, 1D signals first need to be reshaped to (1,timesteps,1) to (timesteps,). \nBut, LSTM takes data in 4D, so we actually need (num_samples,1,timesteps,dimensions). But if we have 4D form first, dstack wouldn't work. Therefore, we first stack the four features using numpy.dstack, then we reshape the data once more to (num_samples,1,timesteps,dimensions) for LSTM. \n* num_samples = number of signals \n* 1 = 1 because it's 1D signal. If it was a 2D case, it wouldn't been width or height of the image, and not 1\n* timesteps = 800000 or whatever it's reduced to per feature extraction.\n* dimensions = will be result of numpy.dstack depending on how many features you have"},{"metadata":{"trusted":true,"_uuid":"72b053074688c03961f5db595334cc8005a9297c","scrolled":true},"cell_type":"code","source":"#train_std,train_mean,train_max,train_25,train_50,train_75,train_peaks = compress_data_and_extract_features(data_train,compression_bucket_size)\ntrain_std,train_mean,train_0,train_1,train_25,train_50,train_75,train_99,train_100 = compress_data_and_extract_features(data_train,compression_bucket_size)\n#train_LSTM = LSTM_reshape_dstack([train_std,train_mean,train_max,train_25,train_50,train_75,train_peaks])\ntrain_LSTM = LSTM_reshape_dstack([train_std,train_mean,train_0,train_1,train_25,train_50,train_75,train_99,train_100])\ntrain_LSTM_backup = train_LSTM.copy()\ntrain_LSTM = train_LSTM.reshape(train_LSTM.shape[0],1,train_LSTM.shape[1],train_LSTM.shape[2])\n\ndel train_std,train_mean,train_0,train_1,train_25,train_50,train_75,train_99,train_100#,train_peaks\ngc.collect()\n\n#validate_std,validate_mean,validate_max,validate_25,validate_50,validate_75,validate_peaks = compress_data_and_extract_features(data_validate,compression_bucket_size)\nvalidate_std,validate_mean,validate_0,validate_1,validate_25,validate_50,validate_75,validate_99,validate_100 = compress_data_and_extract_features(data_validate,compression_bucket_size)\n#validate_LSTM = LSTM_reshape_dstack([validate_std,validate_mean,validate_0,validate_1,validate_25,validate_50,validate_75,validate_99,validate_100])\nvalidate_LSTM = LSTM_reshape_dstack([validate_std,validate_mean,validate_0,validate_1,validate_25,validate_50,validate_75,validate_99,validate_100])\nvalidate_LSTM_backup = validate_LSTM.copy()\nvalidate_LSTM = validate_LSTM.reshape(validate_LSTM.shape[0],1,validate_LSTM.shape[1],validate_LSTM.shape[2])\n\n#del validate_std,validate_mean,validate_max,validate_25,validate_50,validate_75,validate_peaks\ndel validate_std,validate_mean,validate_0,validate_1,validate_25,validate_50,validate_75,validate_99,validate_100\ngc.collect()\n\n# For easier readability\nnum_signals = train_LSTM.shape[0]\nnum_timesteps = train_LSTM.shape[2]\nnum_features = train_LSTM.shape[3]","execution_count":6,"outputs":[{"output_type":"stream","text":"100%|██████████| 6097/6097 [40:55<00:00, 2.43it/s] \n100%|██████████| 2615/2615 [17:25<00:00, 2.51it/s]\n","name":"stderr"}]},{"metadata":{"_uuid":"a696484df294b92bb680f4736279e416cb5e9010"},"cell_type":"markdown","source":"# Prepare Data for LSTM Input\n[Prepare input data for LSTM layer](https://machinelearningmastery.com/reshape-input-data-long-short-term-memory-networks-keras/) \n[How to Prepare Univariate Time Series Data for Long Short-Term Memory Networks](https://machinelearningmastery.com/prepare-univariate-time-series-data-long-short-term-memory-networks/)"},{"metadata":{"_uuid":"a3b7596751f0eaeb514a43e9eb1d3ab475b5f21e"},"cell_type":"markdown","source":"# Define Model\n[TimeDistributed Documentation](https://keras.io/layers/wrappers/) \n[Convolutional Layers Documentation](https://keras.io/layers/convolutional/) \n[Pooling Layers](https://keras.io/layers/pooling/) \n[How to use TimeDistributed wrapper for LSTM](https://machinelearningmastery.com/timedistributed-layer-for-long-short-term-memory-networks-in-python/) \n[CNN LSTM Tutorial](https://machinelearningmastery.com/cnn-long-short-term-memory-networks/) \n[Number of LSTM Units](https://datascience.stackexchange.com/questions/16350/how-many-lstm-cells-should-i-use/18049)"},{"metadata":{"trusted":true},"cell_type":"code","source":"def keras_auc(y_true, y_pred):\n auc = tf.metrics.auc(y_true, y_pred)[1]\n K.get_session().run(tf.local_variables_initializer())\n return auc\ndef matthews_correlation(y_true, y_pred):\n '''Calculates the Matthews correlation coefficient measure for quality\n of binary classification problems.\n '''\n y_pred_pos = K.round(K.clip(y_pred, 0, 1))\n y_pred_neg = 1 - y_pred_pos\n\n y_pos = K.round(K.clip(y_true, 0, 1))\n y_neg = 1 - y_pos\n\n tp = K.sum(y_pos * y_pred_pos)\n tn = K.sum(y_neg * y_pred_neg)\n\n \n fp = K.sum(y_neg * y_pred_pos)\n fn = K.sum(y_pos * y_pred_neg)\n\n numerator = (tp * tn - fp * fn)\n denominator = K.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn))\n\n return numerator/(denominator+K.epsilon())","execution_count":10,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cb2aaa783b450fe550e55894ed1326437fc42d36"},"cell_type":"code","source":"#num_signals, num_timesteps, num_features = train_LSTM.shape[2]\n\nmodel = Sequential()\n# num_timesteps = 800000\n# num_features = 6\n\n# Define CNN Model\nmodel.add(TimeDistributed(Conv1D(filters=64, kernel_size=6), input_shape=(None,num_timesteps,num_features)))\nmodel.add(TimeDistributed(Activation('relu')))\nmodel.add(TimeDistributed(Conv1D(filters=64, kernel_size=6)))\nmodel.add(TimeDistributed(Activation('relu')))\nmodel.add(TimeDistributed(GlobalMaxPooling1D()))\n\n\nmodel.add(TimeDistributed(Flatten()))\n# Define LSTM Model\nmodel.add(LSTM(128))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(16, activation='relu'))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(1, activation='sigmoid'))\nmodel.summary()\n\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=[matthews_correlation])\ncallbacks = [EarlyStopping(monitor='val_matthews_correlation', patience=20),\n ModelCheckpoint(filepath='best_model.h5', monitor='val_matthews_correlation', save_best_only=True,mode='max')]\n#callbacks = [ModelCheckpoint(filepath='best_model.h5', monitor='val_matthews_correlation', save_best_only=True,mode='max')]","execution_count":26,"outputs":[{"output_type":"stream","text":"_________________________________________________________________\nLayer (type) Output Shape Param # \n=================================================================\ntime_distributed_31 (TimeDis (None, None, 2662, 64) 3520 \n_________________________________________________________________\ntime_distributed_32 (TimeDis (None, None, 2662, 64) 0 \n_________________________________________________________________\ntime_distributed_33 (TimeDis (None, None, 2657, 64) 24640 \n_________________________________________________________________\ntime_distributed_34 (TimeDis (None, None, 2657, 64) 0 \n_________________________________________________________________\ntime_distributed_35 (TimeDis (None, None, 64) 0 \n_________________________________________________________________\ntime_distributed_36 (TimeDis (None, None, 64) 0 \n_________________________________________________________________\nlstm_7 (LSTM) (None, 128) 98816 \n_________________________________________________________________\ndropout_13 (Dropout) (None, 128) 0 \n_________________________________________________________________\ndense_13 (Dense) (None, 16) 2064 \n_________________________________________________________________\ndropout_14 (Dropout) (None, 16) 0 \n_________________________________________________________________\ndense_14 (Dense) (None, 1) 17 \n=================================================================\nTotal params: 129,057\nTrainable params: 129,057\nNon-trainable params: 0\n_________________________________________________________________\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nmodel.fit(train_LSTM, data_train_target, validation_data=(validate_LSTM[0:1308],data_validate_target[0:1308]),epochs=200, batch_size=128, verbose=1,callbacks=callbacks)","execution_count":27,"outputs":[{"output_type":"stream","text":"Train on 6097 samples, validate on 1308 samples\nEpoch 1/200\n6097/6097 [==============================] - 91s 15ms/step - loss: 0.2951 - matthews_correlation: 0.0054 - val_loss: 0.2260 - val_matthews_correlation: 0.0000e+00\nEpoch 2/200\n6097/6097 [==============================] - 88s 14ms/step - loss: 0.2333 - matthews_correlation: 0.0162 - val_loss: 0.1985 - val_matthews_correlation: 0.0000e+00\nEpoch 3/200\n6097/6097 [==============================] - 88s 14ms/step - loss: 0.2023 - matthews_correlation: 0.0067 - val_loss: 0.1714 - val_matthews_correlation: 0.0000e+00\nEpoch 4/200\n6097/6097 [==============================] - 88s 15ms/step - loss: 0.1819 - matthews_correlation: -0.0020 - val_loss: 0.1573 - val_matthews_correlation: 0.0000e+00\nEpoch 5/200\n6097/6097 [==============================] - 89s 15ms/step - loss: 0.1547 - matthews_correlation: 0.1187 - val_loss: 0.1361 - val_matthews_correlation: 0.0000e+00\nEpoch 6/200\n6097/6097 [==============================] - 89s 15ms/step - loss: 0.1539 - matthews_correlation: 0.1513 - val_loss: 0.1406 - val_matthews_correlation: 0.0000e+00\nEpoch 7/200\n6097/6097 [==============================] - 90s 15ms/step - loss: 0.1410 - matthews_correlation: 0.2368 - val_loss: 0.1274 - val_matthews_correlation: 0.1026\nEpoch 8/200\n6097/6097 [==============================] - 90s 15ms/step - loss: 0.1387 - matthews_correlation: 0.2991 - val_loss: 0.1344 - val_matthews_correlation: 0.1080\nEpoch 9/200\n6097/6097 [==============================] - 90s 15ms/step - loss: 0.1314 - matthews_correlation: 0.3596 - val_loss: 0.1247 - val_matthews_correlation: 0.3596\nEpoch 10/200\n6097/6097 [==============================] - 90s 15ms/step - loss: 0.1321 - matthews_correlation: 0.3906 - val_loss: 0.1368 - val_matthews_correlation: 0.3335\nEpoch 11/200\n6097/6097 [==============================] - 90s 15ms/step - loss: 0.1277 - matthews_correlation: 0.4608 - val_loss: 0.1229 - val_matthews_correlation: 0.3635\nEpoch 12/200\n6097/6097 [==============================] - 90s 15ms/step - loss: 0.1248 - matthews_correlation: 0.4813 - val_loss: 0.1170 - val_matthews_correlation: 0.5148\nEpoch 13/200\n6097/6097 [==============================] - 89s 15ms/step - loss: 0.1232 - matthews_correlation: 0.4972 - val_loss: 0.1177 - val_matthews_correlation: 0.4733\nEpoch 14/200\n6097/6097 [==============================] - 90s 15ms/step - loss: 0.1203 - matthews_correlation: 0.4857 - val_loss: 0.1221 - val_matthews_correlation: 0.5874\nEpoch 15/200\n6097/6097 [==============================] - 90s 15ms/step - loss: 0.1197 - matthews_correlation: 0.5122 - val_loss: 0.1272 - val_matthews_correlation: 0.4326\nEpoch 16/200\n6097/6097 [==============================] - 90s 15ms/step - loss: 0.1168 - matthews_correlation: 0.5590 - val_loss: 0.1204 - val_matthews_correlation: 0.4600\nEpoch 17/200\n6097/6097 [==============================] - 89s 15ms/step - loss: 0.1264 - matthews_correlation: 0.5216 - val_loss: 0.1197 - val_matthews_correlation: 0.5030\nEpoch 18/200\n6097/6097 [==============================] - 90s 15ms/step - loss: 0.1164 - matthews_correlation: 0.5428 - val_loss: 0.1143 - val_matthews_correlation: 0.6619\nEpoch 19/200\n6097/6097 [==============================] - 89s 15ms/step - loss: 0.1082 - matthews_correlation: 0.5754 - val_loss: 0.1156 - val_matthews_correlation: 0.6799\nEpoch 20/200\n6097/6097 [==============================] - 89s 15ms/step - loss: 0.1065 - matthews_correlation: 0.5913 - val_loss: 0.1125 - val_matthews_correlation: 0.5676\nEpoch 21/200\n6097/6097 [==============================] - 89s 15ms/step - loss: 0.1054 - matthews_correlation: 0.6066 - val_loss: 0.1177 - val_matthews_correlation: 0.5090\n","name":"stdout"},{"output_type":"execute_result","execution_count":27,"data":{"text/plain":""},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nmodel.fit(train_LSTM, data_train_target, validation_data=(validate_LSTM[1308:],data_validate_target[1308:]),epochs=200, batch_size=128, verbose=1,callbacks=callbacks)","execution_count":28,"outputs":[{"output_type":"stream","text":"Train on 6097 samples, validate on 1307 samples\nEpoch 1/200\n6097/6097 [==============================] - 88s 14ms/step - loss: 0.1006 - matthews_correlation: 0.5912 - val_loss: 0.0924 - val_matthews_correlation: 0.5167\nEpoch 2/200\n6097/6097 [==============================] - 89s 15ms/step - loss: 0.0989 - matthews_correlation: 0.6337 - val_loss: 0.0824 - val_matthews_correlation: 0.6107\nEpoch 3/200\n6097/6097 [==============================] - 89s 15ms/step - loss: 0.1006 - matthews_correlation: 0.6249 - val_loss: 0.0917 - val_matthews_correlation: 0.5462\nEpoch 4/200\n6097/6097 [==============================] - 89s 15ms/step - loss: 0.0959 - matthews_correlation: 0.6840 - val_loss: 0.0881 - val_matthews_correlation: 0.6426\nEpoch 5/200\n6097/6097 [==============================] - 89s 15ms/step - loss: 0.0965 - matthews_correlation: 0.6631 - val_loss: 0.0854 - val_matthews_correlation: 0.5948\nEpoch 6/200\n6097/6097 [==============================] - 89s 15ms/step - loss: 0.0943 - matthews_correlation: 0.6660 - val_loss: 0.0866 - val_matthews_correlation: 0.6473\nEpoch 7/200\n6097/6097 [==============================] - 89s 15ms/step - loss: 0.0926 - matthews_correlation: 0.6537 - val_loss: 0.0987 - val_matthews_correlation: 0.4169\nEpoch 8/200\n6097/6097 [==============================] - 90s 15ms/step - loss: 0.0884 - matthews_correlation: 0.6938 - val_loss: 0.0961 - val_matthews_correlation: 0.5114\nEpoch 9/200\n6097/6097 [==============================] - 89s 15ms/step - loss: 0.0882 - matthews_correlation: 0.6855 - val_loss: 0.0892 - val_matthews_correlation: 0.5948\nEpoch 10/200\n6097/6097 [==============================] - 90s 15ms/step - loss: 0.0843 - matthews_correlation: 0.6786 - val_loss: 0.0820 - val_matthews_correlation: 0.6863\nEpoch 11/200\n6097/6097 [==============================] - 90s 15ms/step - loss: 0.0818 - matthews_correlation: 0.6771 - val_loss: 0.0837 - val_matthews_correlation: 0.6650\nEpoch 12/200\n6097/6097 [==============================] - 89s 15ms/step - loss: 0.0832 - matthews_correlation: 0.7205 - val_loss: 0.0894 - val_matthews_correlation: 0.6591\nEpoch 13/200\n6097/6097 [==============================] - 89s 15ms/step - loss: 0.0823 - matthews_correlation: 0.7079 - val_loss: 0.0916 - val_matthews_correlation: 0.5464\nEpoch 14/200\n6097/6097 [==============================] - 89s 15ms/step - loss: 0.0740 - matthews_correlation: 0.7165 - val_loss: 0.0940 - val_matthews_correlation: 0.5430\nEpoch 15/200\n6097/6097 [==============================] - 89s 15ms/step - loss: 0.0748 - matthews_correlation: 0.7105 - val_loss: 0.0961 - val_matthews_correlation: 0.5392\nEpoch 16/200\n6097/6097 [==============================] - 89s 15ms/step - loss: 0.0770 - matthews_correlation: 0.7351 - val_loss: 0.0897 - val_matthews_correlation: 0.5156\nEpoch 17/200\n6097/6097 [==============================] - 89s 15ms/step - loss: 0.0719 - matthews_correlation: 0.7307 - val_loss: 0.0906 - val_matthews_correlation: 0.5759\nEpoch 18/200\n6097/6097 [==============================] - 89s 15ms/step - loss: 0.0733 - matthews_correlation: 0.7195 - val_loss: 0.0897 - val_matthews_correlation: 0.6447\nEpoch 19/200\n6097/6097 [==============================] - 89s 15ms/step - loss: 0.0710 - matthews_correlation: 0.7482 - val_loss: 0.1008 - val_matthews_correlation: 0.5815\nEpoch 20/200\n6097/6097 [==============================] - 89s 15ms/step - loss: 0.0733 - matthews_correlation: 0.7527 - val_loss: 0.1053 - val_matthews_correlation: 0.5328\nEpoch 21/200\n6097/6097 [==============================] - 90s 15ms/step - loss: 0.0683 - matthews_correlation: 0.7541 - val_loss: 0.1175 - val_matthews_correlation: 0.5520\nEpoch 22/200\n6097/6097 [==============================] - 89s 15ms/step - loss: 0.0717 - matthews_correlation: 0.7633 - val_loss: 0.0899 - val_matthews_correlation: 0.6428\nEpoch 23/200\n6097/6097 [==============================] - 90s 15ms/step - loss: 0.0619 - matthews_correlation: 0.8031 - val_loss: 0.0952 - val_matthews_correlation: 0.6255\nEpoch 24/200\n6097/6097 [==============================] - 89s 15ms/step - loss: 0.0627 - matthews_correlation: 0.7813 - val_loss: 0.0919 - val_matthews_correlation: 0.6537\nEpoch 25/200\n6097/6097 [==============================] - 89s 15ms/step - loss: 0.0578 - matthews_correlation: 0.7847 - val_loss: 0.1150 - val_matthews_correlation: 0.5490\nEpoch 26/200\n6097/6097 [==============================] - 89s 15ms/step - loss: 0.0593 - matthews_correlation: 0.8072 - val_loss: 0.1508 - val_matthews_correlation: 0.4649\nEpoch 27/200\n6097/6097 [==============================] - 89s 15ms/step - loss: 0.0684 - matthews_correlation: 0.7526 - val_loss: 0.0943 - val_matthews_correlation: 0.6386\n","name":"stdout"},{"output_type":"execute_result","execution_count":28,"data":{"text/plain":""},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"callbacks = [ModelCheckpoint(filepath='best_model.h5', monitor='matthews_correlation', save_best_only=True,mode='max')]\n\nmodel.fit(np.concatenate((train_LSTM,validate_LSTM)), np.concatenate((data_train_target,data_validate_target)),epochs=10, batch_size=128, verbose=1,callbacks=callbacks)","execution_count":32,"outputs":[{"output_type":"stream","text":"Epoch 1/10\n8712/8712 [==============================] - 122s 14ms/step - loss: 0.0835 - matthews_correlation: 0.7427\nEpoch 2/10\n8712/8712 [==============================] - 122s 14ms/step - loss: 0.0778 - matthews_correlation: 0.7387\nEpoch 3/10\n8712/8712 [==============================] - 122s 14ms/step - loss: 0.0780 - matthews_correlation: 0.7368\nEpoch 4/10\n8712/8712 [==============================] - 122s 14ms/step - loss: 0.0714 - matthews_correlation: 0.7388\nEpoch 5/10\n8712/8712 [==============================] - 122s 14ms/step - loss: 0.0713 - matthews_correlation: 0.7559\nEpoch 6/10\n8712/8712 [==============================] - 121s 14ms/step - loss: 0.0708 - matthews_correlation: 0.7783\nEpoch 7/10\n8712/8712 [==============================] - 122s 14ms/step - loss: 0.0677 - matthews_correlation: 0.7675\nEpoch 8/10\n8712/8712 [==============================] - 121s 14ms/step - loss: 0.0727 - matthews_correlation: 0.7513\nEpoch 9/10\n 896/8712 [==>...........................] - ETA: 1:48 - loss: 0.0520 - matthews_correlation: 0.8354","name":"stdout"},{"output_type":"error","ename":"KeyboardInterrupt","evalue":"","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)","\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0mcallbacks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mModelCheckpoint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'best_model.h5'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmonitor\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'matthews_correlation'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msave_best_only\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'max'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_LSTM\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mvalidate_LSTM\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_train_target\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdata_validate_target\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m128\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m 1037\u001b[0m \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1038\u001b[0m \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1039\u001b[0;31m validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m 1040\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1041\u001b[0m def evaluate(self, x=None, y=None,\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[0;34m(model, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m 197\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 199\u001b[0;31m \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 200\u001b[0m \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 201\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ml\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_labels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m 2713\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_legacy_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2714\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2715\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2716\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2717\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m 2673\u001b[0m \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_metadata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2674\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2675\u001b[0;31m \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2676\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mfetched\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2677\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1437\u001b[0m ret = tf_session.TF_SessionRunCallable(\n\u001b[1;32m 1438\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1439\u001b[0;31m run_metadata_ptr)\n\u001b[0m\u001b[1;32m 1440\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1441\u001b[0m \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mKeyboardInterrupt\u001b[0m: "]}]},{"metadata":{"trusted":true,"_uuid":"7efd5373b301a20be186d1c21907cbff818e97d7","scrolled":true},"cell_type":"code","source":"# #num_signals, num_timesteps, num_features = train_LSTM.shape[2]\n\n# model = Sequential()\n# # num_timesteps = 800000\n# # num_features = 6\n\n# # Define CNN Model\n# model.add(TimeDistributed(Conv1D(filters=32, kernel_size=2), input_shape=(None,num_timesteps,num_features)))\n# model.add(TimeDistributed(Activation('relu')))\n# model.add(TimeDistributed(MaxPooling1D(pool_size=6)))\n\n\n# model.add(TimeDistributed(Conv1D(filters=16, kernel_size=2)))\n# model.add(TimeDistributed(Activation('relu')))\n# model.add(TimeDistributed(MaxPooling1D(pool_size=6)))\n\n# model.add(TimeDistributed(Conv1D(filters=8, kernel_size=2)))\n# model.add(TimeDistributed(Activation('relu')))\n# model.add(TimeDistributed(MaxPooling1D(pool_size=6)))\n\n# model.add(TimeDistributed(Conv1D(filters=4, kernel_size=2)))\n# model.add(TimeDistributed(Activation('relu')))\n# model.add(TimeDistributed(MaxPooling1D(pool_size=6)))\n\n\n\n# # model.add(TimeDistributed(GlobalMaxPooling1D()))\n\n# # model.add(TimeDistributed(Conv1D(filters=16, kernel_size=4)))\n# # model.add(TimeDistributed(Activation('relu')))\n# # #model.add(TimeDistributed(BatchNormalization()))\n# # model.add(TimeDistributed(MaxPooling1D(pool_size=2)))\n# # model.add(Dropout(0.2))\n\n# # model.add(TimeDistributed(Conv1D(filters=16, kernel_size=2)))\n# # model.add(TimeDistributed(Activation('relu')))\n# # #model.add(TimeDistributed(BatchNormalization()))\n# # model.add(TimeDistributed(MaxPooling1D(pool_size=2)))\n# # model.add(Dropout(0.2))\n\n\n\n# model.add(TimeDistributed(Flatten()))\n# # Define LSTM Model\n# model.add(LSTM(128))\n# model.add(Dropout(0.5))\n# model.add(Dense(32, activation='relu'))\n# model.add(Dropout(0.5))\n# model.add(Dense(1, activation='sigmoid'))\n# model.summary()\n\n# model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[matthews_correlation])\n\n# #callbacks = [EarlyStopping(monitor='val_matthews_correlation', patience=50),\n# # ModelCheckpoint(filepath='best_model.h5', monitor='val_matthews_correlation', save_best_only=True,mode='max')]\n# callbacks = [ModelCheckpoint(filepath='best_model.h5', monitor='val_matthews_correlation', save_best_only=True,mode='max')]\n# model.fit(train_LSTM, data_train_target, validation_data=(validate_LSTM,data_validate_target),epochs=300, batch_size=64, verbose=1,callbacks=callbacks)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab1d3a34595fa5ce0b3976dc5296399aeea30b1f"},"cell_type":"code","source":"del train_LSTM, data_train_target, validate_LSTM, data_validate_target\ngc.collect()\n\n# model.save_weights('model1.hdf5')\nmodel.load_weights('best_model.h5')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7f816f9da53a30ca6b24cef05456cde9e99e848c"},"cell_type":"markdown","source":"# Predict using Test Data"},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"99d3a5a3cd47e9f3b386e20aa7ecdcddd3c0a9f2"},"cell_type":"code","source":"def divide_test_data(start,end):\n test_metadata = pd.read_csv('../input/metadata_test.csv') \n test_index_list = test_metadata['signal_id'][start:end].values\n test_index_lists = []\n for i in range(len(test_index_list)):\n test_index_lists.append(str(test_index_list[i]))\n return test_index_lists\n test_index_lists_master.append(test_index_lists)\n \n# Process test data in 6 chunks due to memory issues\ntest_index_lists_master=[]\ntest_index_lists_master.append(divide_test_data(0,3390))\ntest_index_lists_master.append(divide_test_data(3390,6780))\ntest_index_lists_master.append(divide_test_data(6780,10170))\ntest_index_lists_master.append(divide_test_data(10170,13560))\ntest_index_lists_master.append(divide_test_data(13560,16950))\ntest_index_lists_master.append(divide_test_data(16950,20337))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18f8c54a78eb9f64c536b37bfea84ba8ba69ae1b"},"cell_type":"code","source":"y_pred = []\nfor i in range(len(test_index_lists_master)):\n test = np.array(pq.read_pandas(('../input/test.parquet'),columns=test_index_lists_master[i]).to_pandas().T) \n for i in range(len(test)):\n test[i]=signal_processing(test[i]) \n# test_min = min(test[i])\n# test_max = max(test[i])\n# test[i]= (test[i]-test_min)/(test_max-test_min)*2-1 \n \n \n #test_std,test_mean,test_max,test_25,test_50,test_75,test_peaks = compress_data_and_extract_features(test,compression_bucket_size)\n test_std,test_mean,test_0,test_1,test_25,test_50,test_75,test_99,test_100 = compress_data_and_extract_features(test,compression_bucket_size)\n del test\n #test_LSTM = LSTM_reshape_dstack([test_std,test_mean,test_max,test_25,test_50,test_75,test_peaks])\n test_LSTM = LSTM_reshape_dstack([test_std,test_mean,test_0,test_1,test_25,test_50,test_75,test_99,test_100])\n #del test_std,test_mean,test_max,test_25,test_50,test_75,test_peaks\n del test_std,test_mean,test_0,test_1,test_25,test_50,test_75,test_99,test_100\n test_LSTM = test_LSTM.reshape(test_LSTM.shape[0],1,test_LSTM.shape[1],test_LSTM.shape[2])\n y_pred.append(model.predict(test_LSTM))\n del test_LSTM\n gc.collect() ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"7e15c0a3f2e4dfb11604c52618ae96f6b9b2f130"},"cell_type":"code","source":"y_pred_final=np.concatenate((y_pred[0],y_pred[1],y_pred[2],y_pred[3],y_pred[4],y_pred[5]))\n# Do this temporarily since target and number of signals didn't match.\n#y_pred_final=np.append(y_pred_final,0)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"4a9659d06e9ed4621ec51d012cab361810d852bb"},"cell_type":"code","source":"test_metadata = pd.read_csv('../input/metadata_test.csv') \ntest_metadata['target']=y_pred_final\ntest_metadata=test_metadata.drop(columns=['phase','id_measurement'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"3859b5d9ec842ca1687713dcc96fa32173886204"},"cell_type":"code","source":"threshhold=0.5\ntest_metadata['target'][test_metadata['target']>=threshhold]=1\ntest_metadata['target'][test_metadata['target']