{"cells":[{"metadata":{},"cell_type":"markdown","source":"## 1. Load Data"},{"metadata":{"_uuid":"e217109aeec443c5cb64fdc716ef433f306c436c","trusted":true},"cell_type":"code","source":"import keras\nimport keras.backend as K\nfrom keras.layers import LSTM,Dropout,Dense,TimeDistributed,Conv1D,MaxPooling1D,Flatten\nfrom keras.models import Sequential\nimport tensorflow as tf\nimport gc\nfrom numba import jit\nfrom IPython.display import display, clear_output\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nimport sys\nsns.set_style(\"whitegrid\")\nprint(tf.__version__)\nimport pyarrow.parquet as pq\nimport pandas as pd\nimport numpy as np\nimport time\nfrom numpy.fft import *\nfrom scipy import fftpack","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"809b6250e1643fa4a4dae5230dc07772c9b56084"},"cell_type":"markdown","source":"## 2. Process and Minimize Data"},{"metadata":{"_uuid":"36f9662e947b286fa6505a4c8183daff1b8c1570","trusted":true},"cell_type":"code","source":"def low_pass(s, threshold=3e7):\n    fourier = rfft(s)\n    frequencies = rfftfreq(s.size, d=2e-2/s.size)\n    fourier[frequencies > threshold] = 0\n    return irfft(fourier)\n\ndef high_pass(s, threshold=100):\n    fourier = rfft(s)\n    frequencies = rfftfreq(s.size, d=2e-2/s.size)\n    fourier[frequencies < threshold] = 0\n    return irfft(fourier)\n\ndef feature_extractor(data):\n    #print(\"666\")\n    #print(np.shape(data))\n    data=list(data)\n    output=[]\n    leg=800000//2000\n    temp=[data[x:x+leg] for x in range(len(data)//leg)]\n    #print(np.shape(temp))\n    #ave=np.average(data)\n    #output= list(map(lambda x: [max(x),min(x)], temp))\n    output= list(map(lambda x: max(x)-min(x), temp))\n    return output\n'''\ndef feature_extractor(data):\n    #print(\"666\")\n    #print(np.shape(data))\n    data=list(data)\n    output=[]\n    temp=[data[x:x+320] for x in range(len(data)//320)]\n    #print(np.shape(temp))\n    \n    output= list(map(lambda x: max(x)-min(x), temp))\n    return output\n\ndef feature_extractor(data):\n    #print(\"666\")\n    #print(np.shape(data))\n    data=list(data)\n    output=[]\n    temp=[data[x:x+500] for x in range(len(data)//500)]\n    #print(np.shape(temp))\n    #temp_max = list(map(lambda x: max(x), temp))\n    \n    #temp_min = list(map(lambda x: min(x), temp))\n    #print(temp_max[1000])\n    #print(np.shape(temp_max),np.shape(temp_min))\n    #for i in temp:\n    #    output.append(np.average(temp))\n    #print(len(output))\n    output=list(map(lambda x: max(x), temp))\n    return output\n\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def divide_test_data(start,end):\n    test_metadata = pd.read_csv('../input/metadata_train.csv') \n    test_index_list = test_metadata['signal_id'][start:end].values\n    test_index_lists = []\n    #print(start,end)\n    for i in range(len(test_index_list)):\n        test_index_lists.append(str(test_index_list[i]))\n    return test_index_lists\n    test_index_lists_master.append(test_index_lists)\ntest_index_lists_master=[]\ntest_index_lists_master.append(divide_test_data(0,1000))\n#test_index_lists_master.append(divide_test_data(500,1000))\ntest_index_lists_master.append(divide_test_data(1000,2000))\n#test_index_lists_master.append(divide_test_data(1500,2000))\ntest_index_lists_master.append(divide_test_data(2000,3000))\n#test_index_lists_master.append(divide_test_data(2500,3000))\ntest_index_lists_master.append(divide_test_data(3000,4000))\n#test_index_lists_master.append(divide_test_data(3500,4000))\ntest_index_lists_master.append(divide_test_data(4000,5000))\n#test_index_lists_master.append(divide_test_data(4500,5000))\ntest_index_lists_master.append(divide_test_data(5000,6000))\n#test_index_lists_master.append(divide_test_data(5500,6000))\ntest_index_lists_master.append(divide_test_data(6000,7000))\n#test_index_lists_master.append(divide_test_data(6500,7000))\ntest_index_lists_master.append(divide_test_data(7000,8000))\n#test_index_lists_master.append(divide_test_data(7500,8000))\ntest_index_lists_master.append(divide_test_data(8000,8712))\n#train_set =np.array(pq.read_pandas(('/home/tfboy/Downloads/vsb-power-line-fault-detection/train.parquet'),columns=test_index_lists_master[0]).to_pandas().T)\n#print(np.shape(list(train_set)))\n#print(len(train_set[0]))\n#print(len(train_set))\nmeta_train= pd.read_csv('../input/metadata_train.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"83ce36dd1a85aaad8b446e452cbc7e19b882164b","trusted":true},"cell_type":"code","source":"\n#meta_train= pd.read_csv('/home/tfboy/Downloads/vsb-power-line-fault-detection/metadata_train.csv')\n#train_set = pq.read_pandas('/home/tfboy/Downloads/vsb-power-line-fault-detection/train.parquet').to_pandas()\n\nx_train = []\ny_train = []\n#for i in tqdm(meta_train.signal_id):\nfor i in range(len(test_index_lists_master)):\n    print(i)\n    t1=time.time()\n    train_set =np.array(pq.read_pandas(('../input/train.parquet'),columns=test_index_lists_master[i]).to_pandas().T)\n    #y_train.append(meta_train.loc[meta_train.signal_id==i, 'target'].values)\n    #print(len(train_set))\n    train_set2=[]\n    #train_set2=list(map(lambda x: expend_data(x), train_set))\n    #print(len(train_set[0]),len(train_set2[0]))\n    #x_train.append(list(pool.map(feature_extractor, train_set)))\n    #x_train.append(list(pool.map(feature_extractor, train_set2)))\n    x_train.append(list(map(lambda x: feature_extractor(x), train_set)))\n    #x_train.append(list(map(lambda x: feature_extractor(x), train_set2)))\n    gc.collect()\n    print(time.time()-t1)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4cc6eef2ae63a2c79d08c5fe1ce8670449fb807a","trusted":true},"cell_type":"code","source":"print(np.shape(x_train))\n\n\ntemp=np.concatenate((x_train[0], x_train[1]), axis=0)\nfor i in range(2,9):\n    print(i)\n    temp=np.concatenate((temp, x_train[i]), axis=0)\nprint(np.shape(temp))\n#del train_set; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cad66118494f25f70c9ace7b85bea500f1c680c8","trusted":true},"cell_type":"code","source":"y_train=[]\nfor i in tqdm(meta_train.signal_id):\n    y_train.append(meta_train.loc[meta_train.signal_id==i, 'target'].values)\n    \n    \n\ny_train = np.array(y_train).reshape(-1,)\nX_train = np.array(temp).reshape(-1,temp[0].shape[0])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f634073305f25a13dbea2713f3987a98cd7a98f5"},"cell_type":"markdown","source":"## 3. Build Primitive CNN + LSTM Model"},{"metadata":{"_uuid":"5250bf32e062836cbf10394e5f9df4d6b695380e","trusted":true},"cell_type":"code","source":"def keras_auc(y_true, y_pred):\n    auc = tf.metrics.auc(y_true, y_pred)[1]\n    K.get_session().run(tf.local_variables_initializer())\n    return auc","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"034ea6d8cf8bd8a09eb5943ebb3faa01cd991e9d","trusted":true},"cell_type":"code","source":"\nn_signals = 1 #So far each instance is one signal. We will diversify them in next step\nn_outputs = 1 #Binary Classification\n#Build the model\nverbose, epochs, batch_size = True, 15, 16\nn_steps, n_length = 200, 10\nprint(np.shape(X_train))\nX_train = X_train.reshape((X_train.shape[0], n_steps, n_length, n_signals))\nprint(np.shape(X_train))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1c05ecc6abef0b822e422950a5824ea8c55ba861","trusted":true},"cell_type":"code","source":"# define model\nmodel = Sequential()\nmodel.add(TimeDistributed(Conv1D(filters=128, kernel_size=3, activation='relu'), input_shape=(None,n_length,n_signals)))\n#model.add(TimeDistributed(Dropout(0.5)))\nmodel.add(TimeDistributed(Conv1D(filters=96, kernel_size=3, activation='relu')))\n#model.add(TimeDistributed(Dropout(0.5)))\n#model.add(TimeDistributed(Conv1D(filters=64, kernel_size=3, activation='relu')))\n#model.add(TimeDistributed(Conv1D(filters=32, kernel_size=3, activation='relu')))\nmodel.add(TimeDistributed(Dropout(0.5)))\n#model.add(TimeDistributed(MaxPooling1D(pool_size=2)))\n#model.add(TimeDistributed(MaxPooling1D(pool_size=3)))\nmodel.add(TimeDistributed(Flatten()))\n#model.add(TimeDistributed(Dense(108, activation='relu')))\n#model.add(LSTM(60, return_sequences=True, input_shape=(108,1)))\nmodel.add(LSTM(80, return_sequences=True))\n#model.add(LSTM(40, return_sequences=True))\nmodel.add(LSTM(20))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(20, activation='relu'))\nmodel.add(Dense(n_outputs, activation='sigmoid'))\n\nfor layer in model.layers:\n    print(layer.output_shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e48f5b3c1e9cffb34ef0f395f668a205b9cd2508","trusted":true},"cell_type":"code","source":"model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[keras_auc])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 4. Training"},{"metadata":{"_uuid":"7092f9b2feb4b98b47a72738b956a38f178e0075","trusted":true},"cell_type":"code","source":"epochs=15\nmodel.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, verbose=verbose)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"470dee0dc3d9edaaad403eb14275308cb84fb9f8","trusted":true},"cell_type":"code","source":"model.save_weights('mnt8_2.hdf5')\nmodel.save('mnt8_2.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 5. Generate result"},{"metadata":{"trusted":true},"cell_type":"code","source":"def divide_test_data(start,end):\n    test_metadata = pd.read_csv('../input/metadata_test.csv') \n    test_index_list = test_metadata['signal_id'][start:end].values\n    test_index_lists = []\n    for i in range(len(test_index_list)):\n        test_index_lists.append(str(test_index_list[i]))\n    return test_index_lists\n    test_index_lists_master.append(test_index_lists)\n    y_pred = []\nresult=[]\nflag=0\nfor i in range(len(test_index_lists_master)):\n    test = np.array(pq.read_pandas(('../input/test.parquet'),columns=test_index_lists_master[i]).to_pandas().T)    \n    print(len(test),len(test[0]))\n    \n    x_test = list(map(lambda x: feature_extractor(x), test))\n\n    X_test = np.array(x_test).reshape(-1,x_test[0].shape[0])\n    #X_test = np.expand_dims(X_test, axis=2)\n    print(len(X_test),len(X_test[0]))\n    n_signals = 1 #So far each instance is one signal. We will diversify them in next step\n    n_outputs = 1 #Binary Classification\n\n\n    n_steps, n_length = 250, 10\n\n    X_test = X_test.reshape((X_test.shape[0], n_steps, n_length, n_signals))\n    preds = model.predict(X_test)\n    threshpreds = (preds>0.5)*1\n    #print(len(threshpreds))\n    #print(len(threshpreds[0]))\n    #print(type(threshpreds))\n    if flag ==0:\n        result=threshpreds\n        flag=1\n    else:\n        result=np.concatenate((result, threshpreds), axis=0)\n    print(len(result))\n    #print(len(result[0]))\n    #print(type(result))\n    gc.collect()   \n    \nsub = pd.read_csv('../input/sample_submission.csv')\nprint(len(sub))\nprint(len(result))\nsub.target = result\nsub.to_csv('sub16.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.0"}},"nbformat":4,"nbformat_minor":1}