{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This kernel uses 1D convolutions on signals from power lines to identify partial faults","metadata":{"_uuid":"7982834001d894783090561a22e00f211cb75732"}},{"cell_type":"code","source":"import pandas as pd\nimport pyarrow.parquet as pq\nimport os\n\nos.listdir('../input/vsb-power-line-fault-detection')","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2022-06-01T12:04:22.650808Z","iopub.execute_input":"2022-06-01T12:04:22.651056Z","iopub.status.idle":"2022-06-01T12:04:23.055327Z","shell.execute_reply.started":"2022-06-01T12:04:22.651009Z","shell.execute_reply":"2022-06-01T12:04:23.053822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Read the parquet file. The full length of each signal is 800000. We will halve it to 400000 readings to create the pipeline.","metadata":{"_uuid":"7a0a816882adf498a88e6a85429d8ec124af4b10"}},{"cell_type":"code","source":"subset_train = pq.read_pandas('../input/vsb-power-line-fault-detection/train.parquet',columns=[str(i) for i in range(5000)]).to_pandas() #, columns=[str(i) for i in range(10)]).to_pandas()","metadata":{"_uuid":"bd92583ca1db17938c34c8699836b3e17aad346f","execution":{"iopub.status.busy":"2022-06-01T12:04:23.056599Z","iopub.execute_input":"2022-06-01T12:04:23.057003Z","iopub.status.idle":"2022-06-01T12:05:17.541125Z","shell.execute_reply.started":"2022-06-01T12:04:23.056834Z","shell.execute_reply":"2022-06-01T12:05:17.540323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subset_train = subset_train.iloc[200000:600000,:]\nsubset_train.info()","metadata":{"_uuid":"a7164fff4b7c9f0bcf494f7f3fdf164e3b21bc6d","execution":{"iopub.status.busy":"2022-06-01T12:05:17.542517Z","iopub.execute_input":"2022-06-01T12:05:17.542786Z","iopub.status.idle":"2022-06-01T12:05:17.732913Z","shell.execute_reply.started":"2022-06-01T12:05:17.542744Z","shell.execute_reply":"2022-06-01T12:05:17.732120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now read the metadata file.","metadata":{"_uuid":"2973ddcfff4f06ac98ccde46d0b5a5157afe0683"}},{"cell_type":"code","source":"subset_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-01T12:05:17.734208Z","iopub.execute_input":"2022-06-01T12:05:17.734728Z","iopub.status.idle":"2022-06-01T12:05:17.847116Z","shell.execute_reply.started":"2022-06-01T12:05:17.734680Z","shell.execute_reply":"2022-06-01T12:05:17.846375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata_train = pd.read_csv('../input/vsb-power-line-fault-detection/metadata_train.csv')\nmetadata_train.info()","metadata":{"_uuid":"6de250a9d4bf7faec4d39160610ef2fcde2c952d","execution":{"iopub.status.busy":"2022-06-01T12:05:17.848508Z","iopub.execute_input":"2022-06-01T12:05:17.848804Z","iopub.status.idle":"2022-06-01T12:05:17.869493Z","shell.execute_reply.started":"2022-06-01T12:05:17.848754Z","shell.execute_reply":"2022-06-01T12:05:17.868798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata_train.head()","metadata":{"_uuid":"f2359163fe48f20610ed7c426efe94d7239252b7","execution":{"iopub.status.busy":"2022-06-01T12:05:17.870568Z","iopub.execute_input":"2022-06-01T12:05:17.871001Z","iopub.status.idle":"2022-06-01T12:05:17.886927Z","shell.execute_reply.started":"2022-06-01T12:05:17.870950Z","shell.execute_reply":"2022-06-01T12:05:17.886238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Import plotting libraries and create some basic plots.","metadata":{"_uuid":"6f5a84750f1ca51a4fca03e4d0ec03cfaa2bc2dd"}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\n\n#plt.hist(metadata_train['target'])\nsns.countplot(metadata_train['target'])","metadata":{"_uuid":"8aaad8abc0bc4e95ab5c592cd3605994b3c5d1f0","execution":{"iopub.status.busy":"2022-06-01T12:05:17.888262Z","iopub.execute_input":"2022-06-01T12:05:17.888733Z","iopub.status.idle":"2022-06-01T12:05:18.977260Z","shell.execute_reply.started":"2022-06-01T12:05:17.888549Z","shell.execute_reply":"2022-06-01T12:05:18.976338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This is a plot of the target values. As expected, a faulty power line is a kind of rare event. Let's visualize some negative and positive (faulty) signals.","metadata":{"_uuid":"d07f9ed35254ca6577def6c94de901129617e1f9"}},{"cell_type":"code","source":"fig = plt.figure(figsize=(10,8))\n\nplt.subplot(431 )    #画一个4*3的画布，竖着4个图横着3个图。431等效于4，3，1 。 这里开始画第一个图，就是下面那句\nplt.plot(subset_train['5'])    # 第一个图的内容是subset_train数据集里面列标题是“5”的那一列。\nplt.subplot(432)\nplt.boxplot(subset_train['5'])\nplt.subplot(433)\nplt.hist(subset_train['5'])\n    \nplt.subplot(434)\nplt.plot(subset_train['1'])\nplt.subplot(435)\nplt.boxplot(subset_train['1'])\nplt.subplot(436)\nplt.hist(subset_train['1'])\n\nplt.subplot(437)\nplt.plot(subset_train['3'])\nplt.subplot(438)\nplt.boxplot(subset_train['3'])\nplt.subplot(439)\nplt.hist(subset_train['3'])\n\nplt.subplot(4,3,10)\nplt.plot(subset_train['4'])\nplt.subplot(4,3,11)\nplt.boxplot(subset_train['4'])\nplt.subplot(4,3,12)\nplt.hist(subset_train['4'])","metadata":{"_uuid":"ab20ac984b83a05d25eaf79f49dcf981c35356ed","execution":{"iopub.status.busy":"2022-06-01T12:05:18.978908Z","iopub.execute_input":"2022-06-01T12:05:18.979189Z","iopub.status.idle":"2022-06-01T12:05:21.906044Z","shell.execute_reply.started":"2022-06-01T12:05:18.979143Z","shell.execute_reply":"2022-06-01T12:05:21.905126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(22,10))\n\nplt.subplot(361 )    #画一个4*3的画布，竖着4个图横着3个图。431等效于4，3，1 。 这里开始画第一个图，就是下面那句\nplt.plot(subset_train['0'])    # 第一个图的内容是subset_train数据集里面列标题是“5”的那一列。\nplt.subplot(362)\nplt.boxplot(subset_train['0'])\nplt.subplot(363)\nplt.hist(subset_train['0'])\n    \nplt.subplot(364)\nplt.plot(subset_train['1'])\nplt.subplot(365)\nplt.boxplot(subset_train['1'])\nplt.subplot(366)\nplt.hist(subset_train['1'])\n\nplt.subplot(367)\nplt.plot(subset_train['2'])\nplt.subplot(368)\nplt.boxplot(subset_train['2'])\nplt.subplot(369)\nplt.hist(subset_train['2'])\n\nplt.subplot(3, 6 ,10)\nplt.plot(subset_train['3'])\nplt.subplot(3,6,11)\nplt.boxplot(subset_train['3'])\nplt.subplot(3,6,12)\nplt.hist(subset_train['3'])\n\nplt.subplot(3,6 ,13)\nplt.plot(subset_train['4'])\nplt.subplot(3,6,14)\nplt.boxplot(subset_train['4'])\nplt.subplot(3,6,15)\nplt.hist(subset_train['4'])\n\nplt.subplot(3,6 ,16)\nplt.plot(subset_train['5'])\nplt.subplot(3,6,17)\nplt.boxplot(subset_train['5'])\nplt.subplot(3,6,18)\nplt.hist(subset_train['5'])","metadata":{"execution":{"iopub.status.busy":"2022-06-01T12:05:21.911086Z","iopub.execute_input":"2022-06-01T12:05:21.913718Z","iopub.status.idle":"2022-06-01T12:05:25.836815Z","shell.execute_reply.started":"2022-06-01T12:05:21.913651Z","shell.execute_reply":"2022-06-01T12:05:25.836050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"At least from these couple of plots, we notice that the faulty signals (last 2) have relatively more outliers than the non-faulty ones. We will analyze this further with more data.\n\nLet's separate the positive and negative signals for further analysis. I'm going to reduce the sample sizes to make sure we don't run out of memory limits.","metadata":{"_uuid":"28b29bfcc7af20c19e15cec6c78afb59449f5fbc"}},{"cell_type":"code","source":"import numpy as np\nS_decimation = subset_train.iloc[0:25000:8,:]\n# Temporarily reduce data size to build the pipeline\nsmall_subset_train = S_decimation\nsmall_subset_train = small_subset_train.transpose()\nsmall_subset_train.index = small_subset_train.index.astype(np.int32)\ntrain_dataset = metadata_train.join(small_subset_train, how='right')\n\n# Uncomment the following to train on the full dataset\n#subset_train = subset_train.transpose()\n#subset_train.index = subset_train.index.astype(np.int32)\n#train_dataset = metadata_train.join(subset_train, how='right')","metadata":{"_uuid":"a5fb73acf543de684313260c9358acf252e9cbdf","execution":{"iopub.status.busy":"2022-06-01T12:05:25.837875Z","iopub.execute_input":"2022-06-01T12:05:25.838160Z","iopub.status.idle":"2022-06-01T12:05:26.099107Z","shell.execute_reply.started":"2022-06-01T12:05:25.838117Z","shell.execute_reply":"2022-06-01T12:05:26.098193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"positive_samples = train_dataset[train_dataset['target']==1]\npositive_samples = positive_samples.iloc[:,3:]\npositive_samples.info()","metadata":{"execution":{"iopub.status.busy":"2022-06-01T12:05:26.100508Z","iopub.execute_input":"2022-06-01T12:05:26.100772Z","iopub.status.idle":"2022-06-01T12:05:26.225918Z","shell.execute_reply.started":"2022-06-01T12:05:26.100728Z","shell.execute_reply":"2022-06-01T12:05:26.225093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"positive_samples.head()","metadata":{"_uuid":"24379b4d531df931d7c15e004d657425e9a7c7de","execution":{"iopub.status.busy":"2022-06-01T12:05:26.227643Z","iopub.execute_input":"2022-06-01T12:05:26.228141Z","iopub.status.idle":"2022-06-01T12:05:26.401676Z","shell.execute_reply.started":"2022-06-01T12:05:26.227897Z","shell.execute_reply":"2022-06-01T12:05:26.400356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"positive_samples.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-01T12:05:26.403549Z","iopub.execute_input":"2022-06-01T12:05:26.404174Z","iopub.status.idle":"2022-06-01T12:05:26.412433Z","shell.execute_reply.started":"2022-06-01T12:05:26.404121Z","shell.execute_reply":"2022-06-01T12:05:26.411534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now let's visualize the positive (faulty) signals using a boxplot for several of them.","metadata":{"_uuid":"3a44931440815cb30e88b057c84e7213b356b025"}},{"cell_type":"code","source":"plt.figure(figsize=(10,4))\nplt.subplot(151)\nplt.boxplot(positive_samples.iloc[201,1:])\nplt.subplot(152)\nplt.boxplot(positive_samples.iloc[150,1:])\nplt.subplot(153)\nplt.boxplot(positive_samples.iloc[110,1:])\n\n","metadata":{"_uuid":"8b942a1a76c8475c9c6be3d84c8d8687ee16e3a8","execution":{"iopub.status.busy":"2022-06-01T12:05:26.415264Z","iopub.execute_input":"2022-06-01T12:05:26.415809Z","iopub.status.idle":"2022-06-01T12:05:27.069428Z","shell.execute_reply.started":"2022-06-01T12:05:26.415750Z","shell.execute_reply":"2022-06-01T12:05:27.068206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see that the data values differ a lot. Let's normalize the data first, this will also be needed for training some type of models later.","metadata":{"_uuid":"7a2d03a69c9b3d7f8f264abb020460b741aa8764"}},{"cell_type":"code","source":"# Normalize the data set\nfrom sklearn.preprocessing import StandardScaler\ny_train_pos = positive_samples.iloc[:, 0]\nX_train_pos = positive_samples.iloc[:, 1:]\nscaler = StandardScaler()         #通过去除均值并将其缩放为单位方差来标准化特征样本\nscaler.fit(X_train_pos.T)            #对\nX_train_pos = scaler.transform(X_train_pos.T).T","metadata":{"_uuid":"29e59231f9412c9d820b391293a880ebaa9f1689","execution":{"iopub.status.busy":"2022-06-01T12:05:27.070647Z","iopub.execute_input":"2022-06-01T12:05:27.071051Z","iopub.status.idle":"2022-06-01T12:05:27.328564Z","shell.execute_reply.started":"2022-06-01T12:05:27.071004Z","shell.execute_reply":"2022-06-01T12:05:27.327626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's visualize the boxplots again using this normalized data.","metadata":{"_uuid":"08dce41d4527ebe95444f46919d949684333b080"}},{"cell_type":"code","source":"X_train_pos","metadata":{"execution":{"iopub.status.busy":"2022-06-01T12:05:27.330088Z","iopub.execute_input":"2022-06-01T12:05:27.330374Z","iopub.status.idle":"2022-06-01T12:05:27.336780Z","shell.execute_reply.started":"2022-06-01T12:05:27.330327Z","shell.execute_reply":"2022-06-01T12:05:27.335937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,4))\nplt.subplot(151)\nplt.boxplot(X_train_pos[0,:])\nplt.subplot(152)\nplt.boxplot(X_train_pos[1,:])\nplt.subplot(153)\nplt.boxplot(X_train_pos[2,:])\nplt.subplot(154)\nplt.boxplot(X_train_pos[3,:])\nplt.subplot(155)\nplt.boxplot(X_train_pos[4,:])","metadata":{"_uuid":"07ddaf401fb44e18f67fd1e6a901baae3ca83d01","execution":{"iopub.status.busy":"2022-06-01T12:05:27.338019Z","iopub.execute_input":"2022-06-01T12:05:27.338608Z","iopub.status.idle":"2022-06-01T12:05:28.006814Z","shell.execute_reply.started":"2022-06-01T12:05:27.338549Z","shell.execute_reply":"2022-06-01T12:05:28.005979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Again we notice that there are a lot of outliers in the positive (faulty) signals.\n\nNow let's extract the negative (non-faulty) samples and visualize the same boxplots, and see if we can notice any apparent difference.","metadata":{"_uuid":"08d130bb14bf8a096fdd3810b7e298a4b56f98cb"}},{"cell_type":"code","source":"negative_samples = train_dataset[train_dataset['target']==0]\nnegative_samples = negative_samples.iloc[:,3:]\n\ny_train_neg = negative_samples.iloc[:, 0]\nX_train_neg = negative_samples.iloc[:, 1:]\nscaler.fit(X_train_neg.T)\nX_train_neg = scaler.transform(X_train_neg.T).T\n\nplt.figure(figsize=(10,4))\nplt.subplot(151)\nplt.boxplot(X_train_neg[0,:])\nplt.subplot(152)\nplt.boxplot(X_train_neg[1,:])\nplt.subplot(153)\nplt.boxplot(X_train_neg[2,:])\nplt.subplot(154)\nplt.boxplot(X_train_neg[3,:])\nplt.subplot(155)\nplt.boxplot(X_train_neg[4,:])","metadata":{"_uuid":"f7c99280ac588f718325b75a41fda68f62475966","execution":{"iopub.status.busy":"2022-06-01T12:05:28.008143Z","iopub.execute_input":"2022-06-01T12:05:28.013236Z","iopub.status.idle":"2022-06-01T12:05:29.255499Z","shell.execute_reply.started":"2022-06-01T12:05:28.013173Z","shell.execute_reply":"2022-06-01T12:05:29.254610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"negative_samples.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-01T12:05:29.260408Z","iopub.execute_input":"2022-06-01T12:05:29.262751Z","iopub.status.idle":"2022-06-01T12:05:29.408076Z","shell.execute_reply.started":"2022-06-01T12:05:29.260672Z","shell.execute_reply":"2022-06-01T12:05:29.407390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"negative_samples.shape\n","metadata":{"execution":{"iopub.status.busy":"2022-06-01T12:05:29.409469Z","iopub.execute_input":"2022-06-01T12:05:29.409738Z","iopub.status.idle":"2022-06-01T12:05:29.415822Z","shell.execute_reply.started":"2022-06-01T12:05:29.409691Z","shell.execute_reply":"2022-06-01T12:05:29.415086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"positive_samples.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-01T12:05:29.417150Z","iopub.execute_input":"2022-06-01T12:05:29.417665Z","iopub.status.idle":"2022-06-01T12:05:29.424532Z","shell.execute_reply.started":"2022-06-01T12:05:29.417615Z","shell.execute_reply":"2022-06-01T12:05:29.423688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_neg","metadata":{"execution":{"iopub.status.busy":"2022-06-01T12:05:29.425796Z","iopub.execute_input":"2022-06-01T12:05:29.426422Z","iopub.status.idle":"2022-06-01T12:05:29.435214Z","shell.execute_reply.started":"2022-06-01T12:05:29.426372Z","shell.execute_reply":"2022-06-01T12:05:29.434208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The negative (non-faulty) signals have much fewer outliers, and their magnitudes also seem to be very low. Seems like the number of outliers could be a promising feature.\n\nNow let's create the test/train split for training a Conv1D model.","metadata":{"_uuid":"d9bc3cef7d6e57668a441a11715d05af5c0e3436"}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train_pos, X_valid_pos, y_train_pos, y_valid_pos = train_test_split(X_train_pos, y_train_pos, \n                                                                    test_size=0.2,\n                                                                    random_state = 0,\n                                                                    shuffle=True)\n\nX_train_neg, X_valid_neg, y_train_neg, y_valid_neg = train_test_split(X_train_neg, y_train_neg, \n                                                                    test_size=0.2,\n                                                                    random_state = 0,\n                                                                    shuffle=True)","metadata":{"_uuid":"3945a11048622a9a88e5335d62bfcb1be870bc3e","execution":{"iopub.status.busy":"2022-06-01T12:05:29.436401Z","iopub.execute_input":"2022-06-01T12:05:29.436928Z","iopub.status.idle":"2022-06-01T12:05:30.032590Z","shell.execute_reply.started":"2022-06-01T12:05:29.436634Z","shell.execute_reply":"2022-06-01T12:05:30.031608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_pos.shape, X_train_neg.shape","metadata":{"_uuid":"28373fafd9cf52acb37e49f7aafc8a6abf57630e","execution":{"iopub.status.busy":"2022-06-01T12:05:30.034052Z","iopub.execute_input":"2022-06-01T12:05:30.034648Z","iopub.status.idle":"2022-06-01T12:05:30.040825Z","shell.execute_reply.started":"2022-06-01T12:05:30.034590Z","shell.execute_reply":"2022-06-01T12:05:30.040046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we know, the positive samples are fewer, so we will only select a subset of negative samples for training.","metadata":{"_uuid":"4496e1c8d6c09d1c95a03768c75706339856e7e1"}},{"cell_type":"code","source":"# Combine positive and negative samples for training...\ndef combine_positive_and_negative_samples(pos_samples, neg_samples, y_pos, y_neg):\n    X_combined = np.concatenate((pos_samples, neg_samples)) \n                                                    # don't select all negative samples, to\n                                                    # keep the samples balanced\n    y_combined = np.concatenate((y_pos, y_neg))\n    #X_train_combined.shape, y_train_combined.shape\n    combined_samples = np.hstack((X_combined, y_combined.reshape(y_combined.shape[0],1)))\n    np.random.shuffle(combined_samples)\n    return combined_samples\n\n# Only use 500 negative samples, to create a balanced dataset with the positive samples...\ntrain_samples = combine_positive_and_negative_samples(X_train_pos, X_train_neg[:500, :], y_train_pos, y_train_neg[:500])\nX_train = train_samples[:,:-1]\ny_train = train_samples[:,-1]\nX_train.shape, y_train.shape","metadata":{"_uuid":"feb7e65bd79d359efaeda750ef519ecf6dca5d3c","execution":{"iopub.status.busy":"2022-06-01T12:05:30.042138Z","iopub.execute_input":"2022-06-01T12:05:30.042796Z","iopub.status.idle":"2022-06-01T12:05:30.083101Z","shell.execute_reply.started":"2022-06-01T12:05:30.042633Z","shell.execute_reply":"2022-06-01T12:05:30.082271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_samples.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-01T12:05:30.084256Z","iopub.execute_input":"2022-06-01T12:05:30.084535Z","iopub.status.idle":"2022-06-01T12:05:30.091369Z","shell.execute_reply.started":"2022-06-01T12:05:30.084487Z","shell.execute_reply":"2022-06-01T12:05:30.090548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create the validation set\n#X_valid_combined = np.concatenate((X_valid_pos, X_valid_neg[:500,:])) # don't select all negative samples, to\n                                                  # keep the samples balanced\n#y_valid_combined = np.concatenate((y_valid_pos, y_valid_neg[:500]))\n#X_valid_combined.shape, y_valid_combined.shape\n#validation_samples = np.hstack((X_valid_combined, y_valid_combined.reshape(y_valid_combined.shape[0],1)))\n#np.random.shuffle(validation_samples)\n\nvalidation_samples = combine_positive_and_negative_samples(X_valid_pos, X_valid_neg[:500,:], y_valid_pos, y_valid_neg[:500])\nX_valid = validation_samples[:,:-1]\ny_valid = validation_samples[:,-1]\nX_valid.shape, y_valid.shape","metadata":{"_uuid":"1e2248e93cb07a9c0fb2466099283dfc9bdfb7bb","execution":{"iopub.status.busy":"2022-06-01T12:05:30.092849Z","iopub.execute_input":"2022-06-01T12:05:30.093615Z","iopub.status.idle":"2022-06-01T12:05:30.122966Z","shell.execute_reply.started":"2022-06-01T12:05:30.093543Z","shell.execute_reply":"2022-06-01T12:05:30.122164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape, X_valid.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-01T12:05:30.125150Z","iopub.execute_input":"2022-06-01T12:05:30.125585Z","iopub.status.idle":"2022-06-01T12:05:30.131457Z","shell.execute_reply.started":"2022-06-01T12:05:30.125538Z","shell.execute_reply":"2022-06-01T12:05:30.130666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"A 1-D ConvNet would be an interesting model to try out on this signal. Earlier we saw that there are a lot of outliers in fauty signals. Since the actual signal value differs at different times, the outliers are relative to this mean signal value. A 1-D ConvNet can analyze the signal in various windows of increasing lengths and create high-level features out of that to classify on.","metadata":{"_uuid":"4383b24325024d54a179ee1a064ab3d2b94fc336"}},{"cell_type":"code","source":"# Reshape training and validation data for keras input layer\nX_train = X_train.reshape(-1,1,3125, 1)\nX_valid = X_valid.reshape(-1,1,3125, 1)\n\nX_train.shape, X_valid.shape, y_train.shape, y_valid.shape\n#print(X_train)","metadata":{"_uuid":"dc8c83b24a0834051a811a1c1bb494397470d40b","execution":{"iopub.status.busy":"2022-06-01T12:05:30.132718Z","iopub.execute_input":"2022-06-01T12:05:30.133182Z","iopub.status.idle":"2022-06-01T12:05:30.142266Z","shell.execute_reply.started":"2022-06-01T12:05:30.133132Z","shell.execute_reply":"2022-06-01T12:05:30.141318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_valid = X_valid.astype(np.float32)\ny_valid = y_valid.astype(np.float32)\nX_train = X_train.astype(np.float32)\ny_train = y_train.astype(np.float32)\nX_train.dtype","metadata":{"execution":{"iopub.status.busy":"2022-06-01T12:05:30.144464Z","iopub.execute_input":"2022-06-01T12:05:30.144695Z","iopub.status.idle":"2022-06-01T12:05:30.157244Z","shell.execute_reply.started":"2022-06-01T12:05:30.144651Z","shell.execute_reply":"2022-06-01T12:05:30.156083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.save('X_valid.npy',X_valid)\nnp.save('y_valid.npy',y_valid)\nnp.save('X_train.npy',X_train)\nnp.save('y_train.npy',y_train)","metadata":{"execution":{"iopub.status.busy":"2022-06-01T12:05:30.158722Z","iopub.execute_input":"2022-06-01T12:05:30.159208Z","iopub.status.idle":"2022-06-01T12:05:30.179109Z","shell.execute_reply.started":"2022-06-01T12:05:30.158992Z","shell.execute_reply":"2022-06-01T12:05:30.178460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nX_valid = np.load('X_valid.npy')\ny_valid = np.load('y_valid.npy')\nX_train = np.load('X_train.npy')\ny_train = np.load('y_train.npy')","metadata":{"execution":{"iopub.status.busy":"2022-06-01T12:05:30.180551Z","iopub.execute_input":"2022-06-01T12:05:30.181013Z","iopub.status.idle":"2022-06-01T12:05:30.197256Z","shell.execute_reply.started":"2022-06-01T12:05:30.180966Z","shell.execute_reply":"2022-06-01T12:05:30.196679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.mean(X_train, axis = 0), np.max(X_train, axis = 0), np.min(X_train, axis = 0), np.std(X_train, axis = 0)","metadata":{"execution":{"iopub.status.busy":"2022-06-01T12:05:30.198741Z","iopub.execute_input":"2022-06-01T12:05:30.199258Z","iopub.status.idle":"2022-06-01T12:05:30.217934Z","shell.execute_reply.started":"2022-06-01T12:05:30.199205Z","shell.execute_reply":"2022-06-01T12:05:30.217098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.sum(y_valid), np.sum(y_train)","metadata":{"execution":{"iopub.status.busy":"2022-06-01T12:05:30.219375Z","iopub.execute_input":"2022-06-01T12:05:30.219950Z","iopub.status.idle":"2022-06-01T12:05:30.225715Z","shell.execute_reply.started":"2022-06-01T12:05:30.219887Z","shell.execute_reply":"2022-06-01T12:05:30.224769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_normalize(data):\n    mu = np.mean(data,axis=0)\n    std = np.std(data,axis=0)\n    return (data - mu)/std","metadata":{"execution":{"iopub.status.busy":"2022-06-01T12:05:30.227118Z","iopub.execute_input":"2022-06-01T12:05:30.227659Z","iopub.status.idle":"2022-06-01T12:05:30.233105Z","shell.execute_reply.started":"2022-06-01T12:05:30.227613Z","shell.execute_reply":"2022-06-01T12:05:30.232333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"X_valid = feature_normalize(X_valid)\nX_train = feature_normalize(X_train)","metadata":{}},{"cell_type":"code","source":"import torch\nimport torchvision\nfrom torchvision import datasets, transforms\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torch.utils.data as Data\nprint(torch.__version__)","metadata":{"execution":{"iopub.status.busy":"2022-06-01T12:05:30.234480Z","iopub.execute_input":"2022-06-01T12:05:30.234977Z","iopub.status.idle":"2022-06-01T12:05:31.082363Z","shell.execute_reply.started":"2022-06-01T12:05:30.234852Z","shell.execute_reply":"2022-06-01T12:05:31.081613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class torch_Dataset(Data.Dataset): # 需要继承 data.Dataset\n    def __init__(self, x, y):\n        self.x = torch.from_numpy(x)\n        self.y = torch.from_numpy(y)\n    def __getitem__(self, index):\n        data = (self.x[index], self.y[index])\n        return data\n    def __len__(self):\n        return len(self.y)","metadata":{"execution":{"iopub.status.busy":"2022-06-01T12:05:31.083500Z","iopub.execute_input":"2022-06-01T12:05:31.083929Z","iopub.status.idle":"2022-06-01T12:05:31.090732Z","shell.execute_reply.started":"2022-06-01T12:05:31.083880Z","shell.execute_reply":"2022-06-01T12:05:31.088831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def training_loader(train_data, batch_size, shuffle):\n    return torch.utils.data.DataLoader(train_data, batch_size, shuffle)\n","metadata":{"execution":{"iopub.status.busy":"2022-06-01T12:05:31.091931Z","iopub.execute_input":"2022-06-01T12:05:31.092468Z","iopub.status.idle":"2022-06-01T12:05:31.103424Z","shell.execute_reply.started":"2022-06-01T12:05:31.092162Z","shell.execute_reply":"2022-06-01T12:05:31.102495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Train_dataset = torch_Dataset(X_train, y_train)\ntest_dataset = torch_Dataset(X_valid, y_valid)\ntrain_loader = training_loader(Train_dataset, batch_size=1, shuffle=True)\ntest_loader = training_loader(test_dataset, batch_size=1, shuffle=True)  \n","metadata":{"execution":{"iopub.status.busy":"2022-06-01T12:05:31.104910Z","iopub.execute_input":"2022-06-01T12:05:31.105611Z","iopub.status.idle":"2022-06-01T12:05:31.119167Z","shell.execute_reply.started":"2022-06-01T12:05:31.105329Z","shell.execute_reply":"2022-06-01T12:05:31.118420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CNN(nn.Module):\n    def __init__(self):\n        super(CNN,self).__init__()\n        self.conv1 = nn.Conv2d(1, 32, kernel_size=(16,1), padding=0, stride=(2,1))\n        self.bn1 = nn.BatchNorm2d(32)\n        self.rl1 =  nn.ReLU()\n        self.pool1 = nn.MaxPool2d((2,1))\n        self.do1 =    nn.Dropout(0.2)\n        self.conv2 = nn.Conv2d(32, 128, kernel_size=(16,1), padding=0, stride=(2,1))\n        self.bn2 = nn.BatchNorm2d(128)\n        self.rl2 =  nn.ReLU()\n        self.pool2 = nn.MaxPool2d((2,1))\n        self.do2 =    nn.Dropout(0.2)\n        self.conv3 = nn.Conv2d(128, 256, kernel_size=(10,1), padding=0, stride=(2,1))\n        self.bn3 = nn.BatchNorm2d(256)\n        self.rl3 =  nn.ReLU()\n        self.pool3 = nn.MaxPool2d((2,1))\n        self.do3 =    nn.Dropout(0.2)\n        self.conv6 = nn.Conv2d(256, 64, kernel_size=(10,1), padding=0, stride=(2,1))\n        self.rl6 =  nn.ReLU()\n        \n        self.fc1   = nn.Linear(1152, 512)\n        self.rl7 =  nn.ReLU()\n        self.do7 =    nn.Dropout(0.2)\n        self.fc2   = nn.Linear(512, 128)\n        self.rl8 =  nn.ReLU()\n        self.do8 =    nn.Dropout(0.2)\n        self.fc3   = nn.Linear(128, 1)\n      \n            \n        \n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = self.rl1(x)\n        x = self.pool1(x)\n        x = self.do1(x)\n        x = self.conv2(x)\n        x = self.bn2(x)\n        x = self.rl2(x)\n        x = self.pool2(x)\n        x = self.do2(x)\n        x = self.conv3(x)\n        x = self.bn3(x)\n        x = self.rl3(x)\n        x = self.pool3(x)\n        x = self.do3(x)\n        x = self.conv6(x)\n        x = self.rl6(x)\n        x = x.view(-1,1152)\n   \n        x = self.fc1(x)\n        x = self.rl7(x)\n        x = self.do7(x)\n        x = self.fc2(x)\n        x = self.rl8(x)\n        x = self.do8(x)\n        x = self.fc3(x)\n       # print(x.shape)\n        x = torch.sigmoid(x)\n        \n        return x\n","metadata":{"execution":{"iopub.status.busy":"2022-06-01T12:05:31.120785Z","iopub.execute_input":"2022-06-01T12:05:31.121245Z","iopub.status.idle":"2022-06-01T12:05:31.136956Z","shell.execute_reply.started":"2022-06-01T12:05:31.121061Z","shell.execute_reply":"2022-06-01T12:05:31.136126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"    if torch.cuda.is_available():\n        device = torch.device( \"cuda\")\n    else:\n        device = torch.device( \"cpu\")\n    print(f\"training on {device} device.\")\n    model = CNN().to(device)\n    optimizer = optim.Adam(model.parameters(), lr=0.0001) #1e-2)\n    loss_fn = nn.BCELoss()\n\n    print(model)","metadata":{"execution":{"iopub.status.busy":"2022-06-01T12:05:31.138368Z","iopub.execute_input":"2022-06-01T12:05:31.138977Z","iopub.status.idle":"2022-06-01T12:05:34.742688Z","shell.execute_reply.started":"2022-06-01T12:05:31.138900Z","shell.execute_reply":"2022-06-01T12:05:34.741888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def training_loop(n_epochs, optimizer, model, loss_fn, train_loader, dev):\n    for epoch in range(1, n_epochs + 1):\n        loss_train = 0.0\n        for imgs, labels in train_loader:\n            imgs = imgs.to(device=dev)\n            #imgs = imgs.float() \n            labels = labels.to(device=dev)\n            outputs = model(imgs)\n            #outputs = outputs.squeeze(-1)       \n            #print(\"train output size \",outputs.size())\n           # print(\"label output size \",labels.size())\n            #print(outputs)\n            optimizer.zero_grad()\n            loss = loss_fn(outputs, labels)\n            loss.backward()\n            optimizer.step()\n            loss_train += loss.item()\n        if epoch == 1 or epoch % 2 == 0:\n            print('{0} Epoch {1:3d}, Training loss {2:.6f}'.format(\n                datetime.datetime.now(), epoch,\n                loss_train / len(train_loader)))\n    return","metadata":{"execution":{"iopub.status.busy":"2022-06-01T12:05:34.744182Z","iopub.execute_input":"2022-06-01T12:05:34.744736Z","iopub.status.idle":"2022-06-01T12:05:34.752206Z","shell.execute_reply.started":"2022-06-01T12:05:34.744684Z","shell.execute_reply":"2022-06-01T12:05:34.751283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"    import warnings\n    import datetime\n    warnings.filterwarnings('ignore') \n    training_loop(\n        n_epochs = 20, #100,\n        optimizer = optimizer,\n        model = model,\n        loss_fn = loss_fn,\n        train_loader = train_loader,\n        dev = device\n    )","metadata":{"execution":{"iopub.status.busy":"2022-06-01T12:05:34.754886Z","iopub.execute_input":"2022-06-01T12:05:34.755164Z","iopub.status.idle":"2022-06-01T12:06:44.260190Z","shell.execute_reply.started":"2022-06-01T12:05:34.755116Z","shell.execute_reply":"2022-06-01T12:06:44.259403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nprint('Trained model written to ', 'torch_chkp.pt')\ntorch.save(model.state_dict(), 'torch_chkp.pt')\nloaded_model = CNN()  # <1>\nloaded_model.load_state_dict(torch.load('torch_chkp.pt'))","metadata":{"execution":{"iopub.status.busy":"2022-06-01T12:06:44.261392Z","iopub.execute_input":"2022-06-01T12:06:44.261837Z","iopub.status.idle":"2022-06-01T12:06:44.304047Z","shell.execute_reply.started":"2022-06-01T12:06:44.261786Z","shell.execute_reply":"2022-06-01T12:06:44.303322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def validate(model, train_loader, val_loader, dev):\n    accdict = {}\n    for name, loader in [(\"train dataset\", train_loader), (\"test dataset  \", val_loader)]:\n        correct = 0\n        total = 0\n        with torch.no_grad():\n            for imgs, labels in loader:\n                imgs = imgs.float() \n                imgs = imgs.to(device=dev)\n                labels = labels.to(device=dev)\n                outputs = model(imgs)\n                \n                predicted = torch.max(outputs) # <1>\n                print(predicted)\n                if(predicted>0.5):\n                   #print(predicted)\n                    falt_detected =1\n                else:\n                    falt_detected =0\n                total += labels.shape[0]\n                correct += int((falt_detected == labels).sum())\n                print(\"predict value:\", falt_detected, \"real value:\" , labels)\n                \n        print(\"Accuracy {0}: {1:.2f}(%)\".format(name , 100*(correct/total)))\n        accdict[name] = correct / total\n    return accdict","metadata":{"execution":{"iopub.status.busy":"2022-06-01T12:17:28.557673Z","iopub.execute_input":"2022-06-01T12:17:28.557976Z","iopub.status.idle":"2022-06-01T12:17:28.566160Z","shell.execute_reply.started":"2022-06-01T12:17:28.557923Z","shell.execute_reply":"2022-06-01T12:17:28.564997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fit(model, train_loader, val_loader, dev):\n    accdict = {}\n    for name, loader in [(\"train dataset\", train_loader), (\"test dataset  \", val_loader)]:\n        correct = 0\n        total = 0\n        with torch.no_grad():\n            for imgs, labels in loader:\n                imgs = imgs.float() \n                imgs = imgs.to(device=dev)\n                labels = labels.to(device=dev)\n                outputs = model(imgs)\n                \n                predicted = torch.max(outputs) # <1>\n                #print(predicted)\n  \n    return accdict","metadata":{"execution":{"iopub.status.busy":"2022-06-01T12:17:33.555746Z","iopub.execute_input":"2022-06-01T12:17:33.556084Z","iopub.status.idle":"2022-06-01T12:17:33.564515Z","shell.execute_reply.started":"2022-06-01T12:17:33.556021Z","shell.execute_reply":"2022-06-01T12:17:33.563437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validate(model, train_loader, test_loader, device)","metadata":{"execution":{"iopub.status.busy":"2022-06-01T12:17:38.270270Z","iopub.execute_input":"2022-06-01T12:17:38.270575Z","iopub.status.idle":"2022-06-01T12:17:43.248640Z","shell.execute_reply.started":"2022-06-01T12:17:38.270524Z","shell.execute_reply":"2022-06-01T12:17:43.247772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\nimport collections\nloaded_model = CNN()  # <1>\nloaded_model.load_state_dict(torch.load('torch_chkp.pt'))\nall_acc_dict = collections.OrderedDict()\ntime_start = time.time()\nall_acc_dict[\"baseline\"] = validate(model, train_loader, test_loader, device)\ntime_end = time.time()\ntimetotal = time_end - time_start\nrunTotal = len(train_loader)\nfps = float(runTotal / timetotal)\nprint(\"FPS=%.2f, total frames = %.0f , time=%.4f seconds\" %(fps,runTotal, timetotal))","metadata":{"execution":{"iopub.status.busy":"2022-06-01T12:18:58.079213Z","iopub.execute_input":"2022-06-01T12:18:58.079758Z","iopub.status.idle":"2022-06-01T12:19:03.311682Z","shell.execute_reply.started":"2022-06-01T12:18:58.079688Z","shell.execute_reply":"2022-06-01T12:19:03.311043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}