{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This kernel uses 1D convolutions on signals from power lines to identify partial faults","metadata":{"_uuid":"7982834001d894783090561a22e00f211cb75732"}},{"cell_type":"code","source":"import pandas as pd\nimport pyarrow.parquet as pq\nimport os\n\nos.listdir('../input/vsb-power-line-fault-detection')","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2022-03-24T10:34:59.855121Z","iopub.execute_input":"2022-03-24T10:34:59.855392Z","iopub.status.idle":"2022-03-24T10:35:00.206963Z","shell.execute_reply.started":"2022-03-24T10:34:59.855344Z","shell.execute_reply":"2022-03-24T10:35:00.206263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Read the parquet file. The full length of each signal is 800000. We will halve it to 400000 readings to create the pipeline.","metadata":{"_uuid":"7a0a816882adf498a88e6a85429d8ec124af4b10"}},{"cell_type":"code","source":"subset_train = pq.read_pandas('../input/vsb-power-line-fault-detection/train.parquet',columns=[str(i) for i in range(5000)]).to_pandas() #, columns=[str(i) for i in range(10)]).to_pandas()","metadata":{"_uuid":"bd92583ca1db17938c34c8699836b3e17aad346f","execution":{"iopub.status.busy":"2022-03-24T10:35:00.208158Z","iopub.execute_input":"2022-03-24T10:35:00.208563Z","iopub.status.idle":"2022-03-24T10:35:47.829626Z","shell.execute_reply.started":"2022-03-24T10:35:00.208512Z","shell.execute_reply":"2022-03-24T10:35:47.828951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subset_train = subset_train.iloc[200000:600000,:]\nsubset_train.info()","metadata":{"_uuid":"a7164fff4b7c9f0bcf494f7f3fdf164e3b21bc6d","execution":{"iopub.status.busy":"2022-03-24T10:35:47.830854Z","iopub.execute_input":"2022-03-24T10:35:47.831104Z","iopub.status.idle":"2022-03-24T10:35:48.018835Z","shell.execute_reply.started":"2022-03-24T10:35:47.831061Z","shell.execute_reply":"2022-03-24T10:35:48.017555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now read the metadata file.","metadata":{"_uuid":"2973ddcfff4f06ac98ccde46d0b5a5157afe0683"}},{"cell_type":"code","source":"subset_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-24T10:35:48.020099Z","iopub.execute_input":"2022-03-24T10:35:48.020350Z","iopub.status.idle":"2022-03-24T10:35:48.132760Z","shell.execute_reply.started":"2022-03-24T10:35:48.020307Z","shell.execute_reply":"2022-03-24T10:35:48.131911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata_train = pd.read_csv('../input/vsb-power-line-fault-detection/metadata_train.csv')\nmetadata_train.info()","metadata":{"_uuid":"6de250a9d4bf7faec4d39160610ef2fcde2c952d","execution":{"iopub.status.busy":"2022-03-24T10:35:48.134367Z","iopub.execute_input":"2022-03-24T10:35:48.134669Z","iopub.status.idle":"2022-03-24T10:35:48.153493Z","shell.execute_reply.started":"2022-03-24T10:35:48.134601Z","shell.execute_reply":"2022-03-24T10:35:48.152637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata_train.head()","metadata":{"_uuid":"f2359163fe48f20610ed7c426efe94d7239252b7","execution":{"iopub.status.busy":"2022-03-24T10:35:48.154782Z","iopub.execute_input":"2022-03-24T10:35:48.155168Z","iopub.status.idle":"2022-03-24T10:35:48.169615Z","shell.execute_reply.started":"2022-03-24T10:35:48.154997Z","shell.execute_reply":"2022-03-24T10:35:48.168787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Import plotting libraries and create some basic plots.","metadata":{"_uuid":"6f5a84750f1ca51a4fca03e4d0ec03cfaa2bc2dd"}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\n\n#plt.hist(metadata_train['target'])\nsns.countplot(metadata_train['target'])","metadata":{"_uuid":"8aaad8abc0bc4e95ab5c592cd3605994b3c5d1f0","execution":{"iopub.status.busy":"2022-03-24T10:35:48.170684Z","iopub.execute_input":"2022-03-24T10:35:48.171141Z","iopub.status.idle":"2022-03-24T10:35:49.073304Z","shell.execute_reply.started":"2022-03-24T10:35:48.171095Z","shell.execute_reply":"2022-03-24T10:35:49.072413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This is a plot of the target values. As expected, a faulty power line is a kind of rare event. Let's visualize some negative and positive (faulty) signals.","metadata":{"_uuid":"d07f9ed35254ca6577def6c94de901129617e1f9"}},{"cell_type":"code","source":"fig = plt.figure(figsize=(10,8))\n\nplt.subplot(431 )    #画一个4*3的画布，竖着4个图横着3个图。431等效于4，3，1 。 这里开始画第一个图，就是下面那句\nplt.plot(subset_train['5'])    # 第一个图的内容是subset_train数据集里面列标题是“5”的那一列。\nplt.subplot(432)\nplt.boxplot(subset_train['5'])\nplt.subplot(433)\nplt.hist(subset_train['5'])\n    \nplt.subplot(434)\nplt.plot(subset_train['1'])\nplt.subplot(435)\nplt.boxplot(subset_train['1'])\nplt.subplot(436)\nplt.hist(subset_train['1'])\n\nplt.subplot(437)\nplt.plot(subset_train['3'])\nplt.subplot(438)\nplt.boxplot(subset_train['3'])\nplt.subplot(439)\nplt.hist(subset_train['3'])\n\nplt.subplot(4,3,10)\nplt.plot(subset_train['4'])\nplt.subplot(4,3,11)\nplt.boxplot(subset_train['4'])\nplt.subplot(4,3,12)\nplt.hist(subset_train['4'])","metadata":{"_uuid":"ab20ac984b83a05d25eaf79f49dcf981c35356ed","execution":{"iopub.status.busy":"2022-03-24T10:35:49.074857Z","iopub.execute_input":"2022-03-24T10:35:49.075348Z","iopub.status.idle":"2022-03-24T10:35:51.924226Z","shell.execute_reply.started":"2022-03-24T10:35:49.075296Z","shell.execute_reply":"2022-03-24T10:35:51.923424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(22,10))\n\nplt.subplot(361 )    #画一个4*3的画布，竖着4个图横着3个图。431等效于4，3，1 。 这里开始画第一个图，就是下面那句\nplt.plot(subset_train['0'])    # 第一个图的内容是subset_train数据集里面列标题是“5”的那一列。\nplt.subplot(362)\nplt.boxplot(subset_train['0'])\nplt.subplot(363)\nplt.hist(subset_train['0'])\n    \nplt.subplot(364)\nplt.plot(subset_train['1'])\nplt.subplot(365)\nplt.boxplot(subset_train['1'])\nplt.subplot(366)\nplt.hist(subset_train['1'])\n\nplt.subplot(367)\nplt.plot(subset_train['2'])\nplt.subplot(368)\nplt.boxplot(subset_train['2'])\nplt.subplot(369)\nplt.hist(subset_train['2'])\n\nplt.subplot(3, 6 ,10)\nplt.plot(subset_train['3'])\nplt.subplot(3,6,11)\nplt.boxplot(subset_train['3'])\nplt.subplot(3,6,12)\nplt.hist(subset_train['3'])\n\nplt.subplot(3,6 ,13)\nplt.plot(subset_train['4'])\nplt.subplot(3,6,14)\nplt.boxplot(subset_train['4'])\nplt.subplot(3,6,15)\nplt.hist(subset_train['4'])\n\nplt.subplot(3,6 ,16)\nplt.plot(subset_train['5'])\nplt.subplot(3,6,17)\nplt.boxplot(subset_train['5'])\nplt.subplot(3,6,18)\nplt.hist(subset_train['5'])","metadata":{"execution":{"iopub.status.busy":"2022-03-24T10:35:51.928595Z","iopub.execute_input":"2022-03-24T10:35:51.930845Z","iopub.status.idle":"2022-03-24T10:35:55.675514Z","shell.execute_reply.started":"2022-03-24T10:35:51.930783Z","shell.execute_reply":"2022-03-24T10:35:55.674888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"At least from these couple of plots, we notice that the faulty signals (last 2) have relatively more outliers than the non-faulty ones. We will analyze this further with more data.\n\nLet's separate the positive and negative signals for further analysis. I'm going to reduce the sample sizes to make sure we don't run out of memory limits.","metadata":{"_uuid":"28b29bfcc7af20c19e15cec6c78afb59449f5fbc"}},{"cell_type":"code","source":"import numpy as np\nS_decimation = subset_train.iloc[0:25000:8,:]\n# Temporarily reduce data size to build the pipeline\nsmall_subset_train = S_decimation\nsmall_subset_train = small_subset_train.transpose()\nsmall_subset_train.index = small_subset_train.index.astype(np.int32)\ntrain_dataset = metadata_train.join(small_subset_train, how='right')\n\n# Uncomment the following to train on the full dataset\n#subset_train = subset_train.transpose()\n#subset_train.index = subset_train.index.astype(np.int32)\n#train_dataset = metadata_train.join(subset_train, how='right')","metadata":{"_uuid":"a5fb73acf543de684313260c9358acf252e9cbdf","execution":{"iopub.status.busy":"2022-03-24T10:35:55.678336Z","iopub.execute_input":"2022-03-24T10:35:55.678578Z","iopub.status.idle":"2022-03-24T10:35:55.923895Z","shell.execute_reply.started":"2022-03-24T10:35:55.678526Z","shell.execute_reply":"2022-03-24T10:35:55.923189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"positive_samples = train_dataset[train_dataset['target']==1]\npositive_samples = positive_samples.iloc[:,3:]\npositive_samples.info()","metadata":{"execution":{"iopub.status.busy":"2022-03-24T10:35:55.925282Z","iopub.execute_input":"2022-03-24T10:35:55.925783Z","iopub.status.idle":"2022-03-24T10:35:56.044197Z","shell.execute_reply.started":"2022-03-24T10:35:55.925568Z","shell.execute_reply":"2022-03-24T10:35:56.043446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"positive_samples.head()","metadata":{"_uuid":"24379b4d531df931d7c15e004d657425e9a7c7de","execution":{"iopub.status.busy":"2022-03-24T10:35:56.045968Z","iopub.execute_input":"2022-03-24T10:35:56.046380Z","iopub.status.idle":"2022-03-24T10:35:56.160638Z","shell.execute_reply.started":"2022-03-24T10:35:56.046219Z","shell.execute_reply":"2022-03-24T10:35:56.160047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"positive_samples.shape","metadata":{"execution":{"iopub.status.busy":"2022-03-24T10:35:56.161827Z","iopub.execute_input":"2022-03-24T10:35:56.162083Z","iopub.status.idle":"2022-03-24T10:35:56.169308Z","shell.execute_reply.started":"2022-03-24T10:35:56.162031Z","shell.execute_reply":"2022-03-24T10:35:56.168546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now let's visualize the positive (faulty) signals using a boxplot for several of them.","metadata":{"_uuid":"3a44931440815cb30e88b057c84e7213b356b025"}},{"cell_type":"code","source":"plt.figure(figsize=(10,4))\nplt.subplot(151)\nplt.boxplot(positive_samples.iloc[201,1:])\nplt.subplot(152)\nplt.boxplot(positive_samples.iloc[150,1:])\nplt.subplot(153)\nplt.boxplot(positive_samples.iloc[110,1:])\n\n","metadata":{"_uuid":"8b942a1a76c8475c9c6be3d84c8d8687ee16e3a8","execution":{"iopub.status.busy":"2022-03-24T10:35:56.170616Z","iopub.execute_input":"2022-03-24T10:35:56.171070Z","iopub.status.idle":"2022-03-24T10:35:56.690975Z","shell.execute_reply.started":"2022-03-24T10:35:56.171014Z","shell.execute_reply":"2022-03-24T10:35:56.690138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see that the data values differ a lot. Let's normalize the data first, this will also be needed for training some type of models later.","metadata":{"_uuid":"7a2d03a69c9b3d7f8f264abb020460b741aa8764"}},{"cell_type":"code","source":"# Normalize the data set\nfrom sklearn.preprocessing import StandardScaler\ny_train_pos = positive_samples.iloc[:, 0]\nX_train_pos = positive_samples.iloc[:, 1:]\nscaler = StandardScaler()         #通过去除均值并将其缩放为单位方差来标准化特征样本\nscaler.fit(X_train_pos.T)            #对\nX_train_pos = scaler.transform(X_train_pos.T).T","metadata":{"_uuid":"29e59231f9412c9d820b391293a880ebaa9f1689","execution":{"iopub.status.busy":"2022-03-24T10:35:56.695476Z","iopub.execute_input":"2022-03-24T10:35:56.697743Z","iopub.status.idle":"2022-03-24T10:35:56.941842Z","shell.execute_reply.started":"2022-03-24T10:35:56.695730Z","shell.execute_reply":"2022-03-24T10:35:56.941071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's visualize the boxplots again using this normalized data.","metadata":{"_uuid":"08dce41d4527ebe95444f46919d949684333b080"}},{"cell_type":"code","source":"X_train_pos","metadata":{"execution":{"iopub.status.busy":"2022-03-24T10:35:56.943041Z","iopub.execute_input":"2022-03-24T10:35:56.943476Z","iopub.status.idle":"2022-03-24T10:35:56.949944Z","shell.execute_reply.started":"2022-03-24T10:35:56.943427Z","shell.execute_reply":"2022-03-24T10:35:56.949159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,4))\nplt.subplot(151)\nplt.boxplot(X_train_pos[0,:])\nplt.subplot(152)\nplt.boxplot(X_train_pos[1,:])\nplt.subplot(153)\nplt.boxplot(X_train_pos[2,:])\nplt.subplot(154)\nplt.boxplot(X_train_pos[3,:])\nplt.subplot(155)\nplt.boxplot(X_train_pos[4,:])","metadata":{"_uuid":"07ddaf401fb44e18f67fd1e6a901baae3ca83d01","execution":{"iopub.status.busy":"2022-03-24T10:35:56.951721Z","iopub.execute_input":"2022-03-24T10:35:56.952454Z","iopub.status.idle":"2022-03-24T10:35:57.631464Z","shell.execute_reply.started":"2022-03-24T10:35:56.951970Z","shell.execute_reply":"2022-03-24T10:35:57.630591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Again we notice that there are a lot of outliers in the positive (faulty) signals.\n\nNow let's extract the negative (non-faulty) samples and visualize the same boxplots, and see if we can notice any apparent difference.","metadata":{"_uuid":"08d130bb14bf8a096fdd3810b7e298a4b56f98cb"}},{"cell_type":"code","source":"negative_samples = train_dataset[train_dataset['target']==0]\nnegative_samples = negative_samples.iloc[:,3:]\n\ny_train_neg = negative_samples.iloc[:, 0]\nX_train_neg = negative_samples.iloc[:, 1:]\nscaler.fit(X_train_neg.T)\nX_train_neg = scaler.transform(X_train_neg.T).T\n\nplt.figure(figsize=(10,4))\nplt.subplot(151)\nplt.boxplot(X_train_neg[0,:])\nplt.subplot(152)\nplt.boxplot(X_train_neg[1,:])\nplt.subplot(153)\nplt.boxplot(X_train_neg[2,:])\nplt.subplot(154)\nplt.boxplot(X_train_neg[3,:])\nplt.subplot(155)\nplt.boxplot(X_train_neg[4,:])","metadata":{"_uuid":"f7c99280ac588f718325b75a41fda68f62475966","execution":{"iopub.status.busy":"2022-03-24T10:35:57.632615Z","iopub.execute_input":"2022-03-24T10:35:57.633034Z","iopub.status.idle":"2022-03-24T10:35:58.967895Z","shell.execute_reply.started":"2022-03-24T10:35:57.632973Z","shell.execute_reply":"2022-03-24T10:35:58.967023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"negative_samples.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-24T10:35:58.972737Z","iopub.execute_input":"2022-03-24T10:35:58.973178Z","iopub.status.idle":"2022-03-24T10:35:59.107355Z","shell.execute_reply.started":"2022-03-24T10:35:58.972991Z","shell.execute_reply":"2022-03-24T10:35:59.106697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"negative_samples.shape\n","metadata":{"execution":{"iopub.status.busy":"2022-03-24T10:35:59.108506Z","iopub.execute_input":"2022-03-24T10:35:59.108799Z","iopub.status.idle":"2022-03-24T10:35:59.114715Z","shell.execute_reply.started":"2022-03-24T10:35:59.108753Z","shell.execute_reply":"2022-03-24T10:35:59.113765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"positive_samples.shape","metadata":{"execution":{"iopub.status.busy":"2022-03-24T10:35:59.116587Z","iopub.execute_input":"2022-03-24T10:35:59.117090Z","iopub.status.idle":"2022-03-24T10:35:59.123299Z","shell.execute_reply.started":"2022-03-24T10:35:59.116907Z","shell.execute_reply":"2022-03-24T10:35:59.122528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_neg","metadata":{"execution":{"iopub.status.busy":"2022-03-24T10:35:59.124451Z","iopub.execute_input":"2022-03-24T10:35:59.124866Z","iopub.status.idle":"2022-03-24T10:35:59.132004Z","shell.execute_reply.started":"2022-03-24T10:35:59.124819Z","shell.execute_reply":"2022-03-24T10:35:59.131151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The negative (non-faulty) signals have much fewer outliers, and their magnitudes also seem to be very low. Seems like the number of outliers could be a promising feature.\n\nNow let's create the test/train split for training a Conv1D model.","metadata":{"_uuid":"d9bc3cef7d6e57668a441a11715d05af5c0e3436"}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train_pos, X_valid_pos, y_train_pos, y_valid_pos = train_test_split(X_train_pos, y_train_pos, \n                                                                    test_size=0.2,\n                                                                    random_state = 0,\n                                                                    shuffle=True)\n\nX_train_neg, X_valid_neg, y_train_neg, y_valid_neg = train_test_split(X_train_neg, y_train_neg, \n                                                                    test_size=0.2,\n                                                                    random_state = 0,\n                                                                    shuffle=True)","metadata":{"_uuid":"3945a11048622a9a88e5335d62bfcb1be870bc3e","execution":{"iopub.status.busy":"2022-03-24T10:35:59.133304Z","iopub.execute_input":"2022-03-24T10:35:59.133777Z","iopub.status.idle":"2022-03-24T10:35:59.718667Z","shell.execute_reply.started":"2022-03-24T10:35:59.133726Z","shell.execute_reply":"2022-03-24T10:35:59.717933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_pos.shape, X_train_neg.shape","metadata":{"_uuid":"28373fafd9cf52acb37e49f7aafc8a6abf57630e","execution":{"iopub.status.busy":"2022-03-24T10:35:59.720126Z","iopub.execute_input":"2022-03-24T10:35:59.720633Z","iopub.status.idle":"2022-03-24T10:35:59.726623Z","shell.execute_reply.started":"2022-03-24T10:35:59.720420Z","shell.execute_reply":"2022-03-24T10:35:59.725683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we know, the positive samples are fewer, so we will only select a subset of negative samples for training.","metadata":{"_uuid":"4496e1c8d6c09d1c95a03768c75706339856e7e1"}},{"cell_type":"code","source":"# Combine positive and negative samples for training...\ndef combine_positive_and_negative_samples(pos_samples, neg_samples, y_pos, y_neg):\n    X_combined = np.concatenate((pos_samples, neg_samples)) \n                                                    # don't select all negative samples, to\n                                                    # keep the samples balanced\n    y_combined = np.concatenate((y_pos, y_neg))\n    #X_train_combined.shape, y_train_combined.shape\n    combined_samples = np.hstack((X_combined, y_combined.reshape(y_combined.shape[0],1)))\n    np.random.shuffle(combined_samples)\n    return combined_samples\n\n# Only use 500 negative samples, to create a balanced dataset with the positive samples...\ntrain_samples = combine_positive_and_negative_samples(X_train_pos, X_train_neg[:500, :], y_train_pos, y_train_neg[:500])\nX_train = train_samples[:,:-1]\ny_train = train_samples[:,-1]\nX_train.shape, y_train.shape","metadata":{"_uuid":"feb7e65bd79d359efaeda750ef519ecf6dca5d3c","execution":{"iopub.status.busy":"2022-03-24T10:35:59.728021Z","iopub.execute_input":"2022-03-24T10:35:59.728434Z","iopub.status.idle":"2022-03-24T10:35:59.767850Z","shell.execute_reply.started":"2022-03-24T10:35:59.728247Z","shell.execute_reply":"2022-03-24T10:35:59.767068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_samples.shape","metadata":{"execution":{"iopub.status.busy":"2022-03-24T10:35:59.769308Z","iopub.execute_input":"2022-03-24T10:35:59.769861Z","iopub.status.idle":"2022-03-24T10:35:59.775285Z","shell.execute_reply.started":"2022-03-24T10:35:59.769547Z","shell.execute_reply":"2022-03-24T10:35:59.774492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create the validation set\n#X_valid_combined = np.concatenate((X_valid_pos, X_valid_neg[:500,:])) # don't select all negative samples, to\n                                                  # keep the samples balanced\n#y_valid_combined = np.concatenate((y_valid_pos, y_valid_neg[:500]))\n#X_valid_combined.shape, y_valid_combined.shape\n#validation_samples = np.hstack((X_valid_combined, y_valid_combined.reshape(y_valid_combined.shape[0],1)))\n#np.random.shuffle(validation_samples)\n\nvalidation_samples = combine_positive_and_negative_samples(X_valid_pos, X_valid_neg[:500,:], y_valid_pos, y_valid_neg[:500])\nX_valid = validation_samples[:,:-1]\ny_valid = validation_samples[:,-1]\nX_valid.shape, y_valid.shape","metadata":{"_uuid":"1e2248e93cb07a9c0fb2466099283dfc9bdfb7bb","execution":{"iopub.status.busy":"2022-03-24T10:35:59.776795Z","iopub.execute_input":"2022-03-24T10:35:59.777417Z","iopub.status.idle":"2022-03-24T10:35:59.806485Z","shell.execute_reply.started":"2022-03-24T10:35:59.777363Z","shell.execute_reply":"2022-03-24T10:35:59.805678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape, X_valid.shape","metadata":{"execution":{"iopub.status.busy":"2022-03-24T10:35:59.807919Z","iopub.execute_input":"2022-03-24T10:35:59.808417Z","iopub.status.idle":"2022-03-24T10:35:59.814002Z","shell.execute_reply.started":"2022-03-24T10:35:59.808150Z","shell.execute_reply":"2022-03-24T10:35:59.813172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"A 1-D ConvNet would be an interesting model to try out on this signal. Earlier we saw that there are a lot of outliers in fauty signals. Since the actual signal value differs at different times, the outliers are relative to this mean signal value. A 1-D ConvNet can analyze the signal in various windows of increasing lengths and create high-level features out of that to classify on.","metadata":{"_uuid":"4383b24325024d54a179ee1a064ab3d2b94fc336"}},{"cell_type":"code","source":"# Reshape training and validation data for keras input layer\nX_train = X_train.reshape(-1,1,3125, 1)\nX_valid = X_valid.reshape(-1,1,3125, 1)\n\nX_train.shape, X_valid.shape, y_train.shape, y_valid.shape\n#print(X_train)","metadata":{"_uuid":"dc8c83b24a0834051a811a1c1bb494397470d40b","execution":{"iopub.status.busy":"2022-03-24T10:35:59.815423Z","iopub.execute_input":"2022-03-24T10:35:59.816009Z","iopub.status.idle":"2022-03-24T10:35:59.823127Z","shell.execute_reply.started":"2022-03-24T10:35:59.815958Z","shell.execute_reply":"2022-03-24T10:35:59.822261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_valid = X_valid.astype(np.float32)\ny_valid = y_valid.astype(np.float32)\nX_train = X_train.astype(np.float32)\ny_train = y_train.astype(np.float32)\nX_train.dtype","metadata":{"execution":{"iopub.status.busy":"2022-03-24T10:35:59.824649Z","iopub.execute_input":"2022-03-24T10:35:59.825486Z","iopub.status.idle":"2022-03-24T10:35:59.837512Z","shell.execute_reply.started":"2022-03-24T10:35:59.825242Z","shell.execute_reply":"2022-03-24T10:35:59.836765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.save('X_valid.npy',X_valid)\nnp.save('y_valid.npy',y_valid)\nnp.save('X_train.npy',X_train)\nnp.save('y_train.npy',y_train)","metadata":{"execution":{"iopub.status.busy":"2022-03-24T10:35:59.839326Z","iopub.execute_input":"2022-03-24T10:35:59.839775Z","iopub.status.idle":"2022-03-24T10:35:59.857843Z","shell.execute_reply.started":"2022-03-24T10:35:59.839729Z","shell.execute_reply":"2022-03-24T10:35:59.857289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nX_valid = np.load('X_valid.npy')\ny_valid = np.load('y_valid.npy')\nX_train = np.load('X_train.npy')\ny_train = np.load('y_train.npy')","metadata":{"execution":{"iopub.status.busy":"2022-03-24T10:35:59.859344Z","iopub.execute_input":"2022-03-24T10:35:59.859795Z","iopub.status.idle":"2022-03-24T10:35:59.875436Z","shell.execute_reply.started":"2022-03-24T10:35:59.859596Z","shell.execute_reply":"2022-03-24T10:35:59.874900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.mean(X_train, axis = 0), np.max(X_train, axis = 0), np.min(X_train, axis = 0), np.std(X_train, axis = 0)","metadata":{"execution":{"iopub.status.busy":"2022-03-24T10:35:59.876739Z","iopub.execute_input":"2022-03-24T10:35:59.877164Z","iopub.status.idle":"2022-03-24T10:35:59.894844Z","shell.execute_reply.started":"2022-03-24T10:35:59.877119Z","shell.execute_reply":"2022-03-24T10:35:59.894023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.sum(y_valid), np.sum(y_train)","metadata":{"execution":{"iopub.status.busy":"2022-03-24T10:35:59.896014Z","iopub.execute_input":"2022-03-24T10:35:59.896452Z","iopub.status.idle":"2022-03-24T10:35:59.901552Z","shell.execute_reply.started":"2022-03-24T10:35:59.896407Z","shell.execute_reply":"2022-03-24T10:35:59.900751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_normalize(data):\n    mu = np.mean(data,axis=0)\n    std = np.std(data,axis=0)\n    return (data - mu)/std","metadata":{"execution":{"iopub.status.busy":"2022-03-24T10:35:59.902921Z","iopub.execute_input":"2022-03-24T10:35:59.903507Z","iopub.status.idle":"2022-03-24T10:35:59.907962Z","shell.execute_reply.started":"2022-03-24T10:35:59.903457Z","shell.execute_reply":"2022-03-24T10:35:59.907190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"X_valid = feature_normalize(X_valid)\nX_train = feature_normalize(X_train)","metadata":{}},{"cell_type":"code","source":"import torch\nimport torchvision\nfrom torchvision import datasets, transforms\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torch.utils.data as Data\nprint(torch.__version__)","metadata":{"execution":{"iopub.status.busy":"2022-03-24T10:35:59.909226Z","iopub.execute_input":"2022-03-24T10:35:59.909704Z","iopub.status.idle":"2022-03-24T10:36:00.623557Z","shell.execute_reply.started":"2022-03-24T10:35:59.909639Z","shell.execute_reply":"2022-03-24T10:36:00.622763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class torch_Dataset(Data.Dataset): # 需要继承 data.Dataset\n    def __init__(self, x, y):\n        self.x = torch.from_numpy(x)\n        self.y = torch.from_numpy(y)\n    def __getitem__(self, index):\n        data = (self.x[index], self.y[index])\n        return data\n    def __len__(self):\n        return len(self.y)","metadata":{"execution":{"iopub.status.busy":"2022-03-24T10:36:00.625408Z","iopub.execute_input":"2022-03-24T10:36:00.625822Z","iopub.status.idle":"2022-03-24T10:36:00.632722Z","shell.execute_reply.started":"2022-03-24T10:36:00.625650Z","shell.execute_reply":"2022-03-24T10:36:00.630199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def training_loader(train_data, batch_size, shuffle):\n    return torch.utils.data.DataLoader(train_data, batch_size, shuffle)\n","metadata":{"execution":{"iopub.status.busy":"2022-03-24T10:36:00.633987Z","iopub.execute_input":"2022-03-24T10:36:00.634448Z","iopub.status.idle":"2022-03-24T10:36:00.642396Z","shell.execute_reply.started":"2022-03-24T10:36:00.634399Z","shell.execute_reply":"2022-03-24T10:36:00.641525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Train_dataset = torch_Dataset(X_train, y_train)\ntest_dataset = torch_Dataset(X_valid, y_valid)\ntrain_loader = training_loader(Train_dataset, batch_size=1, shuffle=True)\ntest_loader = training_loader(test_dataset, batch_size=1, shuffle=True)  \n","metadata":{"execution":{"iopub.status.busy":"2022-03-24T10:36:00.643690Z","iopub.execute_input":"2022-03-24T10:36:00.644284Z","iopub.status.idle":"2022-03-24T10:36:00.655865Z","shell.execute_reply.started":"2022-03-24T10:36:00.644222Z","shell.execute_reply":"2022-03-24T10:36:00.655055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CNN(nn.Module):\n    def __init__(self):\n        super(CNN,self).__init__()\n        self.conv1 = nn.Conv2d(1, 32, kernel_size=(16,1), padding=0, stride=(2,1))\n        self.bn1 = nn.BatchNorm2d(32)\n        self.rl1 =  nn.ReLU()\n        self.pool1 = nn.MaxPool2d((2,1))\n        self.do1 =    nn.Dropout(0.2)\n        self.conv2 = nn.Conv2d(32, 128, kernel_size=(16,1), padding=0, stride=(2,1))\n        self.bn2 = nn.BatchNorm2d(128)\n        self.rl2 =  nn.ReLU()\n        self.pool2 = nn.MaxPool2d((2,1))\n        self.do2 =    nn.Dropout(0.2)\n        self.conv3 = nn.Conv2d(128, 256, kernel_size=(10,1), padding=0, stride=(2,1))\n        self.bn3 = nn.BatchNorm2d(256)\n        self.rl3 =  nn.ReLU()\n        self.pool3 = nn.MaxPool2d((2,1))\n        self.do3 =    nn.Dropout(0.2)\n        self.conv6 = nn.Conv2d(256, 64, kernel_size=(10,1), padding=0, stride=(2,1))\n        self.rl6 =  nn.ReLU()\n        \n        self.fc1   = nn.Linear(1152, 512)\n        self.rl7 =  nn.ReLU()\n        self.do7 =    nn.Dropout(0.2)\n        self.fc2   = nn.Linear(512, 128)\n        self.rl8 =  nn.ReLU()\n        self.do8 =    nn.Dropout(0.2)\n        self.fc3   = nn.Linear(128, 1)\n      \n            \n        \n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = self.rl1(x)\n        x = self.pool1(x)\n        x = self.do1(x)\n        x = self.conv2(x)\n        x = self.bn2(x)\n        x = self.rl2(x)\n        x = self.pool2(x)\n        x = self.do2(x)\n        x = self.conv3(x)\n        x = self.bn3(x)\n        x = self.rl3(x)\n        x = self.pool3(x)\n        x = self.do3(x)\n        x = self.conv6(x)\n        x = self.rl6(x)\n        x = x.view(-1,1152)\n   \n        x = self.fc1(x)\n        x = self.rl7(x)\n        x = self.do7(x)\n        x = self.fc2(x)\n        x = self.rl8(x)\n        x = self.do8(x)\n        x = self.fc3(x)\n       # print(x.shape)\n        x = torch.sigmoid(x)\n        \n        return x\n","metadata":{"execution":{"iopub.status.busy":"2022-03-24T10:36:00.657224Z","iopub.execute_input":"2022-03-24T10:36:00.657782Z","iopub.status.idle":"2022-03-24T10:36:00.671487Z","shell.execute_reply.started":"2022-03-24T10:36:00.657735Z","shell.execute_reply":"2022-03-24T10:36:00.670623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"    if torch.cuda.is_available():\n        device = torch.device( \"cuda\")\n    else:\n        device = torch.device( \"cpu\")\n    print(f\"training on {device} device.\")\n    model = CNN().to(device)\n    optimizer = optim.Adam(model.parameters(), lr=0.0001) #1e-2)\n    loss_fn = nn.BCELoss()\n\n    print(model)","metadata":{"execution":{"iopub.status.busy":"2022-03-24T10:36:00.672758Z","iopub.execute_input":"2022-03-24T10:36:00.673311Z","iopub.status.idle":"2022-03-24T10:36:03.874721Z","shell.execute_reply.started":"2022-03-24T10:36:00.673265Z","shell.execute_reply":"2022-03-24T10:36:03.873904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def training_loop(n_epochs, optimizer, model, loss_fn, train_loader, dev):\n    for epoch in range(1, n_epochs + 1):\n        loss_train = 0.0\n        for imgs, labels in train_loader:\n            imgs = imgs.to(device=dev)\n            #imgs = imgs.float() \n            labels = labels.to(device=dev)\n            outputs = model(imgs)\n            #outputs = outputs.squeeze(-1)       \n            #print(\"train output size \",outputs.size())\n           # print(\"label output size \",labels.size())\n            #print(outputs)\n            optimizer.zero_grad()\n            loss = loss_fn(outputs, labels)\n            loss.backward()\n            optimizer.step()\n            loss_train += loss.item()\n        if epoch == 1 or epoch % 2 == 0:\n            print('{0} Epoch {1:3d}, Training loss {2:.6f}'.format(\n                datetime.datetime.now(), epoch,\n                loss_train / len(train_loader)))\n    return","metadata":{"execution":{"iopub.status.busy":"2022-03-24T10:36:03.876318Z","iopub.execute_input":"2022-03-24T10:36:03.876842Z","iopub.status.idle":"2022-03-24T10:36:03.884448Z","shell.execute_reply.started":"2022-03-24T10:36:03.876793Z","shell.execute_reply":"2022-03-24T10:36:03.883766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"    import warnings\n    import datetime\n    warnings.filterwarnings('ignore') \n    training_loop(\n        n_epochs = 20, #100,\n        optimizer = optimizer,\n        model = model,\n        loss_fn = loss_fn,\n        train_loader = train_loader,\n        dev = device\n    )","metadata":{"execution":{"iopub.status.busy":"2022-03-24T10:36:03.885571Z","iopub.execute_input":"2022-03-24T10:36:03.886089Z","iopub.status.idle":"2022-03-24T10:37:14.437452Z","shell.execute_reply.started":"2022-03-24T10:36:03.885845Z","shell.execute_reply":"2022-03-24T10:37:14.436698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nprint('Trained model written to ', 'torch_chkp.pt')\ntorch.save(model.state_dict(), 'torch_chkp.pt')\nloaded_model = CNN()  # <1>\nloaded_model.load_state_dict(torch.load('torch_chkp.pt'))","metadata":{"execution":{"iopub.status.busy":"2022-03-24T10:37:14.438588Z","iopub.execute_input":"2022-03-24T10:37:14.439026Z","iopub.status.idle":"2022-03-24T10:37:14.478559Z","shell.execute_reply.started":"2022-03-24T10:37:14.438978Z","shell.execute_reply":"2022-03-24T10:37:14.477728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def validate(model, train_loader, val_loader, dev):\n    accdict = {}\n    for name, loader in [(\"train dataset\", train_loader), (\"test dataset  \", val_loader)]:\n        correct = 0\n        total = 0\n        with torch.no_grad():\n            for imgs, labels in loader:\n                imgs = imgs.float() \n                imgs = imgs.to(device=dev)\n                labels = labels.to(device=dev)\n                outputs = model(imgs)\n                \n                predicted = torch.max(outputs) # <1>\n                #print(predicted)\n                if(predicted>0.5):\n                   #print(predicted)\n                    falt_detected =1\n                else:\n                    falt_detected =0\n                total += labels.shape[0]\n                correct += int((falt_detected == labels).sum())\n                #print(\"predict value:\", falt_detected, \"real value:\" , labels)\n                \n        print(\"Accuracy {0}: {1:.2f}(%)\".format(name , 100*(correct/total)))\n        accdict[name] = correct / total\n    return accdict","metadata":{"execution":{"iopub.status.busy":"2022-03-24T10:37:14.480010Z","iopub.execute_input":"2022-03-24T10:37:14.480437Z","iopub.status.idle":"2022-03-24T10:37:14.487587Z","shell.execute_reply.started":"2022-03-24T10:37:14.480252Z","shell.execute_reply":"2022-03-24T10:37:14.486821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fit(model, train_loader, val_loader, dev):\n    accdict = {}\n    for name, loader in [(\"train dataset\", train_loader), (\"test dataset  \", val_loader)]:\n        correct = 0\n        total = 0\n        with torch.no_grad():\n            for imgs, labels in loader:\n                imgs = imgs.float() \n                imgs = imgs.to(device=dev)\n                labels = labels.to(device=dev)\n                outputs = model(imgs)\n                \n                predicted = torch.max(outputs) # <1>\n                #print(predicted)\n  \n    return accdict","metadata":{"execution":{"iopub.status.busy":"2022-03-24T10:37:14.488830Z","iopub.execute_input":"2022-03-24T10:37:14.489380Z","iopub.status.idle":"2022-03-24T10:37:14.498924Z","shell.execute_reply.started":"2022-03-24T10:37:14.489330Z","shell.execute_reply":"2022-03-24T10:37:14.498269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validate(model, train_loader, test_loader, device)","metadata":{"execution":{"iopub.status.busy":"2022-03-24T10:41:47.699567Z","iopub.execute_input":"2022-03-24T10:41:47.699858Z","iopub.status.idle":"2022-03-24T10:41:49.550922Z","shell.execute_reply.started":"2022-03-24T10:41:47.699808Z","shell.execute_reply":"2022-03-24T10:41:49.550328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\nimport collections\nloaded_model = CNN()  # <1>\nloaded_model.load_state_dict(torch.load('torch_chkp.pt'))\nall_acc_dict = collections.OrderedDict()\ntime_start = time.time()\nall_acc_dict[\"baseline\"] = validate(model, train_loader, test_loader, device)\ntime_end = time.time()\ntimetotal = time_end - time_start\nrunTotal = len(train_loader)\nfps = float(runTotal / timetotal)\nprint(\"FPS=%.2f, total frames = %.0f , time=%.4f seconds\" %(fps,runTotal, timetotal))","metadata":{"execution":{"iopub.status.busy":"2022-03-24T10:37:16.514778Z","iopub.execute_input":"2022-03-24T10:37:16.515209Z","iopub.status.idle":"2022-03-24T10:37:18.410628Z","shell.execute_reply.started":"2022-03-24T10:37:16.515159Z","shell.execute_reply":"2022-03-24T10:37:18.409082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}