{"cells":[{"metadata":{},"cell_type":"markdown","source":"# VSB Power Grid Fault Detection","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.display import Image\nImage(url='https://upload.wikimedia.org/wikipedia/commons/thumb/e/e0/Three_Phase_Electric_Power_Transmission.jpg/1200px-Three_Phase_Electric_Power_Transmission.jpg')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Data Source: https://www.kaggle.com/c/vsb-power-line-fault-detection\n\nUseful read: https://en.wikipedia.org/wiki/Three-phase_electric_power\n","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Importing Libraries","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport pyarrow.parquet as pq #reading parquet files \nimport matplotlib.pyplot as plt\nimport os\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Taking first 2000 rows (due to computational limitations) for EDA and visualization","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"INIT_DIR = '../input'\nSIZE = 2001","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pq.read_pandas(os.path.join(INIT_DIR, 'vsb-power-line-fault-detection/train.parquet'), columns=[str(i) for i in range(SIZE)]).to_pandas()\nmetadata = pd.read_csv('../input/vsb-power-line-fault-detection/metadata_train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"metadata.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"metadata.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_metadata = metadata[:SIZE]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_metadata.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Observation:\n\nAs each column represent a signal, it will be  better if we transpose the dataframe","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Transposing the dataframe as each column represents one data point.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head(2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Adding signal id to the main data frame","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train['signal_id'] = list(train_metadata['signal_id'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head(2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Merging Metadata and Signal Data based on signal_id","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.merge(train_metadata, on='signal_id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head(2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Checking for null values in the dataframe","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train.isnull().sum().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Observation: \n\nThere is no null values","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Plotting count vs target plots to check data imbalance","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 4))\nsns.countplot(x=\"target\", data=train, ax=ax1)\nsns.countplot(x=\"target\", data=train, hue=\"phase\", ax=ax2);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Observation:\n\nData is highly imbalances as the target with value 1 are much less than the target with value 0","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Percentage of positive and negative target values","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://www.w3resource.com/graphics/matplotlib/piechart/matplotlib-piechart-exercise-2.php\nplt.rcParams[\"figure.figsize\"] = (40,6.5)\ndata = train['target'].value_counts()\nlabels = ['Target 0', 'Target 1']\ncolors = [\"#1f77b4\", \"#ff7f0e\"]\ntitle = 'Count of signals distributed by phase'\nexplodes = [0, 0.1]\nplt.pie(data,explode=explodes, labels=labels, colors=colors, shadow=True, startangle=20, autopct='%.1f%%')\nplt.title(title, bbox={'facecolor':'0.8', 'pad':5})\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target_count = train.target.value_counts()\nprint(\"negative(target=0) target: {}\".format(target_count[0]))\nprint(\"positive(target=1) target: {}\".format(target_count[1]))\nprint(\"positive data {:.3}%\".format((target_count[1]/(target_count[0]+target_count[1]))*100))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Observation:\n\nData is imbalanced and the faulty signals are only 6.3% of the total signals","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Checking if there are different values of target in the different phase of same signal","execution_count":null},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"train[['id_measurement', 'phase']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target_mismatch = train[[\"id_measurement\", \"target\"]].groupby([\"id_measurement\"]).sum().query(\"target != 3 & target != 0\")\nprint(\"Target values not all postive or negative for same signal: {}\".format(target_mismatch.shape[0]))\ntarget_mismatch","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Checking target for id _measurement==67 where target value is different in different phase","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train[train['id_measurement'] == 67]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Observation:\n\nTarget values can be different for same signal in different phases","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Finding the Unique values of id_measurement in our dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"id_measurement have {} unique values\".format(train.id_measurement.nunique()))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Observation:\n\nThe unique value of the id_measurement is as expected : (total signals) / 3 , as there are three phases of each signal","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Basic description of the id_measurement column ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train.id_measurement.value_counts().describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Observation:\n\nValues are all as expected.\n\nMax count and min count of each signal is 3, as there are three phases of each signal\n\ncount is total/3, as each signal is having three phases.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Printing unique values of phase column","execution_count":null},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"print(\"phase have {} unique values {} in train\".format(len(train.phase.unique()),train.phase.unique()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(train['phase']);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://www.w3resource.com/graphics/matplotlib/piechart/matplotlib-piechart-exercise-2.php\ndata = train['phase'].value_counts()\nlabels = ['Phase 0', 'Phase 1', 'Phase 3']\ncolors = [\"#1f77b4\", \"#ff7f0e\", \"#2ca02c\"]\ntitle = 'Count of signals distributed by phase'\nplt.pie(data, labels=labels, colors=colors, shadow=True, startangle=90, autopct='%.1f%%')\nplt.title(title, bbox={'facecolor':'0.8', 'pad':5})\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Observation:\nPhase columns is having only 3 values 1,2,3 for each signal as there are three phases of each signal","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Plotting 2d plots using t-SNE using different values of perplexity and learning rate\n\n#### Plotting the t-SNE plots only for 1/4th of the points due to computational limitations","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"#### 1. Using perplexity: 30 and learning rate: 200","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.manifold import TSNE\n\ntsne = TSNE(n_components=2, perplexity=30, learning_rate=200, random_state=42)\n\nX_embedding = tsne.fit_transform(train[:500])\ny = np.array(train['target'][:500])\n\nfor_tsne = np.hstack((X_embedding, y.reshape(-1,1)))\nfor_tsne_df = pd.DataFrame(data=for_tsne, columns=['Dimension_x','Dimension_y','Score'])\ncolors = {0:'red', 1:'blue', 2:'green'}\nplt.scatter(for_tsne_df['Dimension_x'], for_tsne_df['Dimension_y'], c=for_tsne_df['Score'].apply(lambda x: colors[x]))\nplt.show()\n\ndel(tsne)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### 2. Using perplexity: 50 and learning rate:200","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.manifold import TSNE\n\ntsne = TSNE(n_components=2, perplexity=50, learning_rate=200, random_state=42)\n\nX_embedding = tsne.fit_transform(train[:500])\ny = np.array(train['target'][:500])\n\nfor_tsne = np.hstack((X_embedding, y.reshape(-1,1)))\nfor_tsne_df = pd.DataFrame(data=for_tsne, columns=['Dimension_x','Dimension_y','Score'])\ncolors = {0:'red', 1:'blue', 2:'green'}\nplt.scatter(for_tsne_df['Dimension_x'], for_tsne_df['Dimension_y'], c=for_tsne_df['Score'].apply(lambda x: colors[x]))\nplt.show()\n\ndel(tsne)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### 3. Using Perplexity:100 and learning rate: 150","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.manifold import TSNE\n\ntsne = TSNE(n_components=2, perplexity=100, learning_rate=150, random_state=42)\n\nX_embedding = tsne.fit_transform(train[:500])\ny = np.array(train['target'][:500])\n\nfor_tsne = np.hstack((X_embedding, y.reshape(-1,1)))\nfor_tsne_df = pd.DataFrame(data=for_tsne, columns=['Dimension_x','Dimension_y','Score'])\ncolors = {0:'red', 1:'blue', 2:'green'}\nplt.scatter(for_tsne_df['Dimension_x'], for_tsne_df['Dimension_y'], c=for_tsne_df['Score'].apply(lambda x: colors[x]))\nplt.show()\n\ndel(tsne)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Observation:\n\nThe points are not well seperated in 2-dimensions as observed by these t-sne plots","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Plotting signals ","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"#### Plotting normal Signal","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#signal with target 0 (normal signal)\ntrain.loc[1]['target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(24, 8))\nplt.plot((train.loc[1].values), alpha=0.7);\nplt.ylim([-100, 100])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Plotting Faulty signal","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#signal with target 1 (Faulty Signal)\ntrain.loc[201]['target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(24, 8))\nplt.plot((train.loc[201].values), alpha=0.7);\nplt.ylim([-100, 100])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Observation:\n\nFaulty signal has more noise","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Plotting all three phases of a signal","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"#### Plotting all three phases of a normal signal","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#signal with target 0 (Normal Signal)\ntrain.loc[0:2][['target', 'id_measurement']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(24, 8))\nplt.plot((train.loc[0].values), alpha=0.7);\nplt.plot((train.loc[1].values), alpha=0.7);\nplt.plot((train.loc[2].values), alpha=0.7);\nplt.ylim([-100, 100])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Plotting all three phases of a faulty signal","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#signal with target 1 (Faulty Signal)\ntrain.loc[3:5][['target', 'id_measurement']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(24, 8))\nplt.plot((train.loc[3].values), alpha=0.7);\nplt.plot((train.loc[4].values), alpha=0.7);\nplt.plot((train.loc[5].values), alpha=0.7);\nplt.ylim([-100, 100])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Observation:\n\nFaulty signal have more noise than the normal signal. \nHence, noise can be a very useful feature for fault detection","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Flatiron\n\n#### reference: https://www.kaggle.com/miklgr500/flatiron","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"The idea of flatiron is similar to High Pass Filter. It allows high frequency to pass. It can be useful for noise extraction","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def flatiron(x, alpha=50, beta=1):\n    new_x = np.zeros_like(x)\n    zero = x[0]\n    for i in range(1, len(x)):\n        zero = zero*(alpha-beta)/alpha + beta*x[i]/alpha\n        new_x[i] =  x[i] - zero\n    return new_x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Plotting normal signal with flattened normal signal","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#Flattening a Normal signal\nnormal_sample_filt =  [None] * 3\nnormal_sample_filt[0] = flatiron(train.loc[0].values)\nnormal_sample_filt[1] = flatiron(train.loc[1].values)\nnormal_sample_filt[2] = flatiron(train.loc[2].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"normal_sample_filt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Code to plot faulty signal with flattened faulty signal\nf, ax = plt.subplots(1, 2, figsize=(24, 8))\n\nax[0].plot((train.loc[0].values), alpha=0.7);\nax[0].plot((train.loc[1].values), alpha=0.7);\nax[0].plot((train.loc[2].values), alpha=0.7);\nax[0].set_title('Normal signal')\nax[0].set_ylim([-100, 100])\n\nax[1].plot((normal_sample_filt)[0], alpha=0.7);\nax[1].plot((normal_sample_filt)[1], alpha=0.7);\nax[1].plot((normal_sample_filt)[2], alpha=0.7);\nax[1].set_title('filtered Normal signal')\nax[1].set_ylim([-100, 100])\n\ndel(normal_sample_filt)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Observation:\n\nWe are able to flatten the signal and are able to visualize the noise in the signal more easily","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"#### Plotting faulty signal with flattened faulty signal","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#Flattening a Faulty signal\nfault_sample_filt =  [None] * 3\nfault_sample_filt[0] = flatiron(train.loc[3].values)\nfault_sample_filt[1] = flatiron(train.loc[4].values)\nfault_sample_filt[2] = flatiron(train.loc[5].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fault_sample_filt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"#Code to plot faulty signal with flattened faulty signal\nf, ax = plt.subplots(1, 2, figsize=(24, 8))\n\nax[0].plot((train.loc[3].values), alpha=0.7);\nax[0].plot((train.loc[4].values), alpha=0.7);\nax[0].plot((train.loc[5].values), alpha=0.7);\nax[0].set_title('fault signal')\nax[0].set_ylim([-100, 100])\n\nax[1].plot((fault_sample_filt)[0], alpha=0.7);\nax[1].plot((fault_sample_filt)[1], alpha=0.7);\nax[1].plot((fault_sample_filt)[2], alpha=0.7);\nax[1].set_title('filtered fault signal')\nax[1].set_ylim([-100, 100])\n\ndel(fault_sample_filt)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Observation:\n\nFaulty signal has more noise than normal signal","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}