{"cells":[{"metadata":{"trusted":true,"_uuid":"1bd271060eaf1b540d1076bbafd31aba4b159eb4"},"cell_type":"code","source":"import numpy as np\nfrom scipy import fftpack, signal\nimport plotly.offline as plt\nimport plotly.graph_objs as go\nimport pandas as pd\nimport seaborn as sns\nimport pyarrow.parquet as pq\nfrom tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b2086e507ece329eb2a8d63bdb63739e31ff7d9f"},"cell_type":"code","source":"plt.init_notebook_mode(connected=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"debccb7a7321f57cbb601d288b27d8255d68d47f"},"cell_type":"code","source":"#set_index makes using .loc very handy later on\ntrain_meta = (pd.read_csv('../input/metadata_train.csv')\n                .set_index(['id_measurement','phase'])\n             )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1c8ac9e5a3cdc5769b32d15100215f9c66638c2e"},"cell_type":"code","source":"train_meta.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"447d7846f4c3e9f50994efe5ba2b4552bcff00a3"},"cell_type":"markdown","source":"## Statistics of positive and negatives"},{"metadata":{"_uuid":"062ec2319bb79912d030fcc5a1aca1c9b3b8f765"},"cell_type":"markdown","source":"As noted by others, when things fails, usually all 3 phase fails\nbut sometimes its only 1 or 2 out of the 3 phases"},{"metadata":{"trusted":true,"_uuid":"f18300e51e7c0b9a26b8fa6b58e10fe7d8fbc0be"},"cell_type":"code","source":"tsum = train_meta.groupby(\"id_measurement\")[\"target\"].sum()\ntsum.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c908faa35bdb7d43fc6dc63300a80da5dbaef72b"},"cell_type":"code","source":"pos1_id_meas = tsum[tsum==1].index\npos2_id_meas = tsum[tsum==2].index\npos3_id_meas = tsum[tsum==3].index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8a7858264da90b44475d556259ea92e377d46bf5"},"cell_type":"code","source":"neg_id_meas = tsum[tsum==0].index[:200] #just pick 200 negative samples for now","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"40c93b1076be6bd1c4b6b6ec09ff745d54aef621"},"cell_type":"code","source":"train_meta_trimmed = train_meta.loc[pos1_id_meas | pos2_id_meas | pos3_id_meas | neg_id_meas]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce8602b3a0fd72c415aba4786c866ac7603c015a"},"cell_type":"code","source":"xs = pq.read_table('../input/train.parquet', columns=[str(i) for i in train_meta_trimmed['signal_id']]).to_pandas()\n#xs = pq.read_table('../input/train.parquet').to_pandas()\nprint((xs.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fbaafc12ed04ef8754e20b0ee537b31fd6c9cf76"},"cell_type":"code","source":"xs.columns = [int(c) for c in xs.columns]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a0835fdc384930f7510504261c7fa0cf1f7de796"},"cell_type":"markdown","source":"## Ploting some waveforms"},{"metadata":{"trusted":true,"_uuid":"6807ec6b3e466f3ff2f0ed30fc51fba99eb2f41a"},"cell_type":"code","source":"def plot_3phase(id_measurement):\n    df = train_meta.loc[id_measurement,:]\n    print(df)\n    sigs = [ xs.loc[:,i] for i in df['signal_id']]\n    data = [go.Scattergl( x=xs.index, y=sig) for sig in sigs]\n    plt.iplot(data)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c178d814f7199e69ab5aca22802b33bfd9d16aa7"},"cell_type":"markdown","source":"case where 1 out of 3 phase is positive\n\nFrankly, I can't spot the difference.\nThere are spikes in all 3 phase, and many of them appear at the same time across the 3 waveforms."},{"metadata":{"trusted":true,"_uuid":"2d6444a00544027a170e2df659b2c235314bd86b"},"cell_type":"code","source":"pos1_id_meas","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"ce91d5c4a18fc86a3516324db48590a906356f63"},"cell_type":"code","source":"plot_3phase(96)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"86351aed0b3d526eced98e41cd303242ea22a954"},"cell_type":"markdown","source":"case where 2 out of 3 phase is positive"},{"metadata":{"trusted":false,"_uuid":"b1f59df600936ad4103b57ab3741539903a6d10d"},"cell_type":"code","source":"pos2_id_meas","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"7081946a19d36893c9e728e8100fc8b2173ff3e9"},"cell_type":"code","source":"plot_3phase(67)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3d8f1147648af900752bc0b98c51fc0a605de90d"},"cell_type":"markdown","source":"case where 3 out of 3 phase is positive"},{"metadata":{"trusted":false,"_uuid":"2aa9d2ede752ed17f1350a744ecb143c87c052e0"},"cell_type":"code","source":"pos3_id_meas","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"6ceccb628c512bbaf7270415b5076682002bb95a"},"cell_type":"code","source":"plot_3phase(1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"32d8d7d11b05f0804ff21a84d3147f64cfe15e88"},"cell_type":"markdown","source":"case where all phases are negative"},{"metadata":{"trusted":false,"_uuid":"55acfe99e8ec8637f5d987cfd9b8679606536515"},"cell_type":"code","source":"neg_id_meas","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"92d861b7c0fefdeb9731b68795d798f242b2cf14"},"cell_type":"code","source":"plot_3phase(0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e8c1fa5424dc79b1464395d5b2d9eadf30d2d2ea"},"cell_type":"markdown","source":"## Allign waveforms"},{"metadata":{"trusted":false,"_uuid":"46f5193e705c4e336ad6ab7f1fa93f7ed49e3160"},"cell_type":"code","source":"def undo_phase(sig):\n    \"\"\"\n    find the phase of the 50Hxz component and undo it\n    \"\"\"\n    sig_fft = fftpack.fft(sig)\n    ang = np.angle(sig_fft[1])\n    shift = int((-ang-np.pi/2.)/(2.*np.pi)*len(sig))\n    sig_shifted = pd.concat( [sig.iloc[shift:] , sig.iloc[:shift]]).reset_index(drop=True)\n    return sig_shifted","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"cf604ac5a7ff2596d94a3d6ef3a1ebf55ce8f37d"},"cell_type":"code","source":"for aCol in tqdm(xs.columns):\n    xs[aCol] = undo_phase(xs[aCol])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1bb9f35fd602b6d824bbc1eba5eaa840dd765edd"},"cell_type":"markdown","source":"Example after allign"},{"metadata":{"trusted":false,"_uuid":"1e7595130d8dad899c4b22c6fb916e08e8f6974c"},"cell_type":"code","source":"xs_pos = xs[train_meta_trimmed[train_meta_trimmed['target']==1]['signal_id']]\nxs_neg = xs[train_meta_trimmed[train_meta_trimmed['target']==0]['signal_id']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"003eb08f68b4f273808864c35eb5b6eb85b53865"},"cell_type":"code","source":"xs_pos_mean = xs_pos.mean(axis=1)\nxs_pos_sd = xs_pos.std(axis=1)\n\nxs_neg_mean = xs_neg.mean(axis=1)\nxs_neg_sd = xs_neg.std(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"17cc059e81b28c111326488fdda1595b02493ee7"},"cell_type":"code","source":"pos_p1SD = go.Scattergl(\n    name='Pos+1SD',\n    x=xs.index,\n    y=xs_pos_mean + xs_pos_sd,\n    mode='lines',\n    marker=dict(color=\"#444\"),\n    line=dict(color='rgb(55, 0, 0)', width=1),\n    )\n\npos = go.Scatter(\n    name='Pos',\n    x=xs.index,\n    y=xs_pos_mean,\n    mode='lines',\n    line=dict(color='rgb(255, 0, 0)'),\n    )\n\npos_m1SD = go.Scatter(\n    name='Pos-1SD',\n    x=xs.index,\n    y=xs_pos_mean - xs_pos_sd,\n    marker=dict(color=\"#444\"),\n    line=dict(color='rgb(55, 0, 0)', width=1),\n    mode='lines')\n\nneg_p1SD = go.Scattergl(\n    name='Neg+1SD',\n    x=xs.index,\n    y=xs_neg_mean + xs_neg_sd,\n    mode='lines',\n    marker=dict(color=\"#444\"),\n    line=dict(color='rgb(0, 0, 55)', width=1),\n    )\n\nneg = go.Scatter(\n    name='Neg',\n    x=xs.index,\n    y=xs_neg_mean,\n    mode='lines',\n    line=dict(color='rgb(0, 0, 255)'),\n    )\n\nneg_m1SD = go.Scatter(\n    name='Neg-1SD',\n    x=xs.index,\n    y=xs_neg_mean - xs_neg_sd,\n    marker=dict(color=\"#444\"),\n    line=dict(color='rgb(0, 0, 55)', width=1),\n    mode='lines')\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"69f6572a5e0f6b2391b53e0a6d9566a52fc448cb"},"cell_type":"markdown","source":"Comapre positive and negative average waveform,\nthe average is similar.\n\nThe positves has slightly larger stdev"},{"metadata":{"trusted":false,"_uuid":"cf2c78b2fd14b2ca9a2ce2ab33f5e6231c0f4d31"},"cell_type":"code","source":"# data = [pos_p1SD, pos, pos_m1SD, neg_p1SD, neg, neg_m1SD] #this crash my computer","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"acdad5b00c5d59e4446fbb0e70b491a322fa0050"},"cell_type":"code","source":"data = [pos_p1SD, pos, pos_m1SD]\nplt.iplot(data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"c7361f58eb239107de7030a830eee512fecebc5b"},"cell_type":"code","source":"data = [neg_p1SD, neg, neg_m1SD]\nplt.iplot(data)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"648f0580cd00e7e94e1d691ab3d6c2447a406cef"},"cell_type":"markdown","source":"## Thoughts\n1) Like others has proposed, the overall waveform seems not indicative of partial dischage positives\n\n2) The spikes should be more relevent, but from the 1 out of 3 positives cases study, I cannot tell whats's so special about the positive one\n\nWhat do you think?"},{"metadata":{"trusted":false,"_uuid":"0ba609288ea10d86e4f145864eb1d9506c922369"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}