{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport os\nimport seaborn as sns\nimport pyarrow.parquet as pq #reading parquet files ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-01T14:39:07.547193Z","iopub.execute_input":"2021-07-01T14:39:07.548124Z","iopub.status.idle":"2021-07-01T14:39:08.436872Z","shell.execute_reply.started":"2021-07-01T14:39:07.548009Z","shell.execute_reply":"2021-07-01T14:39:08.435541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dataset upload\nINIT_DIR = '../input'\nSIZE = 2001\ntrain = pq.read_pandas(os.path.join(INIT_DIR, 'vsb-power-line-fault-detection/train.parquet'), columns=[str(i) for i in range(SIZE)]).to_pandas()\nmetadata = pd.read_csv('../input/vsb-power-line-fault-detection/metadata_train.csv')","metadata":{"execution":{"iopub.status.busy":"2021-07-01T14:39:18.359740Z","iopub.execute_input":"2021-07-01T14:39:18.360164Z","iopub.status.idle":"2021-07-01T14:39:41.238370Z","shell.execute_reply.started":"2021-07-01T14:39:18.360129Z","shell.execute_reply":"2021-07-01T14:39:41.236463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_metadata = metadata[:SIZE]\ntrain = train.T\ntrain['signal_id'] = list(train_metadata['signal_id'])\ntrain = train.merge(train_metadata, on='signal_id')","metadata":{"execution":{"iopub.status.busy":"2021-07-01T14:39:45.228905Z","iopub.execute_input":"2021-07-01T14:39:45.229306Z","iopub.status.idle":"2021-07-01T14:39:48.319410Z","shell.execute_reply.started":"2021-07-01T14:39:45.229274Z","shell.execute_reply":"2021-07-01T14:39:48.318176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"phase1_normal=train.loc[0].values\nphase2_normal=train.loc[1].values\nphase3_normal=train.loc[2].values\n\nphase1_faulty=train.loc[3].values\nphase2_faulty=train.loc[4].values\nphase3_faulty=train.loc[5].values\n\ndt=(20e-3)/len(phase1_faulty)\nfs=1/dt\nt=np.linspace(0,len(phase1_normal)*(1/fs),len(phase1_normal))","metadata":{"execution":{"iopub.status.busy":"2021-07-01T14:40:28.907995Z","iopub.execute_input":"2021-07-01T14:40:28.908461Z","iopub.status.idle":"2021-07-01T14:40:33.347677Z","shell.execute_reply.started":"2021-07-01T14:40:28.908412Z","shell.execute_reply":"2021-07-01T14:40:33.346373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12, 6))\nplt.title(\"normal lines\")\nplt.xlabel(\"time [ms]\")\nplt.plot(t,phase1_normal,'r-', alpha=0.7,label='phase 1')\nplt.plot(t,phase2_normal,'k-', alpha=0.7,label='phase 2')\nplt.plot(t,phase3_normal,'b-', alpha=0.7,label='phase 3')\nplt.ylim(-80,60)\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2021-07-01T14:48:26.122891Z","iopub.execute_input":"2021-07-01T14:48:26.123411Z","iopub.status.idle":"2021-07-01T14:48:27.849198Z","shell.execute_reply.started":"2021-07-01T14:48:26.123376Z","shell.execute_reply":"2021-07-01T14:48:27.848235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12, 6))\nplt.title(\"faulty lines\")\nplt.xlabel(\"time [ms]\")\nplt.plot(t,phase1_faulty,'r-', alpha=0.7,label=\"phase 1\")\nplt.plot(t,phase2_faulty,'k-', alpha=0.7,label=\"phase 2\")\nplt.plot(t,phase3_faulty,'b-', alpha=0.7,label=\"phase 3\")\nplt.ylim(-80,60)\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2021-07-01T14:49:05.171718Z","iopub.execute_input":"2021-07-01T14:49:05.172366Z","iopub.status.idle":"2021-07-01T14:49:06.786967Z","shell.execute_reply.started":"2021-07-01T14:49:05.172313Z","shell.execute_reply":"2021-07-01T14:49:06.786012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.fft import fft, fftfreq,ifft,fftshift\ndt=(20e-3)/len(phase1_faulty)\nfs=1/dt\n\nyf_n = fft(np.array(phase1_normal))\nxf_n = fftfreq(yf_n.size, dt)\n\nyf_f = fft(np.array(phase1_faulty))\nxf_f = fftfreq(yf_f.size, dt)\n\nfig, axes = plt.subplots(2,1, dpi=130, figsize=(4,4), sharex=True)\naxes[0].semilogy(fftshift(xf_n), np.absolute(fftshift(yf_n)),'k',lw=2,label='phase 1 normal')\naxes[0].legend()\naxes[0].set_xlim(0,fs/2)\naxes[0].set_ylim(1e1,1e7)\naxes[0].set_xlabel(\"frequency [Hz]\")\naxes[0].set_ylabel(\"|FFT|\")\naxes[0].grid()\n\naxes[1].semilogy(fftshift(xf_f), np.absolute(fftshift(yf_f)),'r',lw=2,label='phase 1 faulty')\naxes[1].legend()\naxes[1].set_xlim(0,fs/2)\naxes[1].set_ylim(1e1,1e7)\naxes[1].set_xlabel(\"frequency [Hz]\")\naxes[1].set_ylabel(\"|FFT|\")\naxes[1].grid()","metadata":{"execution":{"iopub.status.busy":"2021-07-01T14:50:37.961160Z","iopub.execute_input":"2021-07-01T14:50:37.961783Z","iopub.status.idle":"2021-07-01T14:50:40.329675Z","shell.execute_reply.started":"2021-07-01T14:50:37.961729Z","shell.execute_reply":"2021-07-01T14:50:40.328421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.fft import fft, fftfreq,ifft,fftshift\ndt=(20e-3)/len(phase2_faulty)\nfs=1/dt\n\nyf_n = fft(np.array(phase2_normal))\nxf_n = fftfreq(yf_n.size, dt)\n\nyf_f = fft(np.array(phase2_faulty))\nxf_f = fftfreq(yf_f.size, dt)\n\nfig, axes = plt.subplots(2,1, dpi=130, figsize=(4,4), sharex=True)\naxes[0].semilogy(fftshift(xf_n), np.absolute(fftshift(yf_n)),'k',lw=2,label='phase 2 normal')\naxes[0].legend()\naxes[0].set_xlim(0,fs/2)\naxes[0].set_ylim(1e1,1e7)\naxes[0].set_xlabel(\"frequency [Hz]\")\naxes[0].set_ylabel(\"|FFT|\")\naxes[0].grid()\n\naxes[1].semilogy(fftshift(xf_f), np.absolute(fftshift(yf_f)),'r',lw=2,label='phase 2 faulty')\naxes[1].legend()\naxes[1].set_xlim(0,fs/2)\naxes[1].set_ylim(1e1,1e7)\naxes[1].set_xlabel(\"frequency [Hz]\")\naxes[1].set_ylabel(\"|FFT|\")\naxes[1].grid()","metadata":{"execution":{"iopub.status.busy":"2021-07-01T14:51:33.838976Z","iopub.execute_input":"2021-07-01T14:51:33.839363Z","iopub.status.idle":"2021-07-01T14:51:36.390449Z","shell.execute_reply.started":"2021-07-01T14:51:33.839331Z","shell.execute_reply":"2021-07-01T14:51:36.389253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.fft import fft, fftfreq,ifft,fftshift\ndt=(20e-3)/len(phase3_faulty)\nfs=1/dt\n\nyf_n = fft(np.array(phase3_normal))\nxf_n = fftfreq(yf_n.size, dt)\n\nyf_f = fft(np.array(phase3_faulty))\nxf_f = fftfreq(yf_f.size, dt)\n\nfig, axes = plt.subplots(2,1, dpi=130, figsize=(4,4), sharex=True)\naxes[0].semilogy(fftshift(xf_n), np.absolute(fftshift(yf_n)),'k',lw=2,label='phase 3 normal')\naxes[0].legend()\naxes[0].set_xlim(0,fs/2)\naxes[0].set_ylim(1e1,1e7)\naxes[0].set_xlabel(\"frequency [Hz]\")\naxes[0].set_ylabel(\"|FFT|\")\naxes[0].grid()\n\naxes[1].semilogy(fftshift(xf_f), np.absolute(fftshift(yf_f)),'r',lw=2,label='phase 3 faulty')\naxes[1].legend()\naxes[1].set_xlim(0,fs/2)\naxes[1].set_ylim(1e1,1e7)\naxes[1].set_xlabel(\"frequency [Hz]\")\naxes[1].set_ylabel(\"|FFT|\")\naxes[1].grid()","metadata":{"execution":{"iopub.status.busy":"2021-07-01T14:56:25.137124Z","iopub.execute_input":"2021-07-01T14:56:25.137517Z","iopub.status.idle":"2021-07-01T14:56:27.270324Z","shell.execute_reply.started":"2021-07-01T14:56:25.137482Z","shell.execute_reply":"2021-07-01T14:56:27.268949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy import signal\nf1_n, t1_n, Sxx1_n = signal.spectrogram(phase1_normal, fs)\nf1_f, t1_f, Sxx1_f = signal.spectrogram(phase1_faulty, fs)\n\nfig=plt.figure(figsize=(14,4))\nax1=fig.add_subplot(121)\nim1=ax1.pcolormesh(t1_n, f1_n, 20*np.log10(Sxx1_n), shading='gouraud')\nax1.set_title(\"phase 1 normal\")\nax1.set_ylabel('Frequency [Hz]')\nax1.set_xlabel('Time [sec]')\nfig.colorbar(im1).set_label('Intensity [dB]')\nim1.set_clim(-400, -100)\n\nax2=fig.add_subplot(122)\nim2=ax2.pcolormesh(t1_f, f1_f, 20*np.log10(Sxx1_f), shading='gouraud')\nax2.set_title(\"phase 1 faulty\")\nax2.set_ylabel('Frequency [Hz]')\nax2.set_xlabel('Time [sec]')\nfig.colorbar(im2).set_label('Intensity [dB]')\nim2.set_clim(-400, -100)\n\n","metadata":{"execution":{"iopub.status.busy":"2021-07-01T15:03:22.556518Z","iopub.execute_input":"2021-07-01T15:03:22.557074Z","iopub.status.idle":"2021-07-01T15:03:28.158847Z","shell.execute_reply.started":"2021-07-01T15:03:22.557040Z","shell.execute_reply":"2021-07-01T15:03:28.157852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f2_n, t2_n, Sxx2_n = signal.spectrogram(phase2_normal, fs)\nf2_f, t2_f, Sxx2_f = signal.spectrogram(phase2_faulty, fs)\n\nfig=plt.figure(figsize=(14,4))\nax1=fig.add_subplot(121)\nim1=ax1.pcolormesh(t2_n, f2_n, 20*np.log10(Sxx2_n), shading='gouraud')\nax1.set_title(\"phase 2 normal\")\nax1.set_ylabel('Frequency [Hz]')\nax1.set_xlabel('Time [sec]')\nfig.colorbar(im1).set_label('Intensity [dB]')\nim1.set_clim(-400, -100)\n\nax2=fig.add_subplot(122)\nim2=ax2.pcolormesh(t2_f, f2_f, 20*np.log10(Sxx2_f), shading='gouraud')\nax2.set_title(\"phase 2 faulty\")\nax2.set_ylabel('Frequency [Hz]')\nax2.set_xlabel('Time [sec]')\nfig.colorbar(im2).set_label('Intensity [dB]')\nim2.set_clim(-400, -100)\n","metadata":{"execution":{"iopub.status.busy":"2021-07-01T15:05:51.330846Z","iopub.execute_input":"2021-07-01T15:05:51.331336Z","iopub.status.idle":"2021-07-01T15:05:56.992661Z","shell.execute_reply.started":"2021-07-01T15:05:51.331288Z","shell.execute_reply":"2021-07-01T15:05:56.991525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f3_n, t3_n, Sxx3_n = signal.spectrogram(phase3_normal, fs)\nf3_f, t3_f, Sxx3_f = signal.spectrogram(phase3_faulty, fs)\n\nfig=plt.figure(figsize=(14,4))\nax1=fig.add_subplot(121)\nim1=ax1.pcolormesh(t3_n, f3_n, 20*np.log10(Sxx3_n), shading='gouraud')\nax1.set_title(\"phase 3 normal\")\nax1.set_ylabel('Frequency [Hz]')\nax1.set_xlabel('Time [sec]')\nfig.colorbar(im1).set_label('Intensity [dB]')\nim1.set_clim(-400, -100)\n\nax2=fig.add_subplot(122)\nim2=ax2.pcolormesh(t3_f, f3_f, 20*np.log10(Sxx3_f), shading='gouraud')\nax2.set_title(\"phase 3 faulty\")\nax2.set_ylabel('Frequency [Hz]')\nax2.set_xlabel('Time [sec]')\nfig.colorbar(im2).set_label('Intensity [dB]')\nim2.set_clim(-400, -100)\n","metadata":{"execution":{"iopub.status.busy":"2021-07-01T15:06:40.168762Z","iopub.execute_input":"2021-07-01T15:06:40.169215Z","iopub.status.idle":"2021-07-01T15:06:45.736082Z","shell.execute_reply.started":"2021-07-01T15:06:40.169174Z","shell.execute_reply":"2021-07-01T15:06:45.734948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom numpy.fft import rfft, irfft, rfftfreq\nfrom scipy import fftpack\n\ndef decompose_into_n_signals(srs, n):\n    fourier = rfft(srs)\n    frequencies = rfftfreq(srs.size, d=2e-2/srs.size)\n    out = []\n    for vals in np.array_split(frequencies, n):\n        ft_threshed = fourier.copy()\n        ft_threshed[(vals.min() > frequencies)] = 0\n        ft_threshed[(vals.max() < frequencies)] = 0        \n        out.append(irfft(ft_threshed))\n    return out\n\ndef plot_n_signals(sigs):\n    fig, axarr = plt.subplots(len(sigs), figsize=(12, 12))\n    for i, sig in enumerate(sigs):\n        plt.sca(axarr[i])\n        plt.plot(sig)\n    plt.gcf().suptitle(f\"Decomposition into {len(sigs)} frequency bands\", fontsize=14)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T15:09:53.015412Z","iopub.execute_input":"2021-07-01T15:09:53.016299Z","iopub.status.idle":"2021-07-01T15:09:53.042619Z","shell.execute_reply.started":"2021-07-01T15:09:53.016239Z","shell.execute_reply":"2021-07-01T15:09:53.041375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.fft import fft, fftfreq,ifft,fftshift\n\ndef brickwall(xf,yf,cutoff):\n    for i in range(len(xf)):  \n        if xf[i]>=cutoff or xf[i]<=-cutoff:\n            yf[i]=0\n    return yf  ","metadata":{"execution":{"iopub.status.busy":"2021-07-01T15:27:12.976409Z","iopub.execute_input":"2021-07-01T15:27:12.976870Z","iopub.status.idle":"2021-07-01T15:27:12.984503Z","shell.execute_reply.started":"2021-07-01T15:27:12.976829Z","shell.execute_reply":"2021-07-01T15:27:12.982848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filt_brick_normal = ifft(brickwall(xf_n,yf_n,1.25e7))    \n\nplt.plot(t, phase3_normal, 'b-',linewidth=3, label='data')\nplt.plot(t, filt_brick_normal, 'g--', linewidth=2, label='filtered')\nplt.xlabel('time [ms]')\nplt.ylim(-80,60)\nplt.legend()\n                      ","metadata":{"execution":{"iopub.status.busy":"2021-07-01T15:24:37.820099Z","iopub.execute_input":"2021-07-01T15:24:37.820475Z","iopub.status.idle":"2021-07-01T15:24:40.030894Z","shell.execute_reply.started":"2021-07-01T15:24:37.820442Z","shell.execute_reply":"2021-07-01T15:24:40.029600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filt_brick_faulty = ifft(brickwall(xf_f,yf_f,1.25e7))    \n\nplt.plot(t, phase3_faulty, 'b-',linewidth=3, label='data faulty')\nplt.plot(t, filt_brick_faulty, 'g--', linewidth=2, label='filtered')\nplt.xlabel('time [ms]')\nplt.ylim(-80,60)\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2021-07-01T15:31:43.095858Z","iopub.execute_input":"2021-07-01T15:31:43.096241Z","iopub.status.idle":"2021-07-01T15:31:45.116624Z","shell.execute_reply.started":"2021-07-01T15:31:43.096208Z","shell.execute_reply":"2021-07-01T15:31:45.115310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f3_n, t3_n, Sxx3_n = signal.spectrogram(filt_brick_normal, fs)\nf3_f, t3_f, Sxx3_f = signal.spectrogram(filt_brick_faulty, fs)\n\nfig=plt.figure(figsize=(14,4))\nax1=fig.add_subplot(121)\nim1=ax1.pcolormesh(t3_n, f3_n, 20*np.log10(Sxx3_n), shading='gouraud')\nax1.set_title(\"phase 3 normal\")\nax1.set_ylabel('Frequency [Hz]')\nax1.set_xlabel('Time [sec]')\nfig.colorbar(im1).set_label('Intensity [dB]')\nim1.set_clim(-400, -100)\n\nax2=fig.add_subplot(122)\nim2=ax2.pcolormesh(t3_f, f3_f, 20*np.log10(Sxx3_f), shading='gouraud')\nax2.set_title(\"phase 3 faulty\")\nax2.set_ylabel('Frequency [Hz]')\nax2.set_xlabel('Time [sec]')\nfig.colorbar(im2).set_label('Intensity [dB]')\nim2.set_clim(-400, -100)\n","metadata":{"execution":{"iopub.status.busy":"2021-07-01T15:25:19.234339Z","iopub.execute_input":"2021-07-01T15:25:19.234772Z","iopub.status.idle":"2021-07-01T15:25:30.807915Z","shell.execute_reply.started":"2021-07-01T15:25:19.234732Z","shell.execute_reply":"2021-07-01T15:25:30.806663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_n_signals(decompose_into_n_signals(phase1_faulty, 5))","metadata":{"execution":{"iopub.status.busy":"2021-07-01T15:09:56.020875Z","iopub.execute_input":"2021-07-01T15:09:56.022996Z","iopub.status.idle":"2021-07-01T15:09:57.958566Z","shell.execute_reply.started":"2021-07-01T15:09:56.022918Z","shell.execute_reply":"2021-07-01T15:09:57.957434Z"},"trusted":true},"execution_count":null,"outputs":[]}]}