{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Fast Fourier Transform & Denoising\n","metadata":{"_uuid":"63108ed7f840eba7d84f1fb5e6774a66316c1b4c"}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom numpy.fft import *\nimport pyarrow.parquet as pq\nimport matplotlib.pyplot as plt\n\nsns.set_style(\"whitegrid\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1 - Loading Data","metadata":{"_uuid":"2791aa2e427c195b5434f37ed425bc8be5775767"}},{"cell_type":"markdown","source":"### Signals","metadata":{"_uuid":"f9db7c1298259905c3a6276d2b09a76625049835"}},{"cell_type":"code","source":"signals = pq.read_table('../input/train.parquet', columns=[str(i) for i in range(999)]).to_pandas()","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"signals = np.array(signals).T.reshape((999//3, 3, 800000))","metadata":{"_uuid":"1255c51404246d23465d50575049db937fd7c6f8","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15, 10))\nplt.plot(signals[0, 0, :], label='Phase 0')\nplt.plot(signals[0, 1, :], label='Phase 1')\nplt.plot(signals[0, 2, :], label='Phase 2')\nplt.legend()\nplt.show()","metadata":{"_uuid":"832d8fb176ea537856fd6eae5bfb3425874a1fbd","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Target","metadata":{"_uuid":"6fb88dc514013b35e5915a5e520aab7ef07babaa"}},{"cell_type":"code","source":"train_df = pd.read_csv('../input/metadata_train.csv')\ntrain_df.head()","metadata":{"_uuid":"2221c7c5dd351386cbaa8162e622e535f64c3a1b","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target = train_df['target'][::3]","metadata":{"_uuid":"013f3ab108163f9272295192892ed44e7ec71e11","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15, 10))\nsns.countplot(target)\nplt.show()","metadata":{"_uuid":"226e535bbaf27f2f663c09fedddd555689ff11a8","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2 - Smoothing by mean\nK neighbour ile signal gürültüsünü azaltmak hedeflenmiştir","metadata":{"_uuid":"b4f641105de1800b26081e6b0fc0dd93d912e452"}},{"cell_type":"code","source":"def sample(signal, kernel_size):\n    sampled = np.zeros((signal.shape[0], signal.shape[1], signal.shape[2]//kernel_size))\n    for i in range(signal.shape[2]//kernel_size):\n        begin = kernel_size * i\n        end = min(kernel_size * (i + 1), signal.shape[2])\n        sampled[:, :, i] = np.mean(signal[:, :, begin:end], axis=2)\n    return sampled","metadata":{"_uuid":"1f48858b8e21bba91768295c1382c1717601a6b2","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sampled = sample(signals, 100)","metadata":{"_uuid":"ca3b12709e3603dcc89625f0ff4adc948a9795e5","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15, 10))\nplt.plot(sampled[0, 0, :], label='Phase 0')\nplt.plot(sampled[0, 1, :], label='Phase 1')\nplt.plot(sampled[0, 2, :], label='Phase 2')\nplt.legend()\nplt.show()","metadata":{"_uuid":"728175d3acac3e7779e486f6df5a54139711212b","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"def filter_signal(signal, threshold=1e8):\n    fourier = rfft(signal)\n    frequencies = rfftfreq(signal.size, d=20e-3/signal.size)\n    fourier[frequencies > threshold] = 0\n    return irfft(fourier)","metadata":{"_uuid":"bf7d1a88706a1dfebfe70675806ed799d529536f","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Testing some thresholds","metadata":{"_uuid":"481f1062cadb64df8776fc2b92fe8202c8e34d8a"}},{"cell_type":"code","source":"filtered = filter_signal(signals[0, 0, :], threshold=1e3)","metadata":{"_uuid":"a22aad3bdbf3369a2a180e4692ba29a99d4be078","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15, 10))\nplt.plot(signals[0, 0, :], label='Raw')\nplt.plot(filtered, label='Filtered')\nplt.legend()\nplt.title(\"FFT Denoising with threshold = 1e3\", size=15)\nplt.show()","metadata":{"_uuid":"ac58d384afa698f801f4a4f49d4550b48835dae3","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered = filter_signal(signals[0, 0, :], threshold=1e5)","metadata":{"_uuid":"92e910fc06f227234c97d61a10409dcb8df13025","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15, 10))\nplt.plot(signals[0, 0, :], label='Raw')\nplt.plot(filtered, label='Filtered')\nplt.legend()\nplt.title(\"FFT Denoising with threshold = 1e5\", size=15)\nplt.show()","metadata":{"_uuid":"1139e9e41ca5914d1c5857c1bd615c4a8c593349","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered = filter_signal(signals[0, 0, :], threshold=1e7)","metadata":{"_uuid":"0ba9d1babc5176696201591b8192e22a51409422","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15, 10))\nplt.plot(signals[0, 0, :], label='Raw')\nplt.plot(filtered, label='Filtered')\nplt.legend()\nplt.title(\"FFT Denoising with threshold = 1e7\", size=15)\nplt.show()","metadata":{"_uuid":"6de0fa39dbbead16f9207681867b984433b6275d","trusted":true},"execution_count":null,"outputs":[]}]}