{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### imports","metadata":{}},{"cell_type":"code","source":"# standard lib\nimport pathlib\n\n# third party\nimport pandas as pd\nimport numpy as np\nimport plotly.express as px\nfrom plotly.subplots import make_subplots\nimport plotly.graph_objects as go\nimport scipy\nimport scipy.signal as signal\nimport pywt\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2021-07-08T12:46:54.185593Z","iopub.execute_input":"2021-07-08T12:46:54.186046Z","iopub.status.idle":"2021-07-08T12:46:56.472520Z","shell.execute_reply.started":"2021-07-08T12:46:54.185962Z","shell.execute_reply":"2021-07-08T12:46:56.471567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### file load","metadata":{}},{"cell_type":"code","source":"%%time\nroot_path = pathlib.Path('/kaggle/input/g2net-gravitational-wave-detection')\ntrain_files = sorted(root_path.joinpath('train').glob('**/*.*'))\ntest_files = sorted(root_path.joinpath('test').glob('**/*.*'))\nprint(f\"train_num: {len(train_files)}, test_num: {len(test_files)}\")\nsub_file = pd.read_csv(root_path.joinpath('sample_submission.csv'))\n# print(sub_file.loc[:10,:])\ntrain_labels = pd.read_csv(root_path.joinpath('training_labels.csv'))\nprint(f\"label stats: \\n{train_labels['target'].value_counts()}\")","metadata":{"execution":{"iopub.status.busy":"2021-07-08T12:46:56.473781Z","iopub.execute_input":"2021-07-08T12:46:56.474082Z","iopub.status.idle":"2021-07-08T12:49:18.611458Z","shell.execute_reply.started":"2021-07-08T12:46:56.474053Z","shell.execute_reply":"2021-07-08T12:49:18.610400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### label splitting","metadata":{}},{"cell_type":"code","source":"train_target0_index = train_labels[train_labels[\"target\"]==0].index\ntrain_target1_index = train_labels[train_labels[\"target\"]==1].index","metadata":{"execution":{"iopub.status.busy":"2021-07-08T12:49:18.613218Z","iopub.execute_input":"2021-07-08T12:49:18.613516Z","iopub.status.idle":"2021-07-08T12:49:18.673801Z","shell.execute_reply.started":"2021-07-08T12:49:18.613487Z","shell.execute_reply":"2021-07-08T12:49:18.672792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### plot 3 samples","metadata":{}},{"cell_type":"code","source":"subplot_titles = []\nfor i in range(3):\n    subplot_titles.append(str(train_files[train_target0_index[i]].stem) + \" (target=0)\")\n    subplot_titles.append(str(train_files[train_target1_index[i]].stem) + \" (target=1)\")\n\nfig = make_subplots(rows=3, cols=2, shared_xaxes=True, shared_yaxes=True, subplot_titles=subplot_titles)\ncolormap = px.colors.sequential.Turbo\ncolors = [colormap[2], colormap[6], colormap[11]]\nfor i in range(3):\n    signal_df = pd.DataFrame(np.load(train_files[train_target0_index[i]]).transpose(), columns=[\"LIGO Hanford\", \"LIGO Livingston\", \"Virgo\"])\n    subfig = px.line(signal_df, title=str(train_files[train_target0_index[i]]))\n    \n    for j,d in enumerate(subfig.data):\n        fig.add_trace((go.Scatter(x=d['x'], y=d['y'], name = d['name'], line=dict(color=colors[j]))), row=i+1, col=1)\n        \nfor i in range(3):\n    signal_df = pd.DataFrame(np.load(train_files[train_target1_index[i]]).transpose(), columns=[\"LIGO Hanford\", \"LIGO Livingston\", \"Virgo\"])\n    subfig = px.line(signal_df)\n    \n    for j,d in enumerate(subfig.data):\n        fig.add_trace((go.Scatter(x=d['x'], y=d['y'], name = d['name'], line=dict(color=colors[j]))), row=i+1, col=2)\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T12:49:18.675395Z","iopub.execute_input":"2021-07-08T12:49:18.675725Z","iopub.status.idle":"2021-07-08T12:49:20.858977Z","shell.execute_reply.started":"2021-07-08T12:49:18.675694Z","shell.execute_reply":"2021-07-08T12:49:20.857910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### FFT analysis(ALL time)\n#### amplitude spectrum","metadata":{}},{"cell_type":"code","source":"sig = np.load(train_files[train_target0_index[0]])\nF = scipy.fft.fft(sig)\nnp.log10(np.abs(F[:,:F.shape[1]//2]))\n\nfig = make_subplots(rows=3, cols=2, shared_xaxes=True, shared_yaxes=True, subplot_titles=subplot_titles)\ncolormap = px.colors.sequential.Turbo\ncolors = [colormap[2], colormap[6], colormap[11]]\nfor i in range(3):\n    sig = np.load(train_files[train_target0_index[i]])\n    F = scipy.fft.fft(sig)\n    F_log = np.log10(np.abs(F[:,:F.shape[1]//2]))\n    signal_df = pd.DataFrame(F_log.transpose(), columns=[\"LIGO Hanford\", \"LIGO Livingston\", \"Virgo\"])\n    subfig = px.line(signal_df, title=str(train_files[train_target0_index[i]]))\n    \n    for j,d in enumerate(subfig.data):\n        fig.add_trace((go.Scatter(x=d['x'], y=d['y'], name = d['name'], line=dict(color=colors[j], width=1))), row=i+1, col=1)\n        \nfor i in range(3):\n    sig = np.load(train_files[train_target1_index[i]])\n    F = scipy.fft.fft(sig)\n    F_log = np.log10(np.abs(F[:,:F.shape[1]//2]))\n    signal_df = pd.DataFrame(F_log.transpose(), columns=[\"LIGO Hanford\", \"LIGO Livingston\", \"Virgo\"])\n    subfig = px.line(signal_df, title=str(train_files[train_target0_index[i]]))\n    \n    for j,d in enumerate(subfig.data):\n        fig.add_trace((go.Scatter(x=d['x'], y=d['y'], name = d['name'], line=dict(color=colors[j], width=1))), row=i+1, col=2)\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T12:49:20.860291Z","iopub.execute_input":"2021-07-08T12:49:20.860596Z","iopub.status.idle":"2021-07-08T12:49:21.642890Z","shell.execute_reply.started":"2021-07-08T12:49:20.860566Z","shell.execute_reply":"2021-07-08T12:49:21.641828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### phase spectrum (diff)","metadata":{}},{"cell_type":"code","source":"fig = make_subplots(rows=3, cols=2, shared_xaxes=True, shared_yaxes=True, subplot_titles=subplot_titles)\ncolormap = px.colors.sequential.Turbo\ncolors = [colormap[2], colormap[6], colormap[11]]\nfor i in range(3):\n    sig = np.load(train_files[train_target0_index[0]])\n    F = scipy.fft.fft(sig)\n    spec_cross = F[0,:F.shape[1]//1]/F[1,:F.shape[1]//1]\n    spec_phase_diff_01 = np.arctan2(spec_cross.imag, spec_cross.real)\n    spec_cross = F[1,:F.shape[1]//1]/F[2,:F.shape[1]//1]\n    spec_phase_diff_12 = np.arctan2(spec_cross.imag, spec_cross.real)\n    spec_cross = F[2,:F.shape[1]//1]/F[0,:F.shape[1]//1]\n    spec_phase_diff_20 = np.arctan2(spec_cross.imag, spec_cross.real)\n    spec_phase_diff = np.concatenate([\n        [spec_phase_diff_01],\n        [spec_phase_diff_12],\n        [spec_phase_diff_20],\n    ], axis=0)\n    \n    signal_df = pd.DataFrame(spec_phase_diff.transpose(), columns=[\"Hanford-Livingston\", \"Livingston-Virgo\", \"Virgo-Hanford\"])\n    subfig = px.line(signal_df, title=str(train_files[train_target0_index[i]]))\n    \n    for j,d in enumerate(subfig.data):\n        fig.add_trace((go.Scatter(x=d['x'], y=d['y'], name = d['name'], line=dict(color=colors[j], width=1))), row=i+1, col=1)\n        \nfor i in range(3):\n    sig = np.load(train_files[train_target1_index[0]])\n    F = scipy.fft.fft(sig)\n    spec_cross = F[0,:F.shape[1]//1]/F[1,:F.shape[1]//1]\n    spec_phase_diff_01 = np.arctan2(spec_cross.imag, spec_cross.real)\n    spec_cross = F[1,:F.shape[1]//1]/F[2,:F.shape[1]//1]\n    spec_phase_diff_12 = np.arctan2(spec_cross.imag, spec_cross.real)\n    spec_cross = F[2,:F.shape[1]//1]/F[0,:F.shape[1]//1]\n    spec_phase_diff_20 = np.arctan2(spec_cross.imag, spec_cross.real)\n    spec_phase_diff = np.concatenate([\n        [spec_phase_diff_01],\n        [spec_phase_diff_12],\n        [spec_phase_diff_20],\n    ], axis=0)\n    \n    signal_df = pd.DataFrame(spec_phase_diff.transpose(), columns=[\"Hanford-Livingston\", \"Livingston-Virgo\", \"Virgo-Hanford\"])\n    subfig = px.line(signal_df, title=str(train_files[train_target0_index[i]]))\n    \n    for j,d in enumerate(subfig.data):\n        fig.add_trace((go.Scatter(x=d['x'], y=d['y'], name = d['name'], line=dict(color=colors[j], width=1))), row=i+1, col=2)\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T12:49:21.644263Z","iopub.execute_input":"2021-07-08T12:49:21.644773Z","iopub.status.idle":"2021-07-08T12:49:22.627827Z","shell.execute_reply.started":"2021-07-08T12:49:21.644735Z","shell.execute_reply":"2021-07-08T12:49:22.626910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### FFT(STFT)\n\n#### amplitude spectrum","metadata":{}},{"cell_type":"code","source":"sample_index=0\nsubplot_titles=[]\nfor site in [\"LIGO Hanford\", \"LIGO Livingston\", \"Virgo\"]:\n    subplot_titles.append(f\"file={str(train_files[train_target0_index[sample_index]].stem)}, site={site}, (target=0)\")\n    subplot_titles.append(f\"file={str(train_files[train_target0_index[sample_index]].stem)}, site={site}, (target=1)\")\nfig = make_subplots(rows=3, cols=2, subplot_titles=subplot_titles)\n\ntime_signal = np.load(train_files[train_target0_index[sample_index]])\nf,t,Sxx = signal.spectrogram(time_signal, fs=2048, nfft=512, nperseg=256)\nfig.add_trace((go.Heatmap(x=f, y=t, z=np.log10(Sxx[0].transpose()), colorbar=None)), row=1, col=1)\nfig.add_trace((go.Heatmap(x=f, y=t, z=np.log10(Sxx[1].transpose()), colorbar=None)), row=2, col=1)\nfig.add_trace((go.Heatmap(x=f, y=t, z=np.log10(Sxx[2].transpose()), colorbar=None)), row=3, col=1)\n\ntime_signal = np.load(train_files[train_target1_index[sample_index]])\nf,t,Sxx = signal.spectrogram(time_signal, fs=2048, nfft=512, nperseg=256)\nfig.add_trace((go.Heatmap(x=f, y=t, z=np.log10(Sxx[0].transpose()), colorbar=None)), row=1, col=2)\nfig.add_trace((go.Heatmap(x=f, y=t, z=np.log10(Sxx[1].transpose()), colorbar=None)), row=2, col=2)\nfig.add_trace((go.Heatmap(x=f, y=t, z=np.log10(Sxx[2].transpose()), colorbar=None)), row=3, col=2)\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T12:49:22.628937Z","iopub.execute_input":"2021-07-08T12:49:22.629339Z","iopub.status.idle":"2021-07-08T12:49:22.775042Z","shell.execute_reply.started":"2021-07-08T12:49:22.629310Z","shell.execute_reply":"2021-07-08T12:49:22.774250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### amplitude spectrum diff","metadata":{}},{"cell_type":"code","source":"sample_index=-1\nsubplot_titles=[]\nfor site in [\"Hanford-Livingston\", \"Livingston-Virgo\", \"Virgo-Hanford\"]:\n    subplot_titles.append(f\"file={str(train_files[train_target0_index[sample_index]].stem)}, site={site}, (target=0)\")\n    subplot_titles.append(f\"file={str(train_files[train_target1_index[sample_index]].stem)}, site={site}, (target=1)\")\nfig = make_subplots(rows=3, cols=2, subplot_titles=subplot_titles, shared_xaxes=True, shared_yaxes=True)\n\ntime_signal = np.load(train_files[train_target0_index[sample_index]])\nf,t,Sxx = signal.spectrogram(time_signal, fs=2048, nfft=512, nperseg=256, mode='complex')\nfig.add_trace((go.Heatmap(x=f, y=t, z=np.log10(np.abs(Sxx[0]/Sxx[1])).transpose(), colorbar=None)), row=1, col=1)\nfig.add_trace((go.Heatmap(x=f, y=t, z=np.log10(np.abs(Sxx[1]/Sxx[2])).transpose(), colorbar=None)), row=2, col=1)\nfig.add_trace((go.Heatmap(x=f, y=t, z=np.log10(np.abs(Sxx[2]/Sxx[0])).transpose(), colorbar=None)), row=3, col=1)\n\ntime_signal = np.load(train_files[train_target1_index[sample_index]])\nf,t,Sxx = signal.spectrogram(time_signal, fs=2048, nfft=512, nperseg=256, mode='complex')\nfig.add_trace((go.Heatmap(x=f, y=t, z=np.log10(np.abs(Sxx[0]/Sxx[1])).transpose(), colorbar=None)), row=1, col=2)\nfig.add_trace((go.Heatmap(x=f, y=t, z=np.log10(np.abs(Sxx[1]/Sxx[2])).transpose(), colorbar=None)), row=2, col=2)\nfig.add_trace((go.Heatmap(x=f, y=t, z=np.log10(np.abs(Sxx[2]/Sxx[0])).transpose(), colorbar=None)), row=3, col=2)\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T12:49:22.776676Z","iopub.execute_input":"2021-07-08T12:49:22.776996Z","iopub.status.idle":"2021-07-08T12:49:22.951006Z","shell.execute_reply.started":"2021-07-08T12:49:22.776965Z","shell.execute_reply":"2021-07-08T12:49:22.950077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### phase spectrum diff","metadata":{}},{"cell_type":"code","source":"sample_index=0\nsubplot_titles=[]\nfor site in [\"Hanford-Livingston\", \"Livingston-Virgo\", \"Virgo-Hanford\"]:\n    subplot_titles.append(f\"file={str(train_files[train_target0_index[sample_index]].stem)}, site={site}, (target=0)\")\n    subplot_titles.append(f\"file={str(train_files[train_target1_index[sample_index]].stem)}, site={site}, (target=1)\")\nfig = make_subplots(rows=3, cols=2, subplot_titles=subplot_titles, shared_xaxes=True, shared_yaxes=True)\n\ntime_signal = np.load(train_files[train_target0_index[sample_index]])\nf,t,Sxx = signal.spectrogram(time_signal, fs=2048, nfft=512, nperseg=256, mode='complex')\nfig.add_trace((go.Heatmap(x=f, y=t, z=np.arctan2((Sxx[0]/Sxx[1]).imag, (Sxx[0]/Sxx[1]).real).transpose(), colorbar=None)), row=1, col=1)\nfig.add_trace((go.Heatmap(x=f, y=t, z=np.arctan2((Sxx[1]/Sxx[2]).imag, (Sxx[1]/Sxx[2]).real).transpose(), colorbar=None)), row=2, col=1)\nfig.add_trace((go.Heatmap(x=f, y=t, z=np.arctan2((Sxx[2]/Sxx[0]).imag, (Sxx[2]/Sxx[0]).real).transpose(), colorbar=None)), row=3, col=1)\n\ntime_signal = np.load(train_files[train_target1_index[sample_index]])\nf,t,Sxx = signal.spectrogram(time_signal, fs=2048, nfft=512, nperseg=256, mode='complex')\nfig.add_trace((go.Heatmap(x=f, y=t, z=np.arctan2((Sxx[0]/Sxx[1]).imag, (Sxx[0]/Sxx[1]).real).transpose(), colorbar=None)), row=1, col=2)\nfig.add_trace((go.Heatmap(x=f, y=t, z=np.arctan2((Sxx[1]/Sxx[2]).imag, (Sxx[1]/Sxx[2]).real).transpose(), colorbar=None)), row=2, col=2)\nfig.add_trace((go.Heatmap(x=f, y=t, z=np.arctan2((Sxx[2]/Sxx[0]).imag, (Sxx[2]/Sxx[0]).real).transpose(), colorbar=None)), row=3, col=2)\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T12:49:22.952371Z","iopub.execute_input":"2021-07-08T12:49:22.952665Z","iopub.status.idle":"2021-07-08T12:49:23.110893Z","shell.execute_reply.started":"2021-07-08T12:49:22.952635Z","shell.execute_reply":"2021-07-08T12:49:23.109903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### wavelet transform","metadata":{}},{"cell_type":"code","source":"sample_index=0\nsubplot_titles=[]\nfor site in [\"LIGO Hanford\", \"LIGO Livingston\", \"Virgo\"]:\n    subplot_titles.append(f\"file={str(train_files[train_target0_index[sample_index]].stem)}, site={site}, (target=0)\")\n    subplot_titles.append(f\"file={str(train_files[train_target0_index[sample_index]].stem)}, site={site}, (target=1)\")\nfig = make_subplots(rows=3, cols=2, subplot_titles=subplot_titles, shared_xaxes=True, shared_yaxes=True)\n\nfs=2048 # Hz\ntime_signal = np.load(train_files[train_target0_index[sample_index]])\ncwt_mat, freqs = pywt.cwt(time_signal, scales=np.arange(1, 31, 0.25), wavelet='cmor1.5-1.0', sampling_period=1/fs, method='fft')\nfig.add_trace((go.Heatmap(x=np.arange(0,time_signal.shape[1])/fs, y=freqs, z=np.log10(np.abs(cwt_mat[:,0,:])), colorbar=None)), row=1, col=1)\nfig.add_trace((go.Heatmap(x=np.arange(0,time_signal.shape[1])/fs, y=freqs, z=np.log10(np.abs(cwt_mat[:,1,:])), colorbar=None)), row=2, col=1)\nfig.add_trace((go.Heatmap(x=np.arange(0,time_signal.shape[1])/fs, y=freqs, z=np.log10(np.abs(cwt_mat[:,2,:])), colorbar=None)), row=3, col=1)\n\ntime_signal = np.load(train_files[train_target1_index[sample_index]])\ncwt_mat, freqs = pywt.cwt(time_signal, scales=np.arange(1, 31, 0.25), wavelet='cmor1.5-1.0', sampling_period=1/fs, method='fft')\nfig.add_trace((go.Heatmap(x=np.arange(0,time_signal.shape[1])/fs, y=freqs, z=np.log10(np.abs(cwt_mat[:,0,:])), colorbar=None)), row=1, col=2)\nfig.add_trace((go.Heatmap(x=np.arange(0,time_signal.shape[1])/fs, y=freqs, z=np.log10(np.abs(cwt_mat[:,1,:])), colorbar=None)), row=2, col=2)\nfig.add_trace((go.Heatmap(x=np.arange(0,time_signal.shape[1])/fs, y=freqs, z=np.log10(np.abs(cwt_mat[:,2,:])), colorbar=None)), row=3, col=2)\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T12:49:23.334702Z","iopub.execute_input":"2021-07-08T12:49:23.335166Z","iopub.status.idle":"2021-07-08T12:49:28.519146Z","shell.execute_reply.started":"2021-07-08T12:49:23.335119Z","shell.execute_reply":"2021-07-08T12:49:28.517782Z"},"trusted":true},"execution_count":null,"outputs":[]}]}