{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n\nimport os\nimport math\nimport cmath\nimport shutil","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-22T13:01:55.722625Z","iopub.execute_input":"2021-08-22T13:01:55.723049Z","iopub.status.idle":"2021-08-22T13:01:55.728072Z","shell.execute_reply.started":"2021-08-22T13:01:55.72301Z","shell.execute_reply":"2021-08-22T13:01:55.727017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -q nnAudio","metadata":{"execution":{"iopub.status.busy":"2021-08-22T12:37:44.208183Z","iopub.execute_input":"2021-08-22T12:37:44.208576Z","iopub.status.idle":"2021-08-22T12:37:53.678683Z","shell.execute_reply.started":"2021-08-22T12:37:44.208543Z","shell.execute_reply":"2021-08-22T12:37:53.677567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nimport pathlib\nfrom pathlib import Path\nfrom numba import njit, jit, cuda, guvectorize\nfrom scipy.signal import butter, filtfilt, sosfiltfilt\nfrom nnAudio.Spectrogram import *\nimport torch\nimport gc\n\nimport joblib\nfrom tqdm.auto import tqdm","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:24:10.950684Z","iopub.execute_input":"2021-08-22T14:24:10.951069Z","iopub.status.idle":"2021-08-22T14:24:11.033956Z","shell.execute_reply.started":"2021-08-22T14:24:10.951038Z","shell.execute_reply":"2021-08-22T14:24:11.03292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# head = pathlib.Path(\"../input/g2net-gravitational-wave-detection\")\ntrain_files = [Path(path) for path in sorted(glob.glob(\"../input/g2net-gravitational-wave-detection/train/*/*/*/*.npy\"))]","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:19:16.482371Z","iopub.execute_input":"2021-08-22T14:19:16.482798Z","iopub.status.idle":"2021-08-22T14:19:32.914809Z","shell.execute_reply.started":"2021-08-22T14:19:16.482751Z","shell.execute_reply":"2021-08-22T14:19:32.913505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wave = np.load(train_files[0])","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:04:25.324654Z","iopub.execute_input":"2021-08-22T14:04:25.325244Z","iopub.status.idle":"2021-08-22T14:04:25.340105Z","shell.execute_reply.started":"2021-08-22T14:04:25.325208Z","shell.execute_reply":"2021-08-22T14:04:25.339092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@njit(nogil=True)\ndef min_max_scaler(wave):\n    for i in range(len(wave)):\n        wave[i] = (wave[i] - min(wave[i])) / (max(wave[i]) - min(wave[i]))\n#         wave[i] = 2 * wave[i] - 1\n        \n    return wave","metadata":{"execution":{"iopub.status.busy":"2021-08-22T13:27:28.081451Z","iopub.execute_input":"2021-08-22T13:27:28.081837Z","iopub.status.idle":"2021-08-22T13:27:28.08828Z","shell.execute_reply.started":"2021-08-22T13:27:28.081805Z","shell.execute_reply":"2021-08-22T13:27:28.087399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Taken from https://www.kaggle.com/anjum48/continuous-wavelet-transform-cwt-in-pytorch#Test-on-GW-data\ndef butter_bandpass_filter(data, low, high, fs, order):\n    sos = butter(order, [low, high], btype=\"bandpass\", output=\"sos\", fs=fs)\n    normalization = np.sqrt((high - low) / (fs / 2))\n    return sosfiltfilt(sos, data) # / normalization","metadata":{"execution":{"iopub.status.busy":"2021-08-22T13:42:32.474802Z","iopub.execute_input":"2021-08-22T13:42:32.475376Z","iopub.status.idle":"2021-08-22T13:42:32.481267Z","shell.execute_reply.started":"2021-08-22T13:42:32.475337Z","shell.execute_reply":"2021-08-22T13:42:32.480302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"T = 2\nfs = 2048.0\ncutoff = 2.5\nnyq = 0.5 * fs\norder = 4\nn = int(T * fs)\nnormal_cutoff = cutoff / nyq","metadata":{"execution":{"iopub.status.busy":"2021-08-22T12:41:50.213552Z","iopub.execute_input":"2021-08-22T12:41:50.214213Z","iopub.status.idle":"2021-08-22T12:41:50.218704Z","shell.execute_reply.started":"2021-08-22T12:41:50.214172Z","shell.execute_reply":"2021-08-22T12:41:50.217874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(dpi=120)\nplt.plot(range(len(wave[0])), wave[0])","metadata":{"execution":{"iopub.status.busy":"2021-08-22T12:42:36.813849Z","iopub.execute_input":"2021-08-22T12:42:36.814376Z","iopub.status.idle":"2021-08-22T12:42:37.024351Z","shell.execute_reply.started":"2021-08-22T12:42:36.814341Z","shell.execute_reply":"2021-08-22T12:42:37.023241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Note from this point on, all wavelet transforms equation are taken from https://pywavelets.readthedocs.io/en/latest/ref/cwt.html\n\nLet's do the Shannon Wavelets transform. We will first do it in numpy and see what it results in after the transformation. Note this is the most unoptimized way of performing calculations. ","metadata":{}},{"cell_type":"code","source":"# B = bandwidth = wavelet width (Hz)\n# C = center frequency (Hz)\ndef shannon_wavelets(wave, B=1, C=1):\n    exp_term = np.exp(1j * 2 * np.pi * C * wave)\n    frac_term = (np.sin(np.pi * B * wave) / (np.pi * B * wave))\n    return np.sqrt(B) * frac_term * exp_term","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:05:41.044873Z","iopub.execute_input":"2021-08-22T14:05:41.045275Z","iopub.status.idle":"2021-08-22T14:05:41.05145Z","shell.execute_reply.started":"2021-08-22T14:05:41.045237Z","shell.execute_reply":"2021-08-22T14:05:41.050293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wave = np.load(train_files[0])","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:05:41.144737Z","iopub.execute_input":"2021-08-22T14:05:41.14535Z","iopub.status.idle":"2021-08-22T14:05:41.161688Z","shell.execute_reply.started":"2021-08-22T14:05:41.145296Z","shell.execute_reply":"2021-08-22T14:05:41.160876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(dpi=120)\nplt.plot(range(len(wave[0])), np.abs(shannon_wavelets(min_max_scaler(wave)[0])))","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:05:41.326064Z","iopub.execute_input":"2021-08-22T14:05:41.326719Z","iopub.status.idle":"2021-08-22T14:05:41.506861Z","shell.execute_reply.started":"2021-08-22T14:05:41.326679Z","shell.execute_reply":"2021-08-22T14:05:41.506003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wave = np.load(train_files[0])\nplt.figure(dpi=120)\nplt.plot(range(len(wave[0])), np.abs(shannon_wavelets(\n    min_max_scaler(wave)[0], B=3)))","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:06:04.361218Z","iopub.execute_input":"2021-08-22T14:06:04.361929Z","iopub.status.idle":"2021-08-22T14:06:04.560282Z","shell.execute_reply.started":"2021-08-22T14:06:04.361887Z","shell.execute_reply":"2021-08-22T14:06:04.559199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"And let's try the original complex morlet. ","metadata":{}},{"cell_type":"code","source":"def complex_morlet(wave, B=1, C=1):\n    first_exp = np.exp(-(wave**2 / B))\n    second_exp = np.exp(1j * 2 * np.pi * C * wave)\n    return np.abs((1 / np.sqrt(np.pi * B)) * first_exp * second_exp)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T13:16:43.771346Z","iopub.execute_input":"2021-08-22T13:16:43.77171Z","iopub.status.idle":"2021-08-22T13:16:43.777246Z","shell.execute_reply.started":"2021-08-22T13:16:43.77168Z","shell.execute_reply":"2021-08-22T13:16:43.776256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wave = np.load(train_files[0])\nplt.figure(dpi=120)\nplt.plot(range(len(wave[0])), np.abs(complex_morlet(min_max_scaler(wave)[0])))","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:06:14.931623Z","iopub.execute_input":"2021-08-22T14:06:14.932081Z","iopub.status.idle":"2021-08-22T14:06:15.126621Z","shell.execute_reply.started":"2021-08-22T14:06:14.932041Z","shell.execute_reply":"2021-08-22T14:06:15.125154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wave = np.load(train_files[0])\nplt.figure(dpi=120)\nplt.plot(range(len(wave[0])), complex_morlet(min_max_scaler(wave)[0], B=20))","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:06:23.062335Z","iopub.execute_input":"2021-08-22T14:06:23.062778Z","iopub.status.idle":"2021-08-22T14:06:23.269626Z","shell.execute_reply.started":"2021-08-22T14:06:23.062728Z","shell.execute_reply":"2021-08-22T14:06:23.26837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Try Complex Gaussian Derivative Wavelets","metadata":{}},{"cell_type":"code","source":"def cgauss_deriv(wave, C=1):\n    const = C * np.exp(-(wave**2))\n    real = np.cos(-wave) * const\n    imag = np.sin(-wave) * const\n    return np.sqrt(real**2 + imag**2)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:06:29.030003Z","iopub.execute_input":"2021-08-22T14:06:29.030697Z","iopub.status.idle":"2021-08-22T14:06:29.037983Z","shell.execute_reply.started":"2021-08-22T14:06:29.03062Z","shell.execute_reply":"2021-08-22T14:06:29.036311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wave = np.load(train_files[0])\nplt.figure(dpi=120)\nplt.plot(range(len(wave[0])), cgauss_deriv(min_max_scaler(wave)[0], C=1))","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:06:29.526958Z","iopub.execute_input":"2021-08-22T14:06:29.527323Z","iopub.status.idle":"2021-08-22T14:06:29.735811Z","shell.execute_reply.started":"2021-08-22T14:06:29.527294Z","shell.execute_reply":"2021-08-22T14:06:29.734679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Try Mexican hat Wavelet","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:08:28.134872Z","iopub.execute_input":"2021-08-22T14:08:28.135291Z","iopub.status.idle":"2021-08-22T14:08:28.142409Z","shell.execute_reply.started":"2021-08-22T14:08:28.135255Z","shell.execute_reply":"2021-08-22T14:08:28.140817Z"}}},{"cell_type":"code","source":"def mexican_hat(wave):\n    const = 2 / (math.sqrt(3) * (np.pi**0.25))\n    other_term = (1 - wave**2) * np.exp(-(wave**2 / 2))\n    return const * other_term","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:13:01.519647Z","iopub.execute_input":"2021-08-22T14:13:01.520217Z","iopub.status.idle":"2021-08-22T14:13:01.527383Z","shell.execute_reply.started":"2021-08-22T14:13:01.520173Z","shell.execute_reply":"2021-08-22T14:13:01.526091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wave = np.load(train_files[0])\nplt.figure(dpi=120)\nplt.plot(range(len(wave[0])), mexican_hat(min_max_scaler(wave)[0]))","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:13:37.1362Z","iopub.execute_input":"2021-08-22T14:13:37.13661Z","iopub.status.idle":"2021-08-22T14:13:37.319713Z","shell.execute_reply.started":"2021-08-22T14:13:37.136575Z","shell.execute_reply":"2021-08-22T14:13:37.318757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's look at frequency B-Spline Wavelets","metadata":{}},{"cell_type":"code","source":"def b_spline_opt(wave, M=2, B=1, C=1):\n    frac_term = ((np.sin(np.pi * B * wave / M) / (np.pi * B * wave / M)) ** M) * np.sqrt(B)\n    real = np.cos(2 * np.pi * C * wave) * frac_term\n    imag = np.sin(2 * np.pi * C * wave) * frac_term\n    return np.sqrt(real**2 + imag**2)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T13:58:29.115455Z","iopub.execute_input":"2021-08-22T13:58:29.115916Z","iopub.status.idle":"2021-08-22T13:58:29.123158Z","shell.execute_reply.started":"2021-08-22T13:58:29.115876Z","shell.execute_reply":"2021-08-22T13:58:29.121755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Looks like complex morlet does the reinforcement for us, so we'll stick to that for now. We will rewrite the function. Particularly, split the complex exponentials up into cosine and sine will speed up calculations. ","metadata":{}},{"cell_type":"code","source":"def shannon_wavelets_opt(wave, B=1, C=1):\n    frac_term = (np.sin(np.pi * B * wave) / (np.pi * B * wave)) * np.sqrt(B)\n    real = np.cos(2 * np.pi * C * wave) * frac_term\n    imag = np.sin(2 * np.pi * C * wave) * frac_term\n    return np.sqrt(real**2 + imag**2)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T13:53:09.181617Z","iopub.execute_input":"2021-08-22T13:53:09.182239Z","iopub.status.idle":"2021-08-22T13:53:09.189118Z","shell.execute_reply.started":"2021-08-22T13:53:09.182203Z","shell.execute_reply":"2021-08-22T13:53:09.187868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def complex_morlet_opt(wave, B=1, C=1):\n    first_exp = np.exp(-(wave**2 / B)) / np.sqrt(np.pi * B)\n    real = first_exp * np.cos(2 * np.pi * C * wave)\n    imag = first_exp * np.sin(2 * np.pi * C * wave)\n    return np.sqrt(real**2 + imag**2)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T13:15:04.721586Z","iopub.execute_input":"2021-08-22T13:15:04.722037Z","iopub.status.idle":"2021-08-22T13:15:04.729118Z","shell.execute_reply.started":"2021-08-22T13:15:04.721999Z","shell.execute_reply":"2021-08-22T13:15:04.727782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%timeit complex_morlet(wave[0], B=20)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T13:16:55.199381Z","iopub.execute_input":"2021-08-22T13:16:55.199751Z","iopub.status.idle":"2021-08-22T13:16:58.551097Z","shell.execute_reply.started":"2021-08-22T13:16:55.19972Z","shell.execute_reply":"2021-08-22T13:16:58.550005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%timeit complex_morlet_opt(wave[0], B=20)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T13:15:04.921252Z","iopub.execute_input":"2021-08-22T13:15:04.921598Z","iopub.status.idle":"2021-08-22T13:15:07.237134Z","shell.execute_reply.started":"2021-08-22T13:15:04.921568Z","shell.execute_reply":"2021-08-22T13:15:07.235743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Assert that it is the same. We would not use real assert function but plot and see that it is \"about the same\" since value will slightly differ most likely. ","metadata":{}},{"cell_type":"code","source":"plt.figure(dpi=120)\nplt.plot(range(len(wave[0])), complex_morlet_opt(min_max_scaler(wave)[0], B=20))","metadata":{"execution":{"iopub.status.busy":"2021-08-22T13:19:24.667528Z","iopub.execute_input":"2021-08-22T13:19:24.667904Z","iopub.status.idle":"2021-08-22T13:19:24.887459Z","shell.execute_reply.started":"2021-08-22T13:19:24.667873Z","shell.execute_reply":"2021-08-22T13:19:24.886162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Okay let's continue building a function. ","metadata":{}},{"cell_type":"code","source":"%timeit _ = np.ascontiguousarray(wave)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T13:23:07.472143Z","iopub.execute_input":"2021-08-22T13:23:07.472542Z","iopub.status.idle":"2021-08-22T13:23:16.637183Z","shell.execute_reply.started":"2021-08-22T13:23:07.472508Z","shell.execute_reply":"2021-08-22T13:23:16.635918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fmin = 21.83\nfmax = 350\ndef apply_qtransform(waves, \n                     transform=CQT2010v2(sr=2048, fmin=fmin, fmax=fmax, n_bins=32, hop_length=64), \n                     cuda=False, order=4):\n    waves = butter_bandpass_filter(waves, fmin, fmax, fs, order)\n    waves = min_max_scaler(waves)\n#     waves = complex_morlet_opt(waves, B=21)\n    waves = b_spline_opt(waves, M=order, B=35)\n    waves = np.hstack(waves)\n#     waves = np.ascontiguousarray(waves)\n    waves = torch.from_numpy(waves).float()\n    if cuda: waves = waves.cuda()\n    image = transform(waves)\n    image[torch.isnan(image)] = 0\n    return image","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:22:14.551696Z","iopub.execute_input":"2021-08-22T14:22:14.552144Z","iopub.status.idle":"2021-08-22T14:22:14.569496Z","shell.execute_reply.started":"2021-08-22T14:22:14.552106Z","shell.execute_reply":"2021-08-22T14:22:14.568211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = apply_qtransform(np.load(train_files[0]))\nimg.shape","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:21:00.644099Z","iopub.execute_input":"2021-08-22T14:21:00.644481Z","iopub.status.idle":"2021-08-22T14:21:00.678678Z","shell.execute_reply.started":"2021-08-22T14:21:00.644449Z","shell.execute_reply":"2021-08-22T14:21:00.677284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:21:00.946331Z","iopub.execute_input":"2021-08-22T14:21:00.946688Z","iopub.status.idle":"2021-08-22T14:21:00.955771Z","shell.execute_reply.started":"2021-08-22T14:21:00.946659Z","shell.execute_reply":"2021-08-22T14:21:00.954698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(dpi=80)\nplt.imshow(img.squeeze())","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:21:00.95736Z","iopub.execute_input":"2021-08-22T14:21:00.957903Z","iopub.status.idle":"2021-08-22T14:21:01.139695Z","shell.execute_reply.started":"2021-08-22T14:21:00.957868Z","shell.execute_reply":"2021-08-22T14:21:01.138429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(1, 11):\n    img = apply_qtransform(np.load(train_files[0]), order=i)\n    plt.figure(dpi=80)\n    plt.imshow(img.squeeze())","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:21:20.147832Z","iopub.execute_input":"2021-08-22T14:21:20.148221Z","iopub.status.idle":"2021-08-22T14:21:22.010801Z","shell.execute_reply.started":"2021-08-22T14:21:20.148191Z","shell.execute_reply":"2021-08-22T14:21:22.009709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(1, 11):\n    img = apply_qtransform(np.load(train_files[i]))\n    plt.figure(dpi=80)\n    plt.imshow(img.squeeze())","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:21:47.773081Z","iopub.execute_input":"2021-08-22T14:21:47.77355Z","iopub.status.idle":"2021-08-22T14:21:49.600784Z","shell.execute_reply.started":"2021-08-22T14:21:47.77351Z","shell.execute_reply":"2021-08-22T14:21:49.599146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:21:56.156121Z","iopub.execute_input":"2021-08-22T14:21:56.156492Z","iopub.status.idle":"2021-08-22T14:21:56.751271Z","shell.execute_reply.started":"2021-08-22T14:21:56.15646Z","shell.execute_reply":"2021-08-22T14:21:56.750138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files[0].name[:-4]","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:20:17.122553Z","iopub.execute_input":"2021-08-22T14:20:17.123215Z","iopub.status.idle":"2021-08-22T14:20:17.131403Z","shell.execute_reply.started":"2021-08-22T14:20:17.123158Z","shell.execute_reply":"2021-08-22T14:20:17.130105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imsave(\"test.jpg\", img.squeeze())","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:23:14.836036Z","iopub.execute_input":"2021-08-22T14:23:14.8364Z","iopub.status.idle":"2021-08-22T14:23:14.846869Z","shell.execute_reply.started":"2021-08-22T14:23:14.83637Z","shell.execute_reply":"2021-08-22T14:23:14.845405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_images(file_path, out_dir):\n    file_name = file_path.name[:-4]\n    waves = np.load(file_path).astype(np.float32)  # (3, 4096)\n    image = apply_qtransform(waves)\n    plt.imsave(out_dir + file_name + \".jpg\", image.squeeze())","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:23:50.151446Z","iopub.execute_input":"2021-08-22T14:23:50.151852Z","iopub.status.idle":"2021-08-22T14:23:50.15795Z","shell.execute_reply.started":"2021-08-22T14:23:50.151819Z","shell.execute_reply":"2021-08-22T14:23:50.156687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = pd.read_csv(\"../input/g2net-gravitational-wave-detection/training_labels.csv\")\nlabels[\"file_path\"] = train_files\npd.set_option(\"display.max_colwidth\", None)\nlabels.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:25:22.554721Z","iopub.execute_input":"2021-08-22T14:25:22.555215Z","iopub.status.idle":"2021-08-22T14:25:23.168726Z","shell.execute_reply.started":"2021-08-22T14:25:22.55518Z","shell.execute_reply":"2021-08-22T14:25:23.167596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ones_train = labels[labels[\"target\"] == 1][\"file_path\"].to_numpy()\n# ones_train[0]\n\n# folder_name = \"train/ones/\"\n# os.makedirs(folder_name, exist_ok=True)\n\n# _ = joblib.Parallel(n_jobs=8, prefer=\"threads\")(\n#     joblib.delayed(save_images)(file_path, out_dir=folder_name) for file_path in tqdm(ones_train)\n# )","metadata":{"execution":{"iopub.status.busy":"2021-08-22T14:27:03.355063Z","iopub.execute_input":"2021-08-22T14:27:03.355444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# folder_name = \"train/zero/\"\n# zeroes_train = labels[labels[\"target\"] == 0][\"file_path\"].to_numpy()\n\n# os.makedirs(folder_name, exist_ok=True)\n\n# _ = joblib.Parallel(n_jobs=8, prefer=\"threads\")(\n#     joblib.delayed(save_images)(file_path, out_dir=folder_name) for file_path in tqdm(zeroes_train)\n# )","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def move_to_destination(origin, destination, percentage_split):\n#     num_images = int(len(os.listdir(origin))*percentage_split)\n#     for image_name, image_number in zip(sorted(os.listdir(origin)), range(num_images)):\n#         shutil.move(os.path.join(origin, image_name), destination)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# os.makedirs(\"./valid/ones\")\n# os.makedirs(\"./valid/zero\")\n# move_to_destination(\"./train/ones\", \"./valid/ones\", 0.2)\n# move_to_destination(\"./train/zero\", \"./valid/zero\", 0.2)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"OUT_DIR = \"train/\"\nos.mkdir(\"train/\")\n\n_ = joblib.Parallel(n_jobs=8, prefer=\"threads\")(\n    joblib.delayed(save_images)(file_path, out_dir=OUT_DIR) for file_path in tqdm(train_files)\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nshutil.make_archive(\"train/\", 'zip', \"train/\")\nshutil.rmtree(\"train/\")\n\n# shutil.make_archive(\"valid/\", \"zip\", \"valid/\")\n# shutil.rmtree(\"valid/\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"OUT_DIR = \"test/\"\nos.mkdir(\"test/\")\ntest_files = [Path(path) for path in sorted(glob.glob(\"../input/g2net-gravitational-wave-detection/test/*/*/*/*.npy\"))]\n\n_ = joblib.Parallel(n_jobs=8, prefer=\"threads\")(\n    joblib.delayed(save_images)(file_path, out_dir=OUT_DIR) for file_path in tqdm(test_files)\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nshutil.make_archive(\"test/\", 'zip', \"test/\")\nshutil.rmtree(\"test/\")","metadata":{},"execution_count":null,"outputs":[]}]}