{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from utils import *\nfrom scipy import signal\nfrom pathlib import Path\nfrom functools import partial\nimport numpy.fft as fft\nplt.rcParams['legend.title_fontsize'] = 'small'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-09-24T16:46:23.687377Z","iopub.execute_input":"2021-09-24T16:46:23.687688Z","iopub.status.idle":"2021-09-24T16:46:26.195763Z","shell.execute_reply.started":"2021-09-24T16:46:23.687610Z","shell.execute_reply":"2021-09-24T16:46:26.194754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_PATH = Path(\"../input/g2net-gravitational-wave-detection/\")\ntrain_df = pd.read_csv(os.path.join(DATA_PATH, 'training_labels.csv'))","metadata":{"execution":{"iopub.status.busy":"2021-09-24T16:46:26.198240Z","iopub.execute_input":"2021-09-24T16:46:26.198750Z","iopub.status.idle":"2021-09-24T16:46:26.613811Z","shell.execute_reply.started":"2021-09-24T16:46:26.198702Z","shell.execute_reply":"2021-09-24T16:46:26.613154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_path(id_, data_path=None, folder=\"train\"):\n    data_path = Path(DATA_PATH) if data_path is None else Path(data_path)\n    return data_path / folder / id_[0] / id_[1] / id_[2] / f\"{id_}.npy\"\n\ndef load_file(id_, data_path=None, folder=\"train\", torch=False):\n    path = get_path(id_)\n    waves = np.load(path)\n    return waves","metadata":{"execution":{"iopub.status.busy":"2021-09-24T16:46:26.614858Z","iopub.execute_input":"2021-09-24T16:46:26.615613Z","iopub.status.idle":"2021-09-24T16:46:26.622576Z","shell.execute_reply.started":"2021-09-24T16:46:26.615571Z","shell.execute_reply":"2021-09-24T16:46:26.621805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df['file_path'] = train_df['id'].apply(partial(get_path, folder=\"train\"))\ntrain_df = pd.read_csv(\"../input/g2net-mapping-id-to-file-path/training_labels_with_paths.csv\")\ntrain_df = train_df.rename(columns={\"filepath\": \"file_path\"})","metadata":{"execution":{"iopub.status.busy":"2021-09-24T16:46:26.623763Z","iopub.execute_input":"2021-09-24T16:46:26.624680Z","iopub.status.idle":"2021-09-24T16:46:28.280763Z","shell.execute_reply.started":"2021-09-24T16:46:26.624639Z","shell.execute_reply":"2021-09-24T16:46:28.280057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-24T16:46:28.282908Z","iopub.execute_input":"2021-09-24T16:46:28.283846Z","iopub.status.idle":"2021-09-24T16:46:28.302300Z","shell.execute_reply.started":"2021-09-24T16:46:28.283801Z","shell.execute_reply":"2021-09-24T16:46:28.301456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2D cnn OOF","metadata":{}},{"cell_type":"code","source":"oof = pd.read_csv(\"../input/g2net-efficientnet-b7-baseline-training/oof_df.csv\")\noof_preds_avg = oof.preds.values.mean()\n\nTHRESH = oof_preds_avg\noof['predictions'] = (oof.preds > THRESH).astype(int)\noof['true_negative'] = (oof.predictions == 0) & (oof.target == 0)\noof['true_positive'] = (oof.predictions == 1) & (oof.target == 1)\noof['false_negative'] = (oof.predictions == 0) & (oof.target == 1)\noof['false_positive'] = (oof.predictions == 1) & (oof.target == 0)\nid_false_negative = oof[oof.false_negative].id.values\nid_false_positive = oof[oof.false_positive].id.values\nid_true_negative = oof[oof.true_negative].id.values\nid_true_positive = oof[oof.true_positive].id.values","metadata":{"execution":{"iopub.status.busy":"2021-09-24T16:46:28.303506Z","iopub.execute_input":"2021-09-24T16:46:28.304084Z","iopub.status.idle":"2021-09-24T16:46:28.680632Z","shell.execute_reply.started":"2021-09-24T16:46:28.304033Z","shell.execute_reply":"2021-09-24T16:46:28.679995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(oof))\nprint(len(id_false_negative))\nprint(len(id_false_positive))\nprint(len(id_true_negative))\nprint(len(id_true_positive))","metadata":{"execution":{"iopub.status.busy":"2021-09-24T16:46:28.681528Z","iopub.execute_input":"2021-09-24T16:46:28.681837Z","iopub.status.idle":"2021-09-24T16:46:28.688995Z","shell.execute_reply.started":"2021-09-24T16:46:28.681811Z","shell.execute_reply":"2021-09-24T16:46:28.688095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.random.seed(1127802825)\nlist_fn = np.random.choice(id_false_negative, size=40)\nlist_fp = np.random.choice(id_false_positive, size=40)\nlist_tn = np.random.choice(id_true_negative, size=40)\nlist_tp = np.random.choice(id_true_positive, size=40)","metadata":{"execution":{"iopub.status.busy":"2021-09-24T16:46:28.691118Z","iopub.execute_input":"2021-09-24T16:46:28.692053Z","iopub.status.idle":"2021-09-24T16:46:28.710482Z","shell.execute_reply.started":"2021-09-24T16:46:28.692006Z","shell.execute_reply":"2021-09-24T16:46:28.709490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_asym_tukey_window(M, alpha=0.7):\n    n = np.arange(0, M)\n    width_1 = int(np.floor(alpha*(M-1)))\n    width_2 = int(np.floor((1-alpha)*(M-1)/2.0))\n    n1 = n[0:width_1+1]\n    assert width_1+1 <= M-width_2-1\n    n2 = n[width_1+1:M-width_2-1]\n    n3 = n[M-width_2-1:]\n\n    w1 = 0.5 * (1 + np.cos(np.pi * (-1 + 1.0*n1/alpha/(M-1))))\n    w2 = np.ones(n2.shape)\n    w3 = 0.5 * (1 + np.cos(np.pi * (-2.0/(1-alpha) + 1 + 2.0*n3/(1-alpha)/(M-1))))\n\n    w = np.concatenate((w1, w2, w3))\n    return w\n\n\ndef get_asym_cosine_window(M, alpha):\n    n = np.arange(0, M)\n    center = int(np.floor(alpha*(M-1)))\n    n1 = n[0:center+1]\n    n2 = n[center+1:]\n    w1 = 0.5 * (1 + np.cos(np.pi * (-1 + 1*n1/alpha/(M-1))))\n    w2 = 0.5 * (1 + np.cos(np.pi * (-1.0/(1-alpha) + 1 + n2/(1-alpha)/(M-1))))\n    return np.concatenate([w1, w2])\n\nw = get_asym_tukey_window(4096, 0.7)\nplt.plot(w);","metadata":{"execution":{"iopub.status.busy":"2021-09-24T16:46:28.711817Z","iopub.execute_input":"2021-09-24T16:46:28.712412Z","iopub.status.idle":"2021-09-24T16:46:28.991885Z","shell.execute_reply.started":"2021-09-24T16:46:28.712377Z","shell.execute_reply":"2021-09-24T16:46:28.991013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SCALING = [1.5e-20, 1.5e-20, 0.5e-20]\n\ndef load_file(id_, data_path=None, folder=\"train\"):\n    data_path = Path(DATA_PATH) if data_path is None else Path(data_path)\n    path = data_path / folder / id_[0] / id_[1] / id_[2] / f\"{id_}.npy\"\n    waves = np.load(path)\n    return waves\n\ndef bandpass(x, lf=20, hf=1000,\n             order=8, sr=2048,\n             btype='bandpass',\n             window='tukey', alpha=0.2):\n    '''\n    Cell 33 of https://www.gw-openscience.org/LVT151012data/LOSC_Event_tutorial_LVT151012.html\n    https://scipy-cookbook.readthedocs.io/items/ButterworthBandpass.html\n    '''\n    sos = signal.butter(order, [lf, hf], btype=\"bandpass\",\n                        output=\"sos\", fs=sr)\n    normalization = np.sqrt((hf - lf) / (sr / 2))\n    if window is not None:\n        if window in ['hann', 'hamming']:\n            window = signal.get_window(window, 2*sr)\n        elif window == 'tukey':\n            window = signal.tukey(2*sr, alpha)\n        elif window == 'asymtukey':\n            window = get_asym_tukey_window(2*sr, alpha)\n        elif window == 'asymcosine':\n            window = get_asym_cosine_window(2*sr, alpha)\n        else:\n            raise NotImplementedError.gourp\n        x *= window\n\n    if x.ndim == 2:\n        for i in range(3):\n            x[i] = signal.sosfilt(sos, x[i]) * normalization\n    elif x.ndim == 3:  # batch\n        for i in range(x.shape[0]):\n            for j in range(3):\n                x[i, j] = signal.sosfilt(sos, x[i, j]) * normalization\n    return x\n\ndef notch_filter(x,\n                 f0=306,  # Frequency to be removed from signal (Hz)\n                 sr=2048, # Sample frequency (Hz)\n                 Q=100.0, # Quality factor\n                 window='tukey', alpha=0.2):\n\n    # Design notch filter\n    b, a = signal.iirnotch(f0, Q, sr)\n    if window is not None:\n        if window in ['hann', 'hamming']:\n            window = signal.get_window(window, 2*sr)\n        elif window == 'tukey':\n            window = signal.tukey(2*sr, alpha)\n        elif window == 'asymtukey':\n            window = get_asym_tukey_window(2*sr, alpha)\n        elif window == 'asymcosine':\n            window = get_asym_cosine_window(2*sr, alpha)\n        else:\n            raise NotImplementedError\n        x *= window\n\n    if x.ndim == 2:\n        for i in range(3):\n            x[i] = signal.filtfilt(b, a, x[i], padlen=0)\n    elif x.ndim == 3:  # batch\n        for i in range(x.shape[0]):\n            for j in range(3):\n                x[i, j] = signal.filtfilt(b, a, x[i, j], padlen=0)\n    return x\n\ndef visualize_sample(\n    id_, idx=1,\n    colors=(\"darkcyan\", \"darkorange\", \"darkviolet\"),\n    signal_names=(\"LIGO Hanford\", \"LIGO Livingston\", \"Virgo\"),\n    bandpass_params={},\n    notch_params={},\n    fig_kwargs={},\n    plot_kwargs={}\n):\n    \"\"\"\n    adapted from https://www.kaggle.com/ihelon/g2net-eda-and-modeling\n    \"\"\"\n    if not fig_kwargs: fig_kwargs=dict(figsize=(25, 15))\n    if not plot_kwargs: plot_kwargs=dict(linewidth=0.6)\n\n    x = load_file(id_)  # (3, 4096)\n    target = train_df[train_df.id == id_][\"target\"].values[0]\n    try:\n        preds = oof[oof.id == id_][\"preds\"].values[0]\n    except:\n        preds = 0.5\n    \n    bp_applied = \"\"\n    if bandpass_params:\n        x /= np.array(SCALING).reshape(-1, 1)\n        x = bandpass(x, **bandpass_params)\n        x /= np.array([2e-2, 2e-2, 1e-1]).reshape(-1, 1)\n        bp_applied = f\"| bandpass lf={bandpass_params['lf']} hf={bandpass_params['hf']}\"\n        print(\"\\nBandpass is applied\")\n    \n    notch_applied = \"\"\n    if notch_params:\n        x = notch_filter(x, **notch_params)\n        notch_applied = f\"| notch filter f0={notch_params['f0']} with Q={notch_params['Q']}\"\n        print(\"\\nNotch filter is applied\")\n\n    fig, axes = plt.subplots(4, 1, **fig_kwargs)\n    fig.tight_layout(rect=[0, 0.03, 1, 0.95])\n    plt.suptitle(f\"id: {id_} |  target: {target} | preds: {preds:.4f}\", fontsize=24)\n    plt.subplots_adjust(hspace = 0.4)\n    \n    for i, ax in enumerate(axes):\n        if i==3: break\n        ax.plot(x[i], color=colors[i], **plot_kwargs)\n        ax.legend([signal_names[i]], fontsize=20, loc=\"lower right\")\n        axes[3].plot(x[i], color=colors[i])\n        ax.autoscale(enable=True, axis='x', tight=True)\n        \n    plt.show()\n    \n    fig, axes = plt.subplots(4, 1, **fig_kwargs)\n    fig.tight_layout(rect=[0, 0.03, 1, 0.95])\n    plt.suptitle(f\"id: {id_} at {signal_names[idx]} |  target: {target} | preds: {preds:.4f}\", fontsize=24)\n    plt.subplots_adjust(hspace = 0.4)\n\n    wave_bp = x[idx]\n    f, Pxx_den = signal.welch(wave_bp, 2048, nperseg=2048)\n    axes[0].plot(f, Pxx_den, color='slateblue', **plot_kwargs)\n    #     axes[3].set_xlabel('frequency [Hz]')\n    axes[0].set_ylabel('PSD [V**2/Hz]')\n    axes[0].set_title(f\"Welch power spectrum linear scale {bp_applied} {notch_applied} \")\n\n    x_ft = fft.rfft(wave_bp, n=4096)\n    freq = fft.rfftfreq(4096)\n    x_ft_mag = x_ft.real**2 + x_ft.imag**2\n\n    ft_max = np.argmax(x_ft_mag[500:700])\n    xticks = np.arange(0, 2048, 256).tolist()\n    xticks.insert((500+ft_max)//256+1, 500+ft_max)\n    axes[1].plot(x_ft_mag, color=\"darkcyan\", **plot_kwargs)\n    axes[1].set_xticks(xticks)\n    axes[1].set_title(f\"FFT mag linear scale {bp_applied} {notch_applied} \")\n\n    axes[2].plot(freq, x_ft.real, color= \"darkorange\", **plot_kwargs)\n    axes[2].set_title(f\"FFT real part linear scale {bp_applied} {notch_applied} \")\n\n    axes[3].plot(freq, x_ft.imag, color= \"darkviolet\", **plot_kwargs)\n    axes[3].set_title(f\"FFT imag part linear scale {bp_applied} {notch_applied} \")\n\n    for ax in axes:\n        ax.autoscale(enable=True, axis='x', tight=True)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-24T16:54:06.454609Z","iopub.execute_input":"2021-09-24T16:54:06.454924Z","iopub.status.idle":"2021-09-24T16:54:06.485134Z","shell.execute_reply.started":"2021-09-24T16:54:06.454890Z","shell.execute_reply":"2021-09-24T16:54:06.484403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize_sample_spectrum(id_, idx=1,\n                              bandpass_params={}):\n    x = load_file(id_)  # (3, 4096)\n    x /= np.array(SCALING).reshape(-1, 1)\n    wave = x[idx]\n    \n    signal_names=(\"LIGO Hanford\", \"LIGO Livingston\", \"Virgo\")\n    target = train_df[train_df.id == id_][\"target\"].values[0]\n    try:\n        preds = oof[oof.id == id_][\"preds\"].values[0]\n    except:\n        preds = 0.5\n    \n    fig, axes = plt.subplots(7, 1, figsize=(25, 30))\n    fig.tight_layout(rect=[0, 0.03, 1, 0.95])\n    plt.suptitle(f\"id: {id_} at {signal_names[idx]} |  target: {target} | preds: {preds:.4f}\", fontsize=24)\n    \n    plt.subplots_adjust(hspace = 0.4)\n    f, Pxx_spec = signal.periodogram(wave, 2048, 'tukey', scaling='spectrum')\n\n    axes[0].semilogy(f, np.sqrt(Pxx_spec), color='darkgreen')\n#     axes[0].set_xlabel('frequency [Hz]')\n    axes[0].set_ylabel('Spectrum [V RMS]')\n    axes[0].set_title(\"Periodogram log scale, no bandpass\")\n    \n    x_ft = fft.rfft(wave, n=4096)\n    freq = fft.rfftfreq(4096)\n    axes[1].semilogy(freq, x_ft.real**2 + x_ft.imag**2, color='mediumseagreen')\n    axes[1].set_title(\"FFT mag log scale, no bandpass\")\n    \n    x_bp = bandpass(x, **bandpass_params)\n    wave_bp = x_bp[idx]\n    f, Pxx_spec = signal.periodogram(wave_bp, 2048, 'tukey', scaling='spectrum')\n    axes[2].plot(f, np.sqrt(Pxx_spec), color='darkorange')\n#     axes[2].set_xlabel('frequency [Hz]')\n    axes[2].set_ylabel('Spectrum [V RMS]')\n    axes[2].set_title(\"Periodogram power spectrum linear scale, bandpass\")\n    \n    f, Pxx_den = signal.welch(wave_bp, 2048, nperseg=2048)\n    axes[3].plot(f, Pxx_den, color='slateblue')\n#     axes[3].set_xlabel('frequency [Hz]')\n    axes[3].set_ylabel('PSD [V**2/Hz]')\n    axes[3].set_title(\"Welch power spectrum linear scale, bandpass\")\n    \n    x_ft = fft.rfft(wave_bp, n=4096)\n    freq = fft.rfftfreq(4096)\n    x_ft_mag = x_ft.real**2 + x_ft.imag**2\n    ft_max = np.argmax(x_ft_mag[500:700])\n    xticks = np.arange(0, 2048, 256).tolist()\n    xticks.insert((500+ft_max)//256+1, 500+ft_max)\n    axes[4].plot(x_ft_mag, color=\"darkcyan\")\n    axes[4].set_xticks(xticks)\n    axes[4].set_title(\"FFT mag linear scale, bandpass\")\n    \n    axes[5].plot(freq, x_ft.real, color= \"darkorange\")\n    axes[5].set_title(\"FFT real part linear scale, bandpass\")\n    \n    axes[6].plot(freq, x_ft.imag, color= \"darkviolet\")\n    axes[6].set_title(\"FFT imag part linear scale, bandpass\")\n    \n    for ax in axes:\n        ax.autoscale(enable=True, axis='x', tight=True)\n    \n    plt.show()\n    ","metadata":{"execution":{"iopub.status.busy":"2021-09-24T16:46:29.023600Z","iopub.execute_input":"2021-09-24T16:46:29.023902Z","iopub.status.idle":"2021-09-24T16:46:29.042260Z","shell.execute_reply.started":"2021-09-24T16:46:29.023866Z","shell.execute_reply":"2021-09-24T16:46:29.041294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_sample('128762dfca', \n                 fig_kwargs={\"figsize\": (25, 20)})","metadata":{"execution":{"iopub.status.busy":"2021-09-24T16:46:29.043509Z","iopub.execute_input":"2021-09-24T16:46:29.043746Z","iopub.status.idle":"2021-09-24T16:46:31.113741Z","shell.execute_reply.started":"2021-09-24T16:46:29.043721Z","shell.execute_reply":"2021-09-24T16:46:31.112768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bandpass_params = dict(lf=35, hf=1000, order=10, window='tukey', alpha=0.1)\n# notch_params = dict(f0=306, Q=120.0, window=None)\n\nfor id_ in list_fp[:2]:\n    print(\"\\n\\n\", id_)\n    visualize_sample(id_, \n                     bandpass_params=bandpass_params,\n                     notch_params={},\n                     )","metadata":{"execution":{"iopub.status.busy":"2021-09-24T16:46:31.115177Z","iopub.execute_input":"2021-09-24T16:46:31.115501Z","iopub.status.idle":"2021-09-24T16:46:35.778820Z","shell.execute_reply.started":"2021-09-24T16:46:31.115466Z","shell.execute_reply":"2021-09-24T16:46:35.777977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2dCNN strong positive","metadata":{}},{"cell_type":"code","source":"bandpass_params = dict(lf=35, hf=1000, order=10, window='tukey', alpha=0.2)\nvisualize_sample('a173c0ac26', bandpass_params=bandpass_params)","metadata":{"execution":{"iopub.status.busy":"2021-09-24T16:46:35.781461Z","iopub.execute_input":"2021-09-24T16:46:35.781700Z","iopub.status.idle":"2021-09-24T16:46:37.900262Z","shell.execute_reply.started":"2021-09-24T16:46:35.781673Z","shell.execute_reply":"2021-09-24T16:46:37.899519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bandpass_params = dict(lf=35, hf=1000, order=10, window='tukey', alpha=0.2)\nnotch_params = dict(f0=306, Q=120.0, window=None)\nvisualize_sample('a173c0ac26', \n                 bandpass_params=bandpass_params,\n                 notch_params=notch_params)","metadata":{"execution":{"iopub.status.busy":"2021-09-24T16:52:22.458526Z","iopub.execute_input":"2021-09-24T16:52:22.458844Z","iopub.status.idle":"2021-09-24T16:52:24.863487Z","shell.execute_reply.started":"2021-09-24T16:52:22.458802Z","shell.execute_reply":"2021-09-24T16:52:24.862625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2dCNN False negatives","metadata":{}},{"cell_type":"code","source":"print(list_fn)","metadata":{"execution":{"iopub.status.busy":"2021-09-24T16:46:40.007958Z","iopub.execute_input":"2021-09-24T16:46:40.008722Z","iopub.status.idle":"2021-09-24T16:46:40.013878Z","shell.execute_reply.started":"2021-09-24T16:46:40.008690Z","shell.execute_reply":"2021-09-24T16:46:40.013048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bandpass_params = dict(lf=40, hf=1000, order=10, window='tukey', alpha=0.1)\nvisualize_sample('e964222f61', \n                 bandpass_params=bandpass_params)\n# this one is difficult, esp idx 2 Virgo","metadata":{"execution":{"iopub.status.busy":"2021-09-24T16:46:40.015276Z","iopub.execute_input":"2021-09-24T16:46:40.015475Z","iopub.status.idle":"2021-09-24T16:46:42.314361Z","shell.execute_reply.started":"2021-09-24T16:46:40.015449Z","shell.execute_reply":"2021-09-24T16:46:42.313422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bandpass_params = dict(lf=35, hf=1000, order=10, window='tukey', alpha=0.1)\nnotch_params = dict(f0=306, Q=120.0, window=None)\nvisualize_sample('e964222f61', idx=2, \n                 bandpass_params=bandpass_params,\n                 notch_params=notch_params)","metadata":{"execution":{"iopub.status.busy":"2021-09-24T16:46:42.315554Z","iopub.execute_input":"2021-09-24T16:46:42.315796Z","iopub.status.idle":"2021-09-24T16:46:44.408972Z","shell.execute_reply.started":"2021-09-24T16:46:42.315771Z","shell.execute_reply":"2021-09-24T16:46:44.408175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bandpass_params = dict(lf=35, hf=1000, order=10, window='tukey', alpha=0.1)\nvisualize_sample('404f9f1041', idx=0, \n                 bandpass_params=bandpass_params,\n                 notch_params=notch_params)","metadata":{"execution":{"iopub.status.busy":"2021-09-24T16:46:44.410249Z","iopub.execute_input":"2021-09-24T16:46:44.410470Z","iopub.status.idle":"2021-09-24T16:46:46.541233Z","shell.execute_reply.started":"2021-09-24T16:46:44.410445Z","shell.execute_reply":"2021-09-24T16:46:46.540323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2dCNN False positives","metadata":{}},{"cell_type":"code","source":"print(list_fp)","metadata":{"execution":{"iopub.status.busy":"2021-09-24T16:46:46.542623Z","iopub.execute_input":"2021-09-24T16:46:46.542860Z","iopub.status.idle":"2021-09-24T16:46:46.548085Z","shell.execute_reply.started":"2021-09-24T16:46:46.542833Z","shell.execute_reply":"2021-09-24T16:46:46.547096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bandpass_params = dict(lf=35, hf=1000, order=10, window='tukey', alpha=0.1)\nnotch_params = dict(f0=40, Q=120.0, window=None)\nvisualize_sample('17053ac746', idx=1, \n                 bandpass_params=bandpass_params,\n                 notch_params=notch_params,\n                )","metadata":{"execution":{"iopub.status.busy":"2021-09-24T16:46:46.549259Z","iopub.execute_input":"2021-09-24T16:46:46.549491Z","iopub.status.idle":"2021-09-24T16:46:48.826923Z","shell.execute_reply.started":"2021-09-24T16:46:46.549465Z","shell.execute_reply":"2021-09-24T16:46:48.826078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## OOF from a 1D CNN model","metadata":{}},{"cell_type":"code","source":"oof_024 = pd.read_csv(\"../input/g2net-models/cnn1d_oof_df_fold_024.csv\")\noof_13 = pd.read_csv(\"../input/g2net-models/cnn1d_oof_df_fold_13.csv\")\noof = pd.concat([oof_024, oof_13]).sort_values('id')\noof_preds_avg = oof.preds.values.mean()\n\nTHRESH = oof_preds_avg\noof['predictions'] = (oof.preds > THRESH).astype(int)\noof['true_negative'] = (oof.predictions == 0) & (oof.target == 0)\noof['true_positive'] = (oof.predictions == 1) & (oof.target == 1)\noof['false_negative'] = (oof.predictions == 0) & (oof.target == 1)\noof['false_positive'] = (oof.predictions == 1) & (oof.target == 0)\nid_false_negative = oof[oof.false_negative].id.values\nid_false_positive = oof[oof.false_positive].id.values\nid_true_negative = oof[oof.true_negative].id.values\nid_true_positive = oof[oof.true_positive].id.values","metadata":{"execution":{"iopub.status.busy":"2021-09-24T16:46:48.828102Z","iopub.execute_input":"2021-09-24T16:46:48.828354Z","iopub.status.idle":"2021-09-24T16:46:50.023239Z","shell.execute_reply.started":"2021-09-24T16:46:48.828326Z","shell.execute_reply":"2021-09-24T16:46:50.022348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(oof))\nprint(len(id_false_negative))\nprint(len(id_false_positive))\nprint(len(id_true_negative))\nprint(len(id_true_positive))","metadata":{"execution":{"iopub.status.busy":"2021-09-24T16:46:50.024785Z","iopub.execute_input":"2021-09-24T16:46:50.025108Z","iopub.status.idle":"2021-09-24T16:46:50.031488Z","shell.execute_reply.started":"2021-09-24T16:46:50.025066Z","shell.execute_reply":"2021-09-24T16:46:50.030528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.random.seed(1127802825)\nlist_fn = np.random.choice(id_false_negative, size=40)\nlist_fp = np.random.choice(id_false_positive, size=40)\nlist_tn = np.random.choice(id_true_negative, size=40)\nlist_tp = np.random.choice(id_true_positive, size=40)","metadata":{"execution":{"iopub.status.busy":"2021-09-24T16:46:50.032762Z","iopub.execute_input":"2021-09-24T16:46:50.033047Z","iopub.status.idle":"2021-09-24T16:46:50.044104Z","shell.execute_reply.started":"2021-09-24T16:46:50.033012Z","shell.execute_reply":"2021-09-24T16:46:50.043320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## False negative for 1d cnn","metadata":{}},{"cell_type":"code","source":"print(list_fn)","metadata":{"execution":{"iopub.status.busy":"2021-09-24T16:46:50.045255Z","iopub.execute_input":"2021-09-24T16:46:50.046129Z","iopub.status.idle":"2021-09-24T16:46:50.057319Z","shell.execute_reply.started":"2021-09-24T16:46:50.046089Z","shell.execute_reply":"2021-09-24T16:46:50.056333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# bandpass_params = dict(lf=35, hf=1000, order=10, window='tukey', alpha=0.1)\nbandpass_params = dict(lf=35, hf=1000, order=10, window='asymtukey', alpha=0.8)\nvisualize_sample('899ca1132c', idx=1, \n                 bandpass_params=bandpass_params,\n                 notch_params=dict(f0=306, Q=120.0, window=None),\n                )","metadata":{"execution":{"iopub.status.busy":"2021-09-24T16:46:50.058752Z","iopub.execute_input":"2021-09-24T16:46:50.059179Z","iopub.status.idle":"2021-09-24T16:46:52.343110Z","shell.execute_reply.started":"2021-09-24T16:46:50.059139Z","shell.execute_reply":"2021-09-24T16:46:52.342263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## False positive for 1d cnn","metadata":{}},{"cell_type":"code","source":"print(list_fp)","metadata":{"execution":{"iopub.status.busy":"2021-09-24T16:46:52.344311Z","iopub.execute_input":"2021-09-24T16:46:52.344544Z","iopub.status.idle":"2021-09-24T16:46:52.349645Z","shell.execute_reply.started":"2021-09-24T16:46:52.344518Z","shell.execute_reply":"2021-09-24T16:46:52.348501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bandpass_params = dict(lf=35, hf=1000, order=10, window='tukey', alpha=0.15)\n# bandpass_params = dict(lf=35, hf=1000, order=10, window='asymtukey', alpha=0.8)\nvisualize_sample('c7fd56bb06', idx=1, \n                 bandpass_params=bandpass_params,\n                 notch_params=dict(f0=306, Q=90.0, window=None),\n                )","metadata":{"execution":{"iopub.status.busy":"2021-09-24T16:56:11.959053Z","iopub.execute_input":"2021-09-24T16:56:11.959363Z","iopub.status.idle":"2021-09-24T16:56:14.217908Z","shell.execute_reply.started":"2021-09-24T16:56:11.959328Z","shell.execute_reply":"2021-09-24T16:56:14.217233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# bandpass_params = dict(lf=35, hf=1000, order=10, window='tukey', alpha=0.1)\nbandpass_params = dict(lf=35, hf=1000, order=10, window='asymtukey', alpha=0.8)\nvisualize_sample('c92bc5ce6b', idx=1, \n                 bandpass_params=bandpass_params,\n                 notch_params=dict(f0=306, Q=120.0, window=None),\n                )","metadata":{"execution":{"iopub.status.busy":"2021-09-24T16:46:52.350893Z","iopub.execute_input":"2021-09-24T16:46:52.351173Z","iopub.status.idle":"2021-09-24T16:46:54.619451Z","shell.execute_reply.started":"2021-09-24T16:46:52.351089Z","shell.execute_reply":"2021-09-24T16:46:54.618557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## True negative","metadata":{}},{"cell_type":"code","source":"print(list_tn)","metadata":{"execution":{"iopub.status.busy":"2021-09-24T16:46:54.620528Z","iopub.execute_input":"2021-09-24T16:46:54.620742Z","iopub.status.idle":"2021-09-24T16:46:54.626342Z","shell.execute_reply.started":"2021-09-24T16:46:54.620717Z","shell.execute_reply":"2021-09-24T16:46:54.625541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# bandpass_params = dict(lf=35, hf=1000, order=10, window='tukey', alpha=0.1)\nbandpass_params = dict(lf=35, hf=1000, order=10, window='asymtukey', alpha=0.8)\nvisualize_sample('f572d13da0', idx=1, \n                 bandpass_params=bandpass_params,\n#                  notch_params=dict(f0=306, Q=120.0, window=None),\n                )","metadata":{"execution":{"iopub.status.busy":"2021-09-24T16:46:54.627445Z","iopub.execute_input":"2021-09-24T16:46:54.627667Z","iopub.status.idle":"2021-09-24T16:46:57.000458Z","shell.execute_reply.started":"2021-09-24T16:46:54.627642Z","shell.execute_reply":"2021-09-24T16:46:56.999600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 000a5b6e5c: strong positive","metadata":{}},{"cell_type":"code","source":"# bandpass_params = dict(lf=35, hf=500, order=10, window='tukey', alpha=0.25)\nbandpass_params = dict(lf=35, hf=1000, order=10, window='asymtukey', alpha=0.8)\nvisualize_sample('000a5b6e5c', idx=1, \n                 bandpass_params=bandpass_params,\n                 notch_params=dict(f0=306, Q=90.0, window=None),\n                )","metadata":{"execution":{"iopub.status.busy":"2021-09-24T16:54:40.719085Z","iopub.execute_input":"2021-09-24T16:54:40.719371Z","iopub.status.idle":"2021-09-24T16:54:43.042292Z","shell.execute_reply.started":"2021-09-24T16:54:40.719343Z","shell.execute_reply":"2021-09-24T16:54:43.041440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bandpass_params = dict(lf=35, hf=1000, order=10, window='tukey', alpha=0.15)\n# bandpass_params = dict(lf=35, hf=1000, order=10, window='asymtukey', alpha=0.8)\nvisualize_sample('000a5b6e5c', idx=1, \n                 bandpass_params=bandpass_params,\n                 notch_params=dict(f0=306, Q=90.0, window=None),\n                )","metadata":{"execution":{"iopub.status.busy":"2021-09-24T16:55:08.574746Z","iopub.execute_input":"2021-09-24T16:55:08.575022Z","iopub.status.idle":"2021-09-24T16:55:11.018394Z","shell.execute_reply.started":"2021-09-24T16:55:08.574993Z","shell.execute_reply":"2021-09-24T16:55:11.017528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}