{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"![](https://storage.googleapis.com/kaggle-competitions/kaggle/23249/logos/header.png)","metadata":{}},{"cell_type":"markdown","source":"<a id=\"top\"></a>\n\n<div class=\"list-group\" id=\"list-tab\" role=\"tablist\">\n<h3 class=\"list-group-item list-group-item-action active\" data-toggle=\"list\" style='color:white; background:#1777C4; border:0' role=\"tab\" aria-controls=\"home\"><center>Quick Navigation</center></h3>\n\n* [Overview](#1)\n* [Data Visualization](#2)\n* [Signal Transformations - Spectogram](#3)\n* [Signal Transformations - MFCC](#4)\n* [Signal Transformations - Q-transform](#5)\n    \n\n* [Competition Metric](#10)\n* [Sample Submission](#20)\n    \n\n* [Modeling](#100)","metadata":{}},{"cell_type":"markdown","source":"<a id=\"1\"></a>\n<h2 style='background:#1777C4; border:0; color:white'><center>Overview<center><h2>","metadata":{}},{"cell_type":"markdown","source":"In this competition you are provided with a training set of time series data containing simulated gravitational wave measurements from a network of 3 gravitational wave interferometers (LIGO Hanford, LIGO Livingston, and Virgo). Each time series contains either detector noise or detector noise plus a simulated gravitational wave signal. The task is to identify when a signal is present in the data (target=1).","metadata":{}},{"cell_type":"markdown","source":"### Files\n**train/** - the training set files, one npy file per observation; labels are provided in a files shown below   \n**test/** - the test set files; you must predict the probability that the observation contains a gravitational wave   \n**training_labels.csv** - target values of whether the associated signal contains a gravitational wave   \n**sample_submission.csv** - a sample submission file in the correct format","metadata":{}},{"cell_type":"code","source":"import os\nimport json\nimport random\nimport collections\n\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2021-09-28T21:54:34.614084Z","iopub.execute_input":"2021-09-28T21:54:34.614416Z","iopub.status.idle":"2021-09-28T21:54:35.428086Z","shell.execute_reply.started":"2021-09-28T21:54:34.614383Z","shell.execute_reply":"2021-09-28T21:54:35.427253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"2\"></a>\n<h2 style='background:#1777C4; border:0; color:white'><center>Data Visualization<center><h2>","metadata":{}},{"cell_type":"code","source":"def convert_image_id_2_path(image_id: str, is_train: bool = True) -> str:\n    folder = \"train\" if is_train else \"test\"\n    return \"../input/g2net-gravitational-wave-detection/{}/{}/{}/{}/{}.npy\".format(\n        folder, image_id[0], image_id[1], image_id[2], image_id \n    )","metadata":{"execution":{"iopub.status.busy":"2021-09-28T21:54:35.42952Z","iopub.execute_input":"2021-09-28T21:54:35.429865Z","iopub.status.idle":"2021-09-28T21:54:35.436353Z","shell.execute_reply.started":"2021-09-28T21:54:35.42983Z","shell.execute_reply":"2021-09-28T21:54:35.435384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(\"../input/g2net-gravitational-wave-detection/training_labels.csv\")\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2021-09-28T21:54:35.438946Z","iopub.execute_input":"2021-09-28T21:54:35.439297Z","iopub.status.idle":"2021-09-28T21:54:35.820139Z","shell.execute_reply.started":"2021-09-28T21:54:35.439261Z","shell.execute_reply":"2021-09-28T21:54:35.81932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(data=train_df, x=\"target\")","metadata":{"execution":{"iopub.status.busy":"2021-09-28T21:54:35.821731Z","iopub.execute_input":"2021-09-28T21:54:35.822083Z","iopub.status.idle":"2021-09-28T21:54:36.055534Z","shell.execute_reply.started":"2021-09-28T21:54:35.822038Z","shell.execute_reply":"2021-09-28T21:54:36.054758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Each data sample (npy file) contains 3 time series (1 for each detector) and each spans 2 sec and is sampled at 2,048 Hz.","metadata":{}},{"cell_type":"code","source":"def visualize_sample(\n    _id, \n    target, \n    colors=(\"black\", \"red\", \"green\"), \n    signal_names=(\"LIGO Hanford\", \"LIGO Livingston\", \"Virgo\")\n):\n    path = convert_image_id_2_path(_id)\n    t = np.arange(4096)/2048\n    x = np.load(path)\n    plt.figure(figsize=(16, 7))\n    for i in range(3):\n        plt.subplot(4, 1, i + 1)\n        plt.plot(t, x[i], color=colors[i])\n        plt.legend([signal_names[i]], fontsize=12, loc=\"lower right\")\n        \n        plt.subplot(4, 1, 4)\n        plt.plot(t, x[i], color=colors[i])\n    \n    plt.subplot(4, 1, 4)\n    plt.legend(signal_names, fontsize=12, loc=\"lower right\")\n\n    plt.suptitle(f\"id: {_id} target: {target}\", fontsize=16)\n    plt.show()\n","metadata":{"execution":{"iopub.status.busy":"2021-09-28T21:54:36.056964Z","iopub.execute_input":"2021-09-28T21:54:36.057504Z","iopub.status.idle":"2021-09-28T21:54:36.075829Z","shell.execute_reply.started":"2021-09-28T21:54:36.057465Z","shell.execute_reply":"2021-09-28T21:54:36.074874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in random.sample(train_df.index.tolist(), 5):\n    _id = train_df.iloc[i][\"id\"]\n    target = train_df.iloc[i][\"target\"]\n\n    visualize_sample(_id, target)","metadata":{"execution":{"iopub.status.busy":"2021-09-28T21:54:36.077325Z","iopub.execute_input":"2021-09-28T21:54:36.07855Z","iopub.status.idle":"2021-09-28T21:54:38.969673Z","shell.execute_reply.started":"2021-09-28T21:54:36.078509Z","shell.execute_reply":"2021-09-28T21:54:38.968848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"3\"></a>\n<h2 style='background:#1777C4; border:0; color:white'><center>Signal Transformations - Spectogram<center><h2>","metadata":{"execution":{"iopub.status.busy":"2021-07-01T07:40:45.589199Z","iopub.execute_input":"2021-07-01T07:40:45.589856Z","iopub.status.idle":"2021-07-01T07:40:45.601296Z","shell.execute_reply.started":"2021-07-01T07:40:45.589711Z","shell.execute_reply":"2021-07-01T07:40:45.599811Z"}}},{"cell_type":"code","source":"import librosa\nimport librosa.display","metadata":{"execution":{"iopub.status.busy":"2021-09-28T21:54:38.971198Z","iopub.execute_input":"2021-09-28T21:54:38.972194Z","iopub.status.idle":"2021-09-28T21:54:40.399531Z","shell.execute_reply.started":"2021-09-28T21:54:38.972143Z","shell.execute_reply":"2021-09-28T21:54:40.398491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize_sample_spectogram(\n    _id, \n    target,\n    signal_names=(\"LIGO Hanford\", \"LIGO Livingston\", \"Virgo\")\n):\n    x = np.load(convert_image_id_2_path(_id))\n    plt.figure(figsize=(16, 5))\n    for i in range(3):\n        X = librosa.stft(x[i] / x[i].max())\n        Xdb = librosa.amplitude_to_db(abs(X))\n        plt.subplot(1, 3, i + 1)\n        librosa.display.specshow(Xdb, sr=2048, x_axis=\"time\", y_axis=\"hz\", vmin=-30, vmax=50) \n        plt.colorbar()\n        plt.title(signal_names[i], fontsize=14)\n\n    plt.suptitle(f\"id: {_id} target: {target}\", fontsize=16)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-28T21:54:40.401235Z","iopub.execute_input":"2021-09-28T21:54:40.401602Z","iopub.status.idle":"2021-09-28T21:54:40.412761Z","shell.execute_reply.started":"2021-09-28T21:54:40.401561Z","shell.execute_reply":"2021-09-28T21:54:40.411975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in random.sample(train_df.index.tolist(), 5):\n    _id = train_df.iloc[i][\"id\"]\n    target = train_df.iloc[i][\"target\"]\n\n    visualize_sample_spectogram(_id, target)","metadata":{"execution":{"iopub.status.busy":"2021-09-28T21:54:40.416786Z","iopub.execute_input":"2021-09-28T21:54:40.417309Z","iopub.status.idle":"2021-09-28T21:54:43.237137Z","shell.execute_reply.started":"2021-09-28T21:54:40.417279Z","shell.execute_reply":"2021-09-28T21:54:43.236342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"4\"></a>\n<h2 style='background:#1777C4; border:0; color:white'><center>Signal Transformations - MFCC<center><h2>","metadata":{}},{"cell_type":"code","source":"def visualize_sample_mfcc(\n    _id, \n    target,\n    signal_names=(\"LIGO Hanford\", \"LIGO Livingston\", \"Virgo\"),\n    sr=2048,\n):\n    x = np.load(convert_image_id_2_path(_id))\n    plt.figure(figsize=(16, 5))\n    for i in range(3):\n        mfccs = librosa.feature.mfcc(x[i] / x[i].max(), sr=sr)\n        plt.subplot(1, 3, i + 1)\n        librosa.display.specshow(mfccs, sr=sr, x_axis=\"time\", vmin=-200, vmax=50, cmap=\"coolwarm\")\n        plt.title(signal_names[i], fontsize=14)\n        plt.colorbar()\n\n    plt.suptitle(f\"id: {_id} target: {target}\", fontsize=16)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-28T21:54:43.238725Z","iopub.execute_input":"2021-09-28T21:54:43.23907Z","iopub.status.idle":"2021-09-28T21:54:43.246568Z","shell.execute_reply.started":"2021-09-28T21:54:43.239034Z","shell.execute_reply":"2021-09-28T21:54:43.245356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in random.sample(train_df.index.tolist(), 5):\n    _id = train_df.iloc[i][\"id\"]\n    target = train_df.iloc[i][\"target\"]\n\n    visualize_sample_mfcc(_id, target)","metadata":{"execution":{"iopub.status.busy":"2021-09-28T21:54:43.247915Z","iopub.execute_input":"2021-09-28T21:54:43.248457Z","iopub.status.idle":"2021-09-28T21:54:46.117042Z","shell.execute_reply.started":"2021-09-28T21:54:43.24842Z","shell.execute_reply":"2021-09-28T21:54:46.116246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"5\"></a>\n<h2 style='background:#1777C4; border:0; color:white'><center>Signal Transformations - Q-transform<center><h2>","metadata":{"execution":{"iopub.status.busy":"2021-07-02T08:46:00.181912Z","iopub.execute_input":"2021-07-02T08:46:00.182312Z","iopub.status.idle":"2021-07-02T08:46:00.188979Z","shell.execute_reply.started":"2021-07-02T08:46:00.182274Z","shell.execute_reply":"2021-07-02T08:46:00.187378Z"}}},{"cell_type":"markdown","source":"Based on [PyCBC: Making Images](https://www.kaggle.com/alexnitz/pycbc-making-images)","metadata":{}},{"cell_type":"code","source":"!pip install pycbc -qq","metadata":{"execution":{"iopub.status.busy":"2021-09-28T21:54:46.118517Z","iopub.execute_input":"2021-09-28T21:54:46.118857Z","iopub.status.idle":"2021-09-28T21:55:09.434634Z","shell.execute_reply.started":"2021-09-28T21:54:46.11883Z","shell.execute_reply":"2021-09-28T21:55:09.433552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pycbc.types","metadata":{"execution":{"iopub.status.busy":"2021-09-28T21:55:09.436281Z","iopub.execute_input":"2021-09-28T21:55:09.436636Z","iopub.status.idle":"2021-09-28T21:55:10.090613Z","shell.execute_reply.started":"2021-09-28T21:55:09.436597Z","shell.execute_reply":"2021-09-28T21:55:10.089591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize_sample_qtransform(\n    _id, \n    target,\n    signal_names=(\"LIGO Hanford\", \"LIGO Livingston\", \"Virgo\"),\n    sr=2048\n):\n    x = np.load(convert_image_id_2_path(_id))\n    plt.figure(figsize=(16, 5))\n    for i in range(3):\n        \n        ts = pycbc.types.TimeSeries(x[i], epoch=0, delta_t=1 / sr) \n        ts = ts.whiten(0.125, 0.125)\n        t, freq, power = ts.qtransform(0.002, logfsteps=100, qrange=(10, 10), frange=(30, 512))\n        \n        plt.subplot(1, 3, i + 1)\n        plt.pcolormesh(t, freq, power, vmax=power.max(), vmin=0, shading='auto')\n        plt.yscale('log')\n        plt.title(signal_names[i], fontsize=14)\n        \n    plt.suptitle(f\"id: {_id} target: {target}\", fontsize=16)\n    plt.show()\n    return power\n\n# def visualize_sample_qtransform(\n#     _id, \n#     target,\n#     signal_names=(\"LIGO Hanford\", \"LIGO Livingston\", \"Virgo\"),\n#     sr=2048\n# ):\n#     x = np.load(convert_image_id_2_path(_id))\n#     plt.figure(figsize=(16, 5))\n#     for i in range(3):\n        \n#         ts = pycbc.types.TimeSeries(x[i], epoch=0, delta_t=1 / sr) \n#         ts = ts.whiten(0.125, 0.125)\n#         t, freq, power = ts.qtransform(\n#             0.002, logfsteps=100, qrange=(10, 10), frange=(20, 512))\n        \n#         plt.subplot(1, 3, i + 1)\n#         plt.pcolormesh(t, freq, power, vmax=15, vmin=0, shading='auto')\n#         plt.yscale('log')\n#         plt.title(signal_names[i], fontsize=14)\n        \n#     plt.suptitle(f\"id: {_id} target: {target}\", fontsize=16)\n#     plt.show()    ","metadata":{"execution":{"iopub.status.busy":"2021-09-28T21:55:10.092159Z","iopub.execute_input":"2021-09-28T21:55:10.092493Z","iopub.status.idle":"2021-09-28T21:55:10.105088Z","shell.execute_reply.started":"2021-09-28T21:55:10.092457Z","shell.execute_reply":"2021-09-28T21:55:10.102379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize_sample_power(\n    _id, \n    target,\n    signal_names=(\"LIGO Hanford\", \"LIGO Livingston\", \"Virgo\"),\n    sr=2048\n):\n    x = np.load(convert_image_id_2_path(_id))\n    plt.figure(figsize=(16, 5))\n    for i in range(3):\n        \n        ts = pycbc.types.TimeSeries(x[i], epoch=0, delta_t=1 / sr) \n        ts = ts.whiten(0.125, 0.125)\n        t, freq, power = ts.qtransform(0.002, logfsteps=100, qrange=(10, 10), frange=(30, 512))\n        \n        plt.subplot(1, 3, i + 1)    \n        plt.hist(power.flatten(), bins='auto')\n        plt.yscale('log')\n\n#     plt.suptitle(f\"id: {_id} target: {target}\", fontsize=16)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-28T21:55:10.107008Z","iopub.execute_input":"2021-09-28T21:55:10.107571Z","iopub.status.idle":"2021-09-28T21:55:10.11729Z","shell.execute_reply.started":"2021-09-28T21:55:10.107529Z","shell.execute_reply":"2021-09-28T21:55:10.116284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize_sample_power_3D(\n    _id, \n    target,\n    signal_names=(\"LIGO Hanford\", \"LIGO Livingston\", \"Virgo\"),\n    sr=2048\n):\n    x = np.load(convert_image_id_2_path(_id))\n#     plt.figure(figsize=(16, 5))\n    fig = plt.figure(figsize=(16,5))\n    ax1 = fig.add_subplot(131, projection='3d')\n    ax2 = fig.add_subplot(132, projection='3d')\n    ax3 = fig.add_subplot(133, projection='3d')\n    \n    for i in range(3):\n        \n        ts = pycbc.types.TimeSeries(x[i], epoch=0, delta_t=1 / sr) \n        ts = ts.whiten(0.125, 0.125)\n        t, freq, power = ts.qtransform(0.002, logfsteps=100, qrange=(10, 10), frange=(30, 512))\n        \n#         plt.subplot(1, 3, i + 1)\n        X, Y = np.meshgrid(t, freq)\n        Z = power\n        if i==0:\n            ax1 = plt.axes(projection='3d')\n            ax1.contour3D(X, Y, Z, 50, cmap='binary')\n        if i==1:\n            ax2 = plt.axes(projection='3d')\n            ax2.contour3D(X, Y, Z, 50, cmap='binary')\n        if i==2:\n            ax3 = plt.axes(projection='3d')\n            ax3.contour3D(X, Y, Z, 50, cmap='binary')\n    plt.show()\n\n","metadata":{"execution":{"iopub.status.busy":"2021-09-28T22:13:55.476155Z","iopub.execute_input":"2021-09-28T22:13:55.476508Z","iopub.status.idle":"2021-09-28T22:13:55.488127Z","shell.execute_reply.started":"2021-09-28T22:13:55.476478Z","shell.execute_reply":"2021-09-28T22:13:55.48532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize_all(\n    _id, \n    target,\n    signal_names=(\"LIGO Hanford\", \"LIGO Livingston\", \"Virgo\"),\n    sr=2048\n):\n    x = np.load(convert_image_id_2_path(_id))\n    plt.figure(figsize=(16, 16))\n    \n    for i in range(3):\n        \n        ts = pycbc.types.TimeSeries(x[i], epoch=0, delta_t=1 / sr) \n        ts = ts.whiten(0.125, 0.125)\n        t, freq, power = ts.qtransform(0.002, logfsteps=100, qrange=(10, 10), frange=(30, 512))\n        \n        plt.subplot(3, 3, i+1)\n        plt.pcolormesh(t, freq, power, vmax=power.max(), vmin=0, shading='auto')\n        plt.yscale('log')\n        plt.title(signal_names[i], fontsize=14)\n\n        plt.subplot(3, 3, i+4)        \n        X, Y = np.meshgrid(t, freq)\n        Z = power\n        ax = plt.axes(projection='3d')\n        ax.contour3D(X, Y, Z, 50, cmap='binary')        \n        \n        plt.subplot(3, 3, i+7)    \n        plt.hist(power.flatten(), bins='auto')\n        plt.yscale('log')\n        \n    plt.suptitle(f\"id: {_id} target: {target}\", fontsize=16)\n    plt.show()    ","metadata":{"execution":{"iopub.status.busy":"2021-09-28T21:55:10.130286Z","iopub.execute_input":"2021-09-28T21:55:10.130644Z","iopub.status.idle":"2021-09-28T21:55:10.142014Z","shell.execute_reply.started":"2021-09-28T21:55:10.130607Z","shell.execute_reply":"2021-09-28T21:55:10.140749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in random.sample(train_df.index.tolist(), 2):\n    _id = train_df.iloc[i][\"id\"]\n    target = train_df.iloc[i][\"target\"]\n\n#     visualize_sample(_id, target)\n    power = visualize_sample_qtransform(_id, target)\n    visualize_sample_power_3D(_id, target)    \n    visualize_sample_power(_id, target)\n#     visualize_all(_id, target)","metadata":{"execution":{"iopub.status.busy":"2021-09-28T22:14:13.505347Z","iopub.execute_input":"2021-09-28T22:14:13.505658Z","iopub.status.idle":"2021-09-28T22:14:31.303369Z","shell.execute_reply.started":"2021-09-28T22:14:13.50563Z","shell.execute_reply":"2021-09-28T22:14:31.302567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"power","metadata":{"execution":{"iopub.status.busy":"2021-09-27T22:50:59.191785Z","iopub.execute_input":"2021-09-27T22:50:59.192127Z","iopub.status.idle":"2021-09-27T22:50:59.199196Z","shell.execute_reply.started":"2021-09-27T22:50:59.19209Z","shell.execute_reply":"2021-09-27T22:50:59.197899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"power.max()","metadata":{"execution":{"iopub.status.busy":"2021-09-27T22:51:04.005229Z","iopub.execute_input":"2021-09-27T22:51:04.005579Z","iopub.status.idle":"2021-09-27T22:51:04.015158Z","shell.execute_reply.started":"2021-09-27T22:51:04.00555Z","shell.execute_reply":"2021-09-27T22:51:04.014237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.iloc[i][\"id\"]","metadata":{"execution":{"iopub.status.busy":"2021-08-08T22:59:11.108159Z","iopub.execute_input":"2021-08-08T22:59:11.108618Z","iopub.status.idle":"2021-08-08T22:59:11.115455Z","shell.execute_reply.started":"2021-08-08T22:59:11.108582Z","shell.execute_reply":"2021-08-08T22:59:11.114412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\nSubmissions are evaluated on [area under the ROC curve](https://en.wikipedia.org/wiki/Receiver_operating_characteristic) between the predicted probability and the observed target.\n","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score, roc_curve, auc","metadata":{"execution":{"iopub.status.busy":"2021-07-18T21:30:57.105122Z","iopub.execute_input":"2021-07-18T21:30:57.105591Z","iopub.status.idle":"2021-07-18T21:30:57.112308Z","shell.execute_reply.started":"2021-07-18T21:30:57.105549Z","shell.execute_reply":"2021-07-18T21:30:57.111118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_y_true = [\n    [1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0.],\n    [1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0.],\n    [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0.], #  IMBALANCE\n]\nlist_y_pred = [\n    [0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5],\n    [0.9, 0.9, 0.9, 0.9, 0.1, 0.9, 0.9, 0.1, 0.9, 0.1, 0.1, 0.5],\n    [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.], #  IMBALANCE\n]\n\nfor y_true, y_pred in zip(list_y_true, list_y_pred):\n    fpr, tpr, _ = roc_curve(y_true, y_pred)\n    roc_auc = auc(fpr, tpr)\n\n    plt.figure(figsize=(5, 5))\n    plt.plot(fpr, tpr, color='darkorange', lw=2, label='ROC curve (area = %0.2f)' % roc_auc)\n    plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')\n    plt.xlim([-0.01, 1.0])\n    plt.ylim([0.0, 1.05])\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.title('Receiver operating characteristic example')\n    plt.legend(loc=\"lower right\")\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-18T21:31:10.059624Z","iopub.execute_input":"2021-07-18T21:31:10.059945Z","iopub.status.idle":"2021-07-18T21:31:10.443871Z","shell.execute_reply.started":"2021-07-18T21:31:10.059915Z","shell.execute_reply":"2021-07-18T21:31:10.442838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"20\"></a>\n<h2 style='background:#1777C4; border:0; color:white'><center>Sample Submission<center><h2>","metadata":{}},{"cell_type":"code","source":"submission = pd.read_csv(\"../input/g2net-gravitational-wave-detection/sample_submission.csv\")\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-18T21:31:25.074947Z","iopub.execute_input":"2021-07-18T21:31:25.075281Z","iopub.status.idle":"2021-07-18T21:31:25.837391Z","shell.execute_reply.started":"2021-07-18T21:31:25.075253Z","shell.execute_reply":"2021-07-18T21:31:25.836471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"100\"></a>\n<h2 style='background:#1777C4; border:0; color:white'><center>Modeling<center><h2>","metadata":{}},{"cell_type":"markdown","source":"Some ideas from [G2Net melspectrogram starter code [LB: 0.830]](https://www.kaggle.com/c/g2net-gravitational-wave-detection/discussion/250278)","metadata":{}},{"cell_type":"code","source":"!pip install efficientnet_pytorch -qq","metadata":{"execution":{"iopub.status.busy":"2021-07-18T21:31:31.24685Z","iopub.execute_input":"2021-07-18T21:31:31.247224Z","iopub.status.idle":"2021-07-18T21:31:38.666032Z","shell.execute_reply.started":"2021-07-18T21:31:31.247192Z","shell.execute_reply":"2021-07-18T21:31:38.664832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\n\nimport torch\nfrom torch import nn\nfrom torch.utils import data as torch_data\nfrom sklearn import model_selection as sk_model_selection\nfrom torch.nn import functional as torch_functional\nfrom torch.autograd import Variable\nimport efficientnet_pytorch","metadata":{"execution":{"iopub.status.busy":"2021-07-18T21:31:38.669993Z","iopub.execute_input":"2021-07-18T21:31:38.670288Z","iopub.status.idle":"2021-07-18T21:31:39.809234Z","shell.execute_reply.started":"2021-07-18T21:31:38.670257Z","shell.execute_reply":"2021-07-18T21:31:39.8078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed_all(seed)\n        torch.backends.cudnn.deterministic = True\n\n\nset_seed(42)","metadata":{"execution":{"iopub.status.busy":"2021-07-02T12:31:34.694209Z","iopub.execute_input":"2021-07-02T12:31:34.694579Z","iopub.status.idle":"2021-07-02T12:31:34.743848Z","shell.execute_reply.started":"2021-07-02T12:31:34.694534Z","shell.execute_reply":"2021-07-02T12:31:34.743055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DataRetriever(torch_data.Dataset):\n    def __init__(self, paths, targets):\n        self.paths = paths\n        self.targets = targets\n          \n    def __len__(self):\n        return len(self.paths)\n    \n    def __getitem__(self, index):\n        file_path= f\"../input/g2net-n-mels-128-train-images/{self.paths[index]}.npy\"\n        image = np.load(file_path)\n        image = cv2.resize(image, (128 * 3, 128))\n        image = image.reshape(3, 128, 128)\n        image = torch.from_numpy(image).float()\n#         print(image.shape)\n#         path = convert_image_id_2_path(self.paths[index])\n#         x = np.load(path)\n        \n#         channels = []\n#         for i in range(3):\n#             ts = pycbc.types.TimeSeries(x[i], epoch=0, delta_t=1 / 2048) \n#             ts = ts.whiten(0.125, 0.125)\n#             _, _, power = ts.qtransform(\n#                 0.002, logfsteps=100, qrange=(10, 10), frange=(20, 512)\n#             )\n#             channels.append(cv2.resize(power / power.max(), (100, 100)))\n        \n#         image = np.array(channels)\n#         image = torch.from_numpy(image).float()\n# #         print(image.shape)\n        \n        y = torch.tensor(self.targets[index], dtype=torch.float)\n            \n        return {\"X\": image, \"y\": y}","metadata":{"execution":{"iopub.status.busy":"2021-07-18T21:31:47.145311Z","iopub.execute_input":"2021-07-18T21:31:47.145688Z","iopub.status.idle":"2021-07-18T21:31:47.153412Z","shell.execute_reply.started":"2021-07-18T21:31:47.145656Z","shell.execute_reply":"2021-07-18T21:31:47.152482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train, df_valid = sk_model_selection.train_test_split(\n    train_df, \n    test_size=0.2, \n    random_state=42, \n    stratify=train_df[\"target\"],\n)","metadata":{"execution":{"iopub.status.busy":"2021-07-18T21:31:49.606574Z","iopub.execute_input":"2021-07-18T21:31:49.606949Z","iopub.status.idle":"2021-07-18T21:31:50.119701Z","shell.execute_reply.started":"2021-07-18T21:31:49.606921Z","shell.execute_reply":"2021-07-18T21:31:50.118709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_retriever = DataRetriever(\n    df_train[\"id\"].values, \n    df_train[\"target\"].values, \n)\n\nvalid_data_retriever = DataRetriever(\n    df_valid[\"id\"].values, \n    df_valid[\"target\"].values,\n)","metadata":{"execution":{"iopub.status.busy":"2021-07-18T21:31:56.145226Z","iopub.execute_input":"2021-07-18T21:31:56.145571Z","iopub.status.idle":"2021-07-18T21:31:56.150621Z","shell.execute_reply.started":"2021-07-18T21:31:56.14554Z","shell.execute_reply":"2021-07-18T21:31:56.149794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.imshow(np.transpose(train_data_retriever[0][\"X\"], axes=(1, 2, 0)))","metadata":{"execution":{"iopub.status.busy":"2021-07-02T12:31:35.190661Z","iopub.execute_input":"2021-07-02T12:31:35.191126Z","iopub.status.idle":"2021-07-02T12:31:35.199902Z","shell.execute_reply.started":"2021-07-02T12:31:35.191088Z","shell.execute_reply":"2021-07-02T12:31:35.199072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader = torch_data.DataLoader(\n    train_data_retriever,\n    batch_size=32,\n    shuffle=True,\n    num_workers=8,\n)\n\nvalid_loader = torch_data.DataLoader(\n    valid_data_retriever, \n    batch_size=32,\n    shuffle=False,\n    num_workers=8,\n)","metadata":{"execution":{"iopub.status.busy":"2021-07-18T21:32:01.952811Z","iopub.execute_input":"2021-07-18T21:32:01.953252Z","iopub.status.idle":"2021-07-18T21:32:01.959586Z","shell.execute_reply.started":"2021-07-18T21:32:01.953207Z","shell.execute_reply":"2021-07-18T21:32:01.958425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class Model(nn.Module):\n#     def __init__(self, device):\n#         super().__init__()\n        \n#         self.device = device\n#         self.hidden_dim = 128\n#         self.layer_dim = 1\n        \n#         self.rnn = nn.RNN(3, self.hidden_dim, self.layer_dim, batch_first=True, nonlinearity='relu')\n        \n#         self.fc = nn.Linear(self.hidden_dim, 1)\n    \n#     def forward(self, x):\n        \n#         h0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim)).to(self.device)\n        \n#         out, hn = self.rnn(x, h0)\n#         out = self.fc(out[:, -1, :]) \n#         return out\n    \n\n# class Model(nn.Module):\n#     def __init__(self):\n#         super().__init__()\n        \n#         self.conv_1 = nn.Conv1d(4096, 1024, 1) \n#         self.conv_2 = nn.Conv1d(1024, 128, 1) \n#         self.conv_3 = nn.Conv1d(128, 32, 1) \n#         self.fc = nn.Linear(32 * 3, 1)\n    \n#     def forward(self, x):\n#         out = self.conv_1(x)\n#         out = self.conv_2(out)\n#         out = self.conv_3(out)\n#         out = torch.flatten(out, 1)\n#         out = self.fc(out)\n        \n#         return out\n    \n    \nclass Model(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.net = efficientnet_pytorch.EfficientNet.from_pretrained(\"efficientnet-b2\")\n        n_features = self.net._fc.in_features\n        print(n_features)\n        self.net._fc = nn.Linear(in_features=n_features, out_features=1, bias=True)\n    \n    def forward(self, x):\n        out = self.net(x)\n        return out","metadata":{"execution":{"iopub.status.busy":"2021-07-18T21:32:06.67057Z","iopub.execute_input":"2021-07-18T21:32:06.670888Z","iopub.status.idle":"2021-07-18T21:32:06.679347Z","shell.execute_reply.started":"2021-07-18T21:32:06.670858Z","shell.execute_reply":"2021-07-18T21:32:06.678533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class LossMeter:\n    def __init__(self):\n        self.avg = 0\n        self.n = 0\n\n    def update(self, val):\n        self.n += 1\n        # incremental update\n        self.avg = val / self.n + (self.n - 1) / self.n * self.avg\n\n        \nclass AccMeter:\n    def __init__(self):\n        self.avg = 0\n        self.n = 0\n        \n    def update(self, y_true, y_pred):\n        y_true = y_true.cpu().numpy().astype(int)\n        y_pred = y_pred.cpu().numpy() >= 0\n        last_n = self.n\n        self.n += len(y_true)\n        true_count = np.sum(y_true == y_pred)\n        # incremental update\n        self.avg = true_count / self.n + last_n / self.n * self.avg","metadata":{"execution":{"iopub.status.busy":"2021-07-18T21:32:10.125689Z","iopub.execute_input":"2021-07-18T21:32:10.126035Z","iopub.status.idle":"2021-07-18T21:32:10.132877Z","shell.execute_reply.started":"2021-07-18T21:32:10.126Z","shell.execute_reply":"2021-07-18T21:32:10.132014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Trainer:\n    def __init__(\n        self, \n        model, \n        device, \n        optimizer, \n        criterion, \n        loss_meter, \n        score_meter\n    ):\n        self.model = model\n        self.device = device\n        self.optimizer = optimizer\n        self.criterion = criterion\n        self.loss_meter = loss_meter\n        self.score_meter = score_meter\n        \n        self.best_valid_score = -np.inf\n        self.n_patience = 0\n        \n        self.messages = {\n            \"epoch\": \"[Epoch {}: {}] loss: {:.5f}, score: {:.5f}, time: {} s\",\n            \"checkpoint\": \"The score improved from {:.5f} to {:.5f}. Save model to '{}'\",\n            \"patience\": \"\\nValid score didn't improve last {} epochs.\"\n        }\n    \n    def fit(self, epochs, train_loader, valid_loader, save_path, patience):        \n        for n_epoch in range(1, epochs + 1):\n            self.info_message(\"EPOCH: {}\", n_epoch)\n            \n            train_loss, train_score, train_time = self.train_epoch(train_loader)\n            valid_loss, valid_score, valid_time = self.valid_epoch(valid_loader)\n            \n            self.info_message(\n                self.messages[\"epoch\"], \"Train\", n_epoch, train_loss, train_score, train_time\n            )\n            \n            self.info_message(\n                self.messages[\"epoch\"], \"Valid\", n_epoch, valid_loss, valid_score, valid_time\n            )\n\n            if True:\n#             if self.best_valid_score < valid_score:\n                self.info_message(\n                    self.messages[\"checkpoint\"], self.best_valid_score, valid_score, save_path\n                )\n                self.best_valid_score = valid_score\n                self.save_model(n_epoch, save_path)\n                self.n_patience = 0\n            else:\n                self.n_patience += 1\n            \n            if self.n_patience >= patience:\n                self.info_message(self.messages[\"patience\"], patience)\n                break\n            \n    def train_epoch(self, train_loader):\n        self.model.train()\n        t = time.time()\n        train_loss = self.loss_meter()\n        train_score = self.score_meter()\n        \n        for step, batch in enumerate(train_loader, 1):\n            X = batch[\"X\"].to(self.device)\n            targets = batch[\"y\"].to(self.device)\n            self.optimizer.zero_grad()\n            outputs = self.model(X).squeeze(1)\n            \n            loss = self.criterion(outputs, targets)\n            loss.backward()\n\n            train_loss.update(loss.detach().item())\n            train_score.update(targets, outputs.detach())\n\n            self.optimizer.step()\n            \n            _loss, _score = train_loss.avg, train_score.avg\n            message = 'Train Step {}/{}, train_loss: {:.5f}, train_score: {:.5f}'\n            self.info_message(message, step, len(train_loader), _loss, _score, end=\"\\r\")\n        \n        return train_loss.avg, train_score.avg, int(time.time() - t)\n    \n    def valid_epoch(self, valid_loader):\n        self.model.eval()\n        t = time.time()\n        valid_loss = self.loss_meter()\n        valid_score = self.score_meter()\n\n        for step, batch in enumerate(valid_loader, 1):\n            with torch.no_grad():\n                X = batch[\"X\"].to(self.device)\n                targets = batch[\"y\"].to(self.device)\n\n                outputs = self.model(X).squeeze(1)\n                loss = self.criterion(outputs, targets)\n\n                valid_loss.update(loss.detach().item())\n                valid_score.update(targets, outputs)\n                \n            _loss, _score = valid_loss.avg, valid_score.avg\n            message = 'Valid Step {}/{}, train_loss: {:.5f}, train_score: {:.5f}'\n            self.info_message(message, step, len(valid_loader), _loss, _score, end=\"\\r\")\n        \n        return valid_loss.avg, valid_score.avg, int(time.time() - t)\n    \n    def save_model(self, n_epoch, save_path):\n        torch.save(\n            {\n                \"model_state_dict\": self.model.state_dict(),\n                \"optimizer_state_dict\": self.optimizer.state_dict(),\n                \"best_valid_score\": self.best_valid_score,\n                \"n_epoch\": n_epoch,\n            },\n            save_path,\n        )\n    \n    @staticmethod\n    def info_message(message, *args, end=\"\\n\"):\n        print(message.format(*args), end=end)","metadata":{"execution":{"iopub.status.busy":"2021-07-18T21:32:10.746591Z","iopub.execute_input":"2021-07-18T21:32:10.746901Z","iopub.status.idle":"2021-07-18T21:32:10.766688Z","shell.execute_reply.started":"2021-07-18T21:32:10.746872Z","shell.execute_reply":"2021-07-18T21:32:10.765511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nmodel = Model()\nmodel.to(device)\n\noptimizer = torch.optim.Adam(model.parameters(), lr=0.001)\ncriterion = torch_functional.binary_cross_entropy_with_logits\n\ntrainer = Trainer(\n    model, \n    device, \n    optimizer, \n    criterion, \n    LossMeter, \n    AccMeter\n)\n\nhistory = trainer.fit(\n    2, \n    train_loader, \n    valid_loader, \n    \"best-model.pth\", \n    100,\n)","metadata":{"execution":{"iopub.status.busy":"2021-07-18T21:32:16.718314Z","iopub.execute_input":"2021-07-18T21:32:16.71863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"checkpoint = torch.load(\"best-model.pth\")\n\nmodel.load_state_dict(checkpoint[\"model_state_dict\"])\nmodel.eval();","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DataRetriever(torch_data.Dataset):\n    def __init__(self, paths):\n        self.paths = paths\n          \n    def __len__(self):\n        return len(self.paths)\n    \n    def __getitem__(self, index):\n        file_path= f\"../input/g2net-n-mels-128-test-images/{self.paths[index]}.npy\"\n        image = np.load(file_path)\n        image = cv2.resize(image, (128 * 3, 128))\n        image = image.reshape(3, 128, 128)\n        image = torch.from_numpy(image).float()\n            \n        return {\"X\": image, \"id\": self.paths[index]}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data_retriever = DataRetriever(\n    submission[\"id\"].values, \n)\n\ntest_loader = torch_data.DataLoader(\n    test_data_retriever,\n    batch_size=128,\n    shuffle=False,\n    num_workers=8,\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = []\nids = []\n\nfor e, batch in enumerate(test_loader):\n    print(f\"{e}/{len(test_loader)}\", end=\"\\r\")\n    with torch.no_grad():\n        y_pred.extend(torch.sigmoid(model(batch[\"X\"].to(device))).cpu().numpy().squeeze())\n        ids.extend(batch[\"id\"])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({\"id\": ids, \"target\": y_pred})\nsubmission.to_csv(\"model_submission.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### WORK IN PROGRESS ...","metadata":{}}]}