{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q pycbc\n\nimport os\nimport cv2\nimport numpy as np\nimport pandas as pd\nfrom tqdm.notebook import tqdm\nimport numpy, pylab, glob, os\nimport pycbc.types\nfrom scipy import signal\nfrom matplotlib import pyplot as plt\n","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":10.692249,"end_time":"2021-07-02T06:42:04.007136","exception":false,"start_time":"2021-07-02T06:41:53.314887","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-03T17:45:11.747285Z","iopub.execute_input":"2021-07-03T17:45:11.747803Z","iopub.status.idle":"2021-07-03T17:45:34.120525Z","shell.execute_reply.started":"2021-07-03T17:45:11.747717Z","shell.execute_reply":"2021-07-03T17:45:34.119707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FOLDS = 8\nSEED = 2809","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.021,"end_time":"2021-07-02T06:42:04.042016","exception":false,"start_time":"2021-07-02T06:42:04.021016","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-03T17:45:34.122623Z","iopub.execute_input":"2021-07-03T17:45:34.122995Z","iopub.status.idle":"2021-07-03T17:45:34.127549Z","shell.execute_reply.started":"2021-07-03T17:45:34.122966Z","shell.execute_reply":"2021-07-03T17:45:34.126631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"0\"></a>\n## 0. EDA","metadata":{"papermill":{"duration":0.01315,"end_time":"2021-07-02T06:42:04.068882","exception":false,"start_time":"2021-07-02T06:42:04.055732","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train_df = pd.read_csv('../input/g2net-gravitational-wave-detection/training_labels.csv')\ntrain_df['path'] = train_df['id'].apply(lambda x: f'../input/g2net-gravitational-wave-detection/train/{x[0]}/{x[1]}/{x[2]}/{x}.npy')","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.889964,"end_time":"2021-07-02T06:42:04.972281","exception":false,"start_time":"2021-07-02T06:42:04.082317","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-03T17:45:34.129223Z","iopub.execute_input":"2021-07-03T17:45:34.129839Z","iopub.status.idle":"2021-07-03T17:45:34.881525Z","shell.execute_reply.started":"2021-07-03T17:45:34.129797Z","shell.execute_reply":"2021-07-03T17:45:34.880815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = pd.read_csv('../input/g2net-gravitational-wave-detection/sample_submission.csv')\nsub_df['path'] = sub_df['id'].apply(lambda x: f'../input/g2net-gravitational-wave-detection/test/{x[0]}/{x[1]}/{x[2]}/{x}.npy')","metadata":{"papermill":{"duration":0.408774,"end_time":"2021-07-02T06:42:05.395482","exception":false,"start_time":"2021-07-02T06:42:04.986708","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-03T17:45:34.882920Z","iopub.execute_input":"2021-07-03T17:45:34.883336Z","iopub.status.idle":"2021-07-03T17:45:35.221309Z","shell.execute_reply.started":"2021-07-03T17:45:34.883295Z","shell.execute_reply":"2021-07-03T17:45:35.220260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.041101,"end_time":"2021-07-02T06:42:05.452872","exception":false,"start_time":"2021-07-02T06:42:05.411771","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-03T17:45:35.224221Z","iopub.execute_input":"2021-07-03T17:45:35.224651Z","iopub.status.idle":"2021-07-03T17:45:35.243660Z","shell.execute_reply.started":"2021-07-03T17:45:35.224607Z","shell.execute_reply":"2021-07-03T17:45:35.242812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_data(path):\n    return np.load(path)\n\n#https://www.kaggle.com/alexnitz/pycbc-making-images\ndef get_qtransform(path):\n    q_vec = []\n    data = get_data(path)\n    for i in range(3):\n        vec = data[i]\n        ts = pycbc.types.TimeSeries(vec, epoch=0, delta_t=1.0/2048) \n        \n        # whiten the data (i.e. normalize the noise power at different frequencies)\n        ts = ts.whiten(0.125, 0.125)\n        \n        # calculate the qtransform\n        time, freq, power = ts.qtransform(15.0/2048, logfsteps=256, qrange=(10, 10), frange=(20, 512))\n        power -= power.min()\n        power /= power.max()\n        q_vec.append(power)\n    return np.dstack(q_vec)\n\ndef get_img_qtransform(path):\n    q_vec = get_qtransform(path)*255\n    return q_vec.astype(np.uint8)","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.025363,"end_time":"2021-07-02T06:42:05.534347","exception":false,"start_time":"2021-07-02T06:42:05.508984","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-03T17:45:35.245880Z","iopub.execute_input":"2021-07-03T17:45:35.246274Z","iopub.status.idle":"2021-07-03T17:45:35.254024Z","shell.execute_reply.started":"2021-07-03T17:45:35.246234Z","shell.execute_reply":"2021-07-03T17:45:35.252903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = get_data(train_df.path.values[0])\n\nfig, ax = plt.subplots(3, 1, figsize=(21, 21))\n\nfor i in range(3):\n    ax[i].plot(data[i])\nplt.show();","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.550825,"end_time":"2021-07-02T06:50:47.762177","exception":false,"start_time":"2021-07-02T06:50:47.211352","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-03T17:45:35.256005Z","iopub.execute_input":"2021-07-03T17:45:35.256554Z","iopub.status.idle":"2021-07-03T17:45:35.731092Z","shell.execute_reply.started":"2021-07-03T17:45:35.256513Z","shell.execute_reply":"2021-07-03T17:45:35.730028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"power = get_img_qtransform(train_df.path.values[1])\n\nfig, ax = plt.subplots(1, 3, figsize=(24, 8))\n\n\nfor i in range(3):\n    ax[i].imshow(power[..., i].T)","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.982528,"end_time":"2021-07-02T06:50:48.766985","exception":false,"start_time":"2021-07-02T06:50:47.784457","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-03T17:45:35.732328Z","iopub.execute_input":"2021-07-03T17:45:35.732612Z","iopub.status.idle":"2021-07-03T17:45:38.395912Z","shell.execute_reply.started":"2021-07-03T17:45:35.732564Z","shell.execute_reply":"2021-07-03T17:45:38.395055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"1\"></a>\n## 1. Grouped by Target","metadata":{"papermill":{"duration":0.042845,"end_time":"2021-07-02T06:50:48.853275","exception":false,"start_time":"2021-07-02T06:50:48.810430","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from sklearn.model_selection import KFold\nskf = KFold(n_splits=FOLDS, shuffle=False)\nsub_df['fold'] = -1\nfor fold, (train_idx, val_idx) in enumerate(skf.split(sub_df)):\n    sub_df.loc[val_idx,'fold'] = fold","metadata":{"_kg_hide-input":true,"papermill":{"duration":1.15099,"end_time":"2021-07-02T06:50:50.046816","exception":false,"start_time":"2021-07-02T06:50:48.895826","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-03T17:45:38.397396Z","iopub.execute_input":"2021-07-03T17:45:38.397676Z","iopub.status.idle":"2021-07-03T17:45:38.603027Z","shell.execute_reply.started":"2021-07-03T17:45:38.397649Z","shell.execute_reply":"2021-07-03T17:45:38.602220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"1\"></a>\n## 2. Save in TFRecords","metadata":{"papermill":{"duration":0.043017,"end_time":"2021-07-02T06:50:50.133395","exception":false,"start_time":"2021-07-02T06:50:50.090378","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import tensorflow as tf\n\ndef _bytes_feature(value):\n    \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n    if isinstance(value, type(tf.constant(0))):\n        value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n    \"\"\"Returns a float_list from a float / double.\"\"\"\n    return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _int64_feature(value):\n    \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))","metadata":{"_kg_hide-input":true,"papermill":{"duration":5.808418,"end_time":"2021-07-02T06:50:55.984641","exception":false,"start_time":"2021-07-02T06:50:50.176223","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-03T17:45:38.606463Z","iopub.execute_input":"2021-07-03T17:45:38.606742Z","iopub.status.idle":"2021-07-03T17:45:43.225180Z","shell.execute_reply.started":"2021-07-03T17:45:38.606714Z","shell.execute_reply":"2021-07-03T17:45:43.224188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_serialize_example(feature0, feature1):\n    feature = {\n      'image'         : _bytes_feature(feature0),\n      'image_id'      : _bytes_feature(feature1),\n  }\n    example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n    return example_proto.SerializeToString()","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.050456,"end_time":"2021-07-02T06:50:56.077627","exception":false,"start_time":"2021-07-02T06:50:56.027171","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-03T17:45:43.229156Z","iopub.execute_input":"2021-07-03T17:45:43.229498Z","iopub.status.idle":"2021-07-03T17:45:43.234440Z","shell.execute_reply.started":"2021-07-03T17:45:43.229466Z","shell.execute_reply":"2021-07-03T17:45:43.233699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show=True\nfolds = sorted(sub_df.fold.unique().tolist())\nfor fold in tqdm(folds):\n    if fold not in range(0, 2):\n        continue\n    fold_df = sub_df[sub_df.fold==fold]\n    if show:\n        print(); print('Writing TFRecord of fold %i :'%(fold))  \n    with tf.io.TFRecordWriter('test%.2i-%i.tfrec'%(fold,fold_df.shape[0])) as writer:\n        samples = fold_df.shape[0]\n        it = tqdm(range(samples)) if show else range(samples)\n        for k in it:\n            row = fold_df.iloc[k,:]\n            image      = get_img_qtransform(row['path'])[...,::-1]\n            image_id   = row['id']\n            example  = train_serialize_example(\n                cv2.imencode('.png', image)[1].tobytes(),\n                str.encode(image_id)\n                )\n            writer.write(example)\n        if show:\n            filepath = 'test%.2i-%i.tfrec'%(fold,fold_df.shape[0])\n            filename = filepath.split('/')[-1]\n            filesize = os.path.getsize(filepath)/10**6\n            print(filename,':',np.around(filesize, 2),'MB')","metadata":{"papermill":{"duration":6969.393445,"end_time":"2021-07-02T08:47:05.514254","exception":false,"start_time":"2021-07-02T06:50:56.120809","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-03T17:45:43.235564Z","iopub.execute_input":"2021-07-03T17:45:43.235974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.046905,"end_time":"2021-07-02T08:47:05.641860","exception":false,"start_time":"2021-07-02T08:47:05.594955","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}