{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q pycbc\n\nimport os\nimport cv2\nimport numpy as np\nimport pandas as pd\nfrom tqdm.notebook import tqdm\nimport numpy, pylab, glob, os\nimport pycbc.types\nfrom scipy import signal\nfrom matplotlib import pyplot as plt","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":11.117111,"end_time":"2021-07-01T06:54:08.633012","exception":false,"start_time":"2021-07-01T06:53:57.515901","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FOLDS = 16\nSEED = 2809","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.019637,"end_time":"2021-07-01T06:54:08.665089","exception":false,"start_time":"2021-07-01T06:54:08.645452","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"0\"></a>\n## 0. EDA","metadata":{"papermill":{"duration":0.012891,"end_time":"2021-07-01T06:54:08.691533","exception":false,"start_time":"2021-07-01T06:54:08.678642","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train_df = pd.read_csv('../input/g2net-gravitational-wave-detection/training_labels.csv')\ntrain_df['path'] = train_df['id'].apply(lambda x: f'../input/g2net-gravitational-wave-detection/train/{x[0]}/{x[1]}/{x[2]}/{x}.npy')","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.867098,"end_time":"2021-07-01T06:54:09.570735","exception":false,"start_time":"2021-07-01T06:54:08.703637","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.037544,"end_time":"2021-07-01T06:54:09.620453","exception":false,"start_time":"2021-07-01T06:54:09.582909","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_data(path):\n    return np.load(path)\n\n#https://www.kaggle.com/alexnitz/pycbc-making-images\ndef get_qtransform(path):\n    q_vec = []\n    data = get_data(path)\n    for i in range(3):\n        vec = data[i]\n        ts = pycbc.types.TimeSeries(vec, epoch=0, delta_t=1.0/2048) \n        \n        # whiten the data (i.e. normalize the noise power at different frequencies)\n        ts = ts.whiten(0.125, 0.125)\n        \n        # calculate the qtransform\n        time, freq, power = ts.qtransform(15.0/2048, logfsteps=256, qrange=(10, 10), frange=(20, 512))\n        power -= power.min()\n        power /= power.max()\n        q_vec.append(power)\n    return np.dstack(q_vec)\n\ndef get_img_qtransform(path):\n    q_vec = get_qtransform(path)*255\n    return q_vec.astype(np.uint8)","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.026457,"end_time":"2021-07-01T06:54:09.701169","exception":false,"start_time":"2021-07-01T06:54:09.674712","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = get_data(train_df.path.values[0])\n\nfig, ax = plt.subplots(3, 1, figsize=(21, 21))\n\nfor i in range(3):\n    ax[i].plot(data[i])\nplt.show();","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.617265,"end_time":"2021-07-01T07:01:52.913012","exception":false,"start_time":"2021-07-01T07:01:52.295747","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"power = get_img_qtransform(train_df.path.values[1])\n\nfig, ax = plt.subplots(1, 3, figsize=(24, 8))\n\n\nfor i in range(3):\n    ax[i].imshow(power[..., i].astype(np.float32).T)","metadata":{"_kg_hide-input":true,"papermill":{"duration":1.02299,"end_time":"2021-07-01T07:01:53.957453","exception":false,"start_time":"2021-07-01T07:01:52.934463","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"1\"></a>\n## 1. Grouped by Target","metadata":{"papermill":{"duration":0.039242,"end_time":"2021-07-01T07:01:54.035979","exception":false,"start_time":"2021-07-01T07:01:53.996737","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\nskf = StratifiedKFold(n_splits=FOLDS, shuffle=True, random_state=SEED)\ntrain_df['fold'] = -1\nfor fold, (train_idx, val_idx) in enumerate(skf.split(train_df, train_df['target'])):\n    train_df.loc[val_idx,'fold'] = fold\ntrain_df.groupby(['fold', 'target'])['id'].count()","metadata":{"_kg_hide-input":true,"papermill":{"duration":1.456508,"end_time":"2021-07-01T07:01:55.532644","exception":false,"start_time":"2021-07-01T07:01:54.076136","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"1\"></a>\n## 2. Save in TFRecords","metadata":{"papermill":{"duration":0.038305,"end_time":"2021-07-01T07:01:55.609932","exception":false,"start_time":"2021-07-01T07:01:55.571627","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import tensorflow as tf\n\ndef _bytes_feature(value):\n    \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n    if isinstance(value, type(tf.constant(0))):\n        value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n    \"\"\"Returns a float_list from a float / double.\"\"\"\n    return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _int64_feature(value):\n    \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))","metadata":{"_kg_hide-input":true,"papermill":{"duration":6.109882,"end_time":"2021-07-01T07:02:01.759324","exception":false,"start_time":"2021-07-01T07:01:55.649442","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_serialize_example(feature0, feature1, feature2):\n    feature = {\n      'image'         : _bytes_feature(feature0),\n      'image_id'      : _bytes_feature(feature1),   \n      'target'        : _int64_feature(feature2),\n  }\n    example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n    return example_proto.SerializeToString()","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.049526,"end_time":"2021-07-01T07:02:01.848214","exception":false,"start_time":"2021-07-01T07:02:01.798688","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show=True\nfolds = sorted(train_df.fold.unique().tolist())\nfor fold in tqdm(folds):\n    if fold not in list(range(0, 2)):\n        continue\n    fold_df = train_df[train_df.fold==fold]\n    if show:\n        print(); print('Writing TFRecord of fold %i :'%(fold))  \n    with tf.io.TFRecordWriter('train%.2i-%i.tfrec'%(fold,fold_df.shape[0])) as writer:\n        samples = fold_df.shape[0]\n        it = tqdm(range(samples)) if show else range(samples)\n        for k in it:\n            row = fold_df.iloc[k,:]\n            image      = get_img_qtransform(row['path'])[...,::-1]\n            image_id   = row['id']\n            target     = np.array(row['target'], dtype=np.uint8)\n            example  = train_serialize_example(\n                cv2.imencode('.png', image)[1].tobytes(),\n                str.encode(image_id),\n                target,\n                )\n            writer.write(example)\n        if show:\n            filepath = 'train%.2i-%i.tfrec'%(fold,fold_df.shape[0])\n            filename = filepath.split('/')[-1]\n            filesize = os.path.getsize(filepath)/10**6\n            print(filename,':',np.around(filesize, 2),'MB')","metadata":{"papermill":{"duration":8747.700983,"end_time":"2021-07-01T09:27:49.589713","exception":false,"start_time":"2021-07-01T07:02:01.88873","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}