{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow.python as tfp\nfrom tqdm import tqdm\n\ntrain_df = pd.read_csv('../input/g2net-gravitational-wave-detection/training_labels.csv')\ntest_df = pd.read_csv('../input/g2net-gravitational-wave-detection/sample_submission.csv')\n\ndef get_train_file_path(image_id):\n    return \"../input/g2net-gravitational-wave-detection/train/{}/{}/{}/{}.npy\".format(\n        image_id[0], image_id[1], image_id[2], image_id)\n\ndef get_test_file_path(image_id):\n    return \"../input/g2net-gravitational-wave-detection/test/{}/{}/{}/{}.npy\".format(\n        image_id[0], image_id[1], image_id[2], image_id)\n\ntrain_df['image_path'] = train_df['id'].apply(get_train_file_path)\ntest_df['image_path'] = test_df['id'].apply(get_test_file_path)\n\ndef _bytes_feature(value):\n    if isinstance(value, tfp.framework.ops.EagerTensor):\n        value = value.numpy()\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\n\ndef _float_feature(value):\n    return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\n\ndef _int64_feature(value):\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\nfrom scipy.signal import butter, lfilter, lfilter_zi, filtfilt\ndef butter_bandpass(lowcut, highcut, fs, order=5):\n    nyq = 0.5 * fs\n    low = lowcut / nyq\n    high = highcut / nyq\n    b, a = butter(order, [low, high], btype='band')\n    return b, a\n\ndef butter_bandpass_filter(data, lowcut, highcut, fs, order=5):\n    b, a = butter_bandpass(lowcut, highcut, fs, order=order)\n    y = lfilter(b, a, data)\n    return y\n\nLOWCUT = 20\nHIGHCUT = 1000\n\ndef create_tf_example(wave_id: str, wave: bytes, target: int) -> tf.train.Example:\n    feature = {\n        \"wave_id\": _bytes_feature(wave_id),\n        \"wave\": _bytes_feature(wave),\n        \"target\": _float_feature(target)\n    }\n    return tf.train.Example(features=tf.train.Features(feature=feature))\n\n\ndef write_tfrecord(df: pd.DataFrame, filename: str):    \n    options = tf.io.TFRecordOptions(\"GZIP\")\n    with tf.io.TFRecordWriter(filename, options=options) as writer:\n        for i in tqdm(range(len(df))):\n            wave_id = str.encode(df.iloc[i][\"id\"])\n            wave_dir = df.iloc[i][\"image_path\"]\n            wave = butter_bandpass_filter(np.load(wave_dir), LOWCUT, HIGHCUT, 2048, order=7).tobytes()\n            #wave = np.load(wave_dir).tobytes()\n            target = df.iloc[i][\"target\"]\n            tf_example = create_tf_example(wave_id, wave, target)\n            writer.write(tf_example.SerializeToString())    ","metadata":{"execution":{"iopub.status.busy":"2023-01-07T04:07:32.267317Z","iopub.execute_input":"2023-01-07T04:07:32.267824Z","iopub.status.idle":"2023-01-07T04:07:40.169854Z","shell.execute_reply.started":"2023-01-07T04:07:32.267709Z","shell.execute_reply":"2023-01-07T04:07:40.168679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h3> One notebook has limited capacity for dataset </h3>\n<h3> So, we can seperate into 5 groups. [0,5], [5,10], [10,15], [15,20] </h3>","metadata":{}},{"cell_type":"code","source":"#train_samples_per_file = 28000\ntest_samples_per_file = 22600\ntest_number_of_files = len(test_df) // test_samples_per_file\n\nfor i in range(0,5): \n    start = i * test_samples_per_file\n    end = (i + 1) * test_samples_per_file\n    df = test_df.iloc[start:end].reset_index(drop=True)\n    filename = f\"bf_test{i}.tfrecords\"                                                                           \n    write_tfrecord(df, filename)","metadata":{"execution":{"iopub.status.busy":"2023-01-07T04:07:40.171148Z","iopub.execute_input":"2023-01-07T04:07:40.171451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h1> If it is helped please upvote for me :) </h1>\n<h1>    Thanks </h1>","metadata":{}}]}