{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow.python as tfp\nfrom tqdm import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-05T23:34:21.136239Z","iopub.execute_input":"2021-08-05T23:34:21.136695Z","iopub.status.idle":"2021-08-05T23:34:27.433917Z","shell.execute_reply.started":"2021-08-05T23:34:21.1366Z","shell.execute_reply":"2021-08-05T23:34:27.433104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/g2net-gravitational-wave-detection/training_labels.csv')\ntest_df = pd.read_csv('../input/g2net-gravitational-wave-detection/sample_submission.csv')\n\ndef get_train_file_path(image_id):\n    return \"../input/g2net-gravitational-wave-detection/train/{}/{}/{}/{}.npy\".format(\n        image_id[0], image_id[1], image_id[2], image_id)\n\ndef get_test_file_path(image_id):\n    return \"../input/g2net-gravitational-wave-detection/test/{}/{}/{}/{}.npy\".format(\n        image_id[0], image_id[1], image_id[2], image_id)\n\ntrain_df['image_path'] = train_df['id'].apply(get_train_file_path)\ntest_df['image_path'] = test_df['id'].apply(get_test_file_path)","metadata":{"execution":{"iopub.status.busy":"2021-08-05T23:34:27.435452Z","iopub.execute_input":"2021-08-05T23:34:27.43601Z","iopub.status.idle":"2021-08-05T23:34:28.835238Z","shell.execute_reply.started":"2021-08-05T23:34:27.435964Z","shell.execute_reply":"2021-08-05T23:34:28.834339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _bytes_feature(value):\n    if isinstance(value, tfp.framework.ops.EagerTensor):\n        value = value.numpy()\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\n\ndef _float_feature(value):\n    return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\n\ndef _int64_feature(value):\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))","metadata":{"execution":{"iopub.status.busy":"2021-08-05T23:34:28.837031Z","iopub.execute_input":"2021-08-05T23:34:28.837693Z","iopub.status.idle":"2021-08-05T23:34:28.846119Z","shell.execute_reply.started":"2021-08-05T23:34:28.837643Z","shell.execute_reply":"2021-08-05T23:34:28.845148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_tf_example(wave_id: str, wave: bytes, target: int) -> tf.train.Example:\n    feature = {\n        \"wave_id\": _bytes_feature(wave_id),\n        \"wave\": _bytes_feature(wave),\n        \"target\": _int64_feature(target)\n    }\n    return tf.train.Example(features=tf.train.Features(feature=feature))\n\n\ndef write_tfrecord(df: pd.DataFrame, filename: str):\n    options = tf.io.TFRecordOptions(\"GZIP\")\n    with tf.io.TFRecordWriter(filename, options=options) as writer:\n        for i in tqdm(range(len(df))):\n            wave_id = str.encode(df.iloc[i][\"id\"])\n            wave_dir = df.iloc[i][\"image_path\"]\n            wave = np.load(wave_dir).tobytes()\n            target = df.iloc[i][\"target\"]\n            tf_example = create_tf_example(wave_id, wave, target)\n            writer.write(tf_example.SerializeToString())","metadata":{"execution":{"iopub.status.busy":"2021-08-05T23:34:28.847756Z","iopub.execute_input":"2021-08-05T23:34:28.848229Z","iopub.status.idle":"2021-08-05T23:34:28.859725Z","shell.execute_reply.started":"2021-08-05T23:34:28.848185Z","shell.execute_reply":"2021-08-05T23:34:28.858903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_samples_per_file = 28000\ntrain_number_of_files = len(train_df) // train_samples_per_file\n\nfor i in range(10, 15):\n    start = i * train_samples_per_file\n    end = (i + 1) * train_samples_per_file\n    df = train_df.iloc[start:end].reset_index(drop=True)\n    filename = f\"train{i}.tfrecords\"\n    write_tfrecord(df, filename)","metadata":{"execution":{"iopub.status.busy":"2021-08-05T23:34:49.29533Z","iopub.execute_input":"2021-08-05T23:34:49.295863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}