{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Import\nimport tensorflow as tf\nimport numpy as np\nimport pandas as pd","metadata":{"execution":{"iopub.status.busy":"2021-07-10T08:38:03.608346Z","iopub.execute_input":"2021-07-10T08:38:03.608711Z","iopub.status.idle":"2021-07-10T08:38:03.613605Z","shell.execute_reply.started":"2021-07-10T08:38:03.60868Z","shell.execute_reply":"2021-07-10T08:38:03.612309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/g2net-gravitational-wave-detection/training_labels.csv')\ntest_df = pd.read_csv('../input/g2net-gravitational-wave-detection/sample_submission.csv')\n\ndef get_train_file_path(image_id):\n    return \"../input/g2net-gravitational-wave-detection/train/{}/{}/{}/{}.npy\".format(\n        image_id[0], image_id[1], image_id[2], image_id)\n\ndef get_test_file_path(image_id):\n    return \"../input/g2net-gravitational-wave-detection/test/{}/{}/{}/{}.npy\".format(\n        image_id[0], image_id[1], image_id[2], image_id)\n\ntrain_df['image_path'] = train_df['id'].apply(get_train_file_path)\ntest_df['image_path'] = test_df['id'].apply(get_test_file_path)\n\ndisplay(train_df.head())\ndisplay(test_df.head())","metadata":{"execution":{"iopub.status.busy":"2021-07-10T08:38:03.623736Z","iopub.execute_input":"2021-07-10T08:38:03.624196Z","iopub.status.idle":"2021-07-10T08:38:04.949767Z","shell.execute_reply.started":"2021-07-10T08:38:03.624156Z","shell.execute_reply":"2021-07-10T08:38:04.94886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len_trains = len(train_df)\nlen_tests = len(test_df)\nprint(len_trains)\nprint(len_tests)","metadata":{"execution":{"iopub.status.busy":"2021-07-10T09:21:57.117248Z","iopub.execute_input":"2021-07-10T09:21:57.117626Z","iopub.status.idle":"2021-07-10T09:21:57.123764Z","shell.execute_reply.started":"2021-07-10T09:21:57.117592Z","shell.execute_reply":"2021-07-10T09:21:57.122014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_samples_per_file = 28000\ntrain_number_of_files = len_trains // train_samples_per_file\ntest_samples_per_file = 11300\ntest_number_of_files = len_tests // test_samples_per_file\nprint(train_number_of_files)\nprint(test_number_of_files)","metadata":{"execution":{"iopub.status.busy":"2021-07-10T09:23:42.076303Z","iopub.execute_input":"2021-07-10T09:23:42.076732Z","iopub.status.idle":"2021-07-10T09:23:42.083021Z","shell.execute_reply.started":"2021-07-10T09:23:42.076696Z","shell.execute_reply":"2021-07-10T09:23:42.08199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create feature dict and tf.train.Example\n# All raw values should be converted to a type compatible with tf.Example. Use\n# the following functions to do these convertions.\ndef _bytes_feature(value):\n    \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\n\ndef _float_feature(value):\n    \"\"\"Returns a float_list from a float / double.\"\"\"\n    return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\n\ndef _int64_feature(value):\n    \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))","metadata":{"execution":{"iopub.status.busy":"2021-07-10T08:38:04.979856Z","iopub.execute_input":"2021-07-10T08:38:04.98055Z","iopub.status.idle":"2021-07-10T08:38:04.99233Z","shell.execute_reply.started":"2021-07-10T08:38:04.980482Z","shell.execute_reply":"2021-07-10T08:38:04.991002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create and write tfrecord file\ndef create_tf_example(image_id, image, target) -> tf.train.Example:\n    # Create sample feature dict\n    feature = {\n        'image_id': _bytes_feature(image_id),\n        'image': _bytes_feature(image),\n        \"target\": _int64_feature(target),\n    }\n    \n    # Create a `example` from the feature dict.\n    return tf.train.Example(features=tf.train.Features(feature=feature))\n    \ndef write_record(number_of_file, number_of_example, df, data_type = \"train\"):\n    # Write the serialized example to a record file.\n    for file_idx in range(number_of_file):\n        file_name = f'{data_type}{file_idx}-{number_of_example}.tfrecords'\n        options = tf.io.TFRecordOptions(\"GZIP\")\n        with tf.io.TFRecordWriter(file_name, options=options) as writer:\n            for example_idx in range(number_of_example):\n                df_idx = file_idx*number_of_example + example_idx\n                image_id = str.encode(df.iloc[df_idx][\"id\"])\n                image_dir = df.iloc[df_idx][\"image_path\"]\n                image = np.load(image_dir).tobytes()\n                target = df.iloc[df_idx][\"target\"]\n                tf_example = create_tf_example(image_id, image, target)\n                writer.write(tf_example.SerializeToString())\n    ","metadata":{"execution":{"iopub.status.busy":"2021-07-10T09:10:26.142598Z","iopub.execute_input":"2021-07-10T09:10:26.142964Z","iopub.status.idle":"2021-07-10T09:10:26.156046Z","shell.execute_reply.started":"2021-07-10T09:10:26.142934Z","shell.execute_reply":"2021-07-10T09:10:26.154679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Execute function\nwrite_record(train_number_of_files, train_samples_per_file, train_df, \"train\")\nwrite_record(test_number_of_files, test_samples_per_file, test_df, \"test\")","metadata":{"execution":{"iopub.status.busy":"2021-07-10T09:10:33.527641Z","iopub.execute_input":"2021-07-10T09:10:33.528051Z","iopub.status.idle":"2021-07-10T09:20:08.293279Z","shell.execute_reply.started":"2021-07-10T09:10:33.528018Z","shell.execute_reply":"2021-07-10T09:20:08.290934Z"},"trusted":true},"execution_count":null,"outputs":[]}]}