{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow.python as tfp\nfrom tqdm import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-20T09:53:50.060105Z","iopub.execute_input":"2021-08-20T09:53:50.060801Z","iopub.status.idle":"2021-08-20T09:53:56.601317Z","shell.execute_reply.started":"2021-08-20T09:53:50.0607Z","shell.execute_reply":"2021-08-20T09:53:56.60024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/g2net-gravitational-wave-detection/training_labels.csv')\ntest_df = pd.read_csv('../input/g2net-gravitational-wave-detection/sample_submission.csv')\n","metadata":{"execution":{"iopub.status.busy":"2021-08-20T09:54:20.349704Z","iopub.execute_input":"2021-08-20T09:54:20.35029Z","iopub.status.idle":"2021-08-20T09:54:21.078967Z","shell.execute_reply.started":"2021-08-20T09:54:20.350223Z","shell.execute_reply":"2021-08-20T09:54:21.078182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_file_path(image_id):\n    return \"../input/g2net-gravitational-wave-detection/train/{}/{}/{}/{}.npy\".format(\n        image_id[0], image_id[1], image_id[2], image_id)\n\ndef get_test_file_path(image_id):\n    return \"../input/g2net-gravitational-wave-detection/test/{}/{}/{}/{}.npy\".format(\n        image_id[0], image_id[1], image_id[2], image_id)\n","metadata":{"execution":{"iopub.status.busy":"2021-08-20T09:54:41.979384Z","iopub.execute_input":"2021-08-20T09:54:41.979786Z","iopub.status.idle":"2021-08-20T09:54:41.985389Z","shell.execute_reply.started":"2021-08-20T09:54:41.979755Z","shell.execute_reply":"2021-08-20T09:54:41.984429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train_df.head())","metadata":{"execution":{"iopub.status.busy":"2021-08-20T09:54:54.912157Z","iopub.execute_input":"2021-08-20T09:54:54.912552Z","iopub.status.idle":"2021-08-20T09:54:54.93944Z","shell.execute_reply.started":"2021-08-20T09:54:54.912519Z","shell.execute_reply":"2021-08-20T09:54:54.938676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['image_path'] = train_df['id'].apply(get_train_file_path)\ndisplay(train_df.head())\ndisplay(train_df.loc[0].image_path)\ntest_df['image_path'] = test_df['id'].apply(get_test_file_path)","metadata":{"execution":{"iopub.status.busy":"2021-08-20T09:55:06.347447Z","iopub.execute_input":"2021-08-20T09:55:06.347972Z","iopub.status.idle":"2021-08-20T09:55:07.157654Z","shell.execute_reply.started":"2021-08-20T09:55:06.347939Z","shell.execute_reply":"2021-08-20T09:55:07.156648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def bytes_feature(value):\n    \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n    if isinstance(value, tfp.framework.ops.EagerTensor):\n        value = value.numpy()\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\n\ndef float_feature(value):\n    return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\n\ndef int64_feature(value):\n    \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n","metadata":{"execution":{"iopub.status.busy":"2021-08-20T09:55:18.128856Z","iopub.execute_input":"2021-08-20T09:55:18.129552Z","iopub.status.idle":"2021-08-20T09:55:18.138066Z","shell.execute_reply.started":"2021-08-20T09:55:18.129498Z","shell.execute_reply":"2021-08-20T09:55:18.136918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_tf_example(wave_id: str, wave: bytes, target: int) -> tf.train.Example:\n     # Create a dictionary mapping the feature name to the tf.train.Example-compatible data type.\n    feature = {\n        \"wave_id\": bytes_feature(wave_id),\n        \"wave\": bytes_feature(wave),\n        \"target\": int64_feature(target)\n    }\n    return tf.train.Example(features=tf.train.Features(feature=feature))\n","metadata":{"execution":{"iopub.status.busy":"2021-08-20T09:55:30.527922Z","iopub.execute_input":"2021-08-20T09:55:30.5283Z","iopub.status.idle":"2021-08-20T09:55:30.534581Z","shell.execute_reply.started":"2021-08-20T09:55:30.528253Z","shell.execute_reply":"2021-08-20T09:55:30.53314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def write_tfrecord(df: pd.DataFrame, filename: str):\n    options = tf.io.TFRecordOptions(\"GZIP\")\n    with tf.io.TFRecordWriter(filename, options=options) as writer:\n        for i in tqdm(range(len(df))):\n            wave_id = str.encode(df.iloc[i][\"id\"])\n            wave_dir = df.iloc[i][\"image_path\"]\n            wave = np.load(wave_dir).tobytes()\n            target = df.iloc[i][\"target\"]\n            tf_example = create_tf_example(wave_id, wave, target)\n            writer.write(tf_example.SerializeToString())","metadata":{"execution":{"iopub.status.busy":"2021-08-20T09:55:41.977053Z","iopub.execute_input":"2021-08-20T09:55:41.97746Z","iopub.status.idle":"2021-08-20T09:55:41.985071Z","shell.execute_reply.started":"2021-08-20T09:55:41.977426Z","shell.execute_reply":"2021-08-20T09:55:41.983792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#00000e74ad\ndisplay(train_df[train_df.id=='00000e74ad'].image_path[0])\ndisplay(np.load(train_df[train_df.id=='00000e74ad'].image_path[0]))","metadata":{"execution":{"iopub.status.busy":"2021-08-20T09:55:53.213337Z","iopub.execute_input":"2021-08-20T09:55:53.213728Z","iopub.status.idle":"2021-08-20T09:55:53.419524Z","shell.execute_reply.started":"2021-08-20T09:55:53.213696Z","shell.execute_reply":"2021-08-20T09:55:53.418403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display('len of input df {}'.format(len(train_df)))\ntrain_samples_per_file = 28000\ntrain_number_of_files = len(train_df) // train_samples_per_file\ndisplay(train_number_of_files)","metadata":{"execution":{"iopub.status.busy":"2021-08-20T09:56:06.483448Z","iopub.execute_input":"2021-08-20T09:56:06.483837Z","iopub.status.idle":"2021-08-20T09:56:06.497574Z","shell.execute_reply.started":"2021-08-20T09:56:06.483803Z","shell.execute_reply":"2021-08-20T09:56:06.496454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#for i in range(train_number_of_files):\nfor i in range(1):\n    start = i * train_samples_per_file\n    end = (i + 1) * train_samples_per_file\n    #print(\"itr {} start {} end {} \".format(i,start,end))\n    df = train_df.iloc[start:end].reset_index(drop=True)\n    #display(df)\n    filename = f\"train{i}.tfrecords\"\n    write_tfrecord(df, filename)","metadata":{},"execution_count":null,"outputs":[]}]}