{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Load tfrecord\n\nThis small script helps to load the data from tfrecord train/test and to transform the audio wav from DCM format into numpy array."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport numpy as np\n#tf.random.set_seed(42)\n\nimport os\nimport tensorflow.keras.backend as K\nAUTO     = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_labeled_tfrecord(example, return_image_name=False):\n    tfrec_format = {\n        'recording_id'                        : tf.io.FixedLenFeature([], tf.string),\n        'label_info'                   : tf.io.FixedLenFeature([], tf.string),\n        'audio_wav'                       : tf.io.FixedLenFeature([], tf.string)\n    }           \n    example = tf.io.parse_single_example(example, tfrec_format)\n    \n    example[\"audio_wav\"], example[\"sample_rate\"] = tf.audio.decode_wav(example[\"audio_wav\"])\n\n    return example\n\n\ndef read_unlabeled_tfrecord(example):\n    tfrec_format = {\n        'recording_id'                        : tf.io.FixedLenFeature([], tf.string),\n        'audio_wav'                       : tf.io.FixedLenFeature([], tf.string)\n    }           \n    example = tf.io.parse_single_example(example, tfrec_format)\n    \n    example[\"audio_wav\"], example[\"sample_rate\"] = tf.audio.decode_wav(example[\"audio_wav\"])\n\n    return example","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"files_train = [\"../input/rfcx-species-audio-detection/tfrecords/train/\" + x for x in os.listdir(\"../input/rfcx-species-audio-detection/tfrecords/train\")]\nfiles_test = [\"../input/rfcx-species-audio-detection/tfrecords/test/\" + x for x in os.listdir(\"../input/rfcx-species-audio-detection/tfrecords/test\")]\nprint(\"number of training tfrecord : \", len(files_train))\nprint(\"number of training tfrecord : \", len(files_test))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"samples_rates = set()\naudio_lengths = set()\nfor path_tfrecord in files_train:\n    raw_dataset = tf.data.TFRecordDataset(path_tfrecord).cache()\n    raw_dataset = raw_dataset.map(lambda example: read_labeled_tfrecord(example), num_parallel_calls=AUTO)\n\n\n\n    for raw_record in raw_dataset:\n        samples_rates.add(raw_record[\"sample_rate\"].numpy())\n        audio_lengths.add(raw_record[\"audio_wav\"].shape[0])\n        \nfor path_tfrecord in files_test:\n    raw_dataset = tf.data.TFRecordDataset(path_tfrecord).cache()\n    raw_dataset = raw_dataset.map(lambda example: read_unlabeled_tfrecord(example), num_parallel_calls=AUTO)\n\n\n\n    for raw_record in raw_dataset:\n        samples_rates.add(raw_record[\"sample_rate\"].numpy())\n        audio_lengths.add(raw_record[\"audio_wav\"].shape[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(samples_rates)\nprint(audio_lengths)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"tfrecord have only samples with the same lengths and same samples rates. good."},{"metadata":{"trusted":true},"cell_type":"code","source":"for path_tfrecord in files_train:\n    raw_dataset = tf.data.TFRecordDataset(path_tfrecord).cache()\n    raw_dataset = raw_dataset.map(lambda example: read_labeled_tfrecord(example), num_parallel_calls=AUTO)\n\n\n\n    for i, raw_record in enumerate(raw_dataset):\n        if i  > 10:\n            break\n        print(raw_record[\"label_info\"].numpy().decode().strip())\n        \n    break","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}