{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## What are TPUs?\nThe Tensor Processing Unit (TPU) is a custom integrated chip, designed specifically to accelerate the process of training machine learning models. \n\n## TPUs for free at Kaggle\n**You can use up to 30 hours per week of TPUs and up to 9h at a time in a single session.**\n**For more info you can visit [here](https://www.kaggle.com/docs/tpu).**\n\n## Why do we need TFRecord format?\nThe TFRecord format is tensorflow's custom data format which is simple for storing a sequence of binary records. The advantages of using TFRecords are amazingly more efficient storage, fast I/O, self-contained files, etc. The main advantage of TPUs are faster I/O which results in faster model training.\n\nFor understanding the basics of TFRecords, please visit Ryan Holbrook notebook: [TFRecords Basics](https://www.kaggle.com/ryanholbrook/tfrecords-basics).\n\n**In this notebook you will learn how to convert audio dataset into TFRecord format.**\n\n# Useful resources which helped me:\n- https://www.kaggle.com/servietsky/fast-import-audio-and-save-spectrograms/notebook\n- https://www.tensorflow.org/tutorials/load_data/tfrecord\n- https://www.kaggle.com/mgornergoogle/five-flowers-with-keras-and-xception-on-tpu\n- https://towardsdatascience.com/a-practical-guide-to-tfrecords-584536bc786c\n- https://keras.io/examples/keras_recipes/creating_tfrecords/\n- https://www.kaggle.com/lqdisme/dog-breed-identification\n- https://cloud.google.com/blog/products/ai-machine-learning/what-makes-tpus-fine-tuned-for-deep-learning\n- https://pub.towardsai.net/writing-tfrecord-files-the-right-way-7c3cee3d7b12\n- https://www.kaggle.com/daisukelab/creating-fat2019-preprocessed-data","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-10-12T08:59:41.137457Z","iopub.execute_input":"2021-10-12T08:59:41.137714Z","iopub.status.idle":"2021-10-12T08:59:41.147033Z","shell.execute_reply.started":"2021-10-12T08:59:41.137687Z","shell.execute_reply":"2021-10-12T08:59:41.14468Z"}}},{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport tensorflow as tf\nimport numpy as np \nimport pandas as pd\nimport os, random\nimport librosa\nimport matplotlib.pyplot as plt\nimport gc\nimport time\nfrom tqdm import tqdm, tqdm_notebook; tqdm.pandas() # Progress bar\nimport math\n\nfrom tensorflow.keras.utils import to_categorical\nseed = 1234\nnp.random.seed(seed)\n\nt_start = time.time()\n\nimport warnings\n\ndef fxn():\n    warnings.warn(\"deprecated\", DeprecationWarning)\n\nwith warnings.catch_warnings():\n    warnings.simplefilter(\"ignore\")\n    fxn()\nwarnings.filterwarnings(\"ignore\")\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'","metadata":{"execution":{"iopub.status.busy":"2022-02-18T09:04:09.032967Z","iopub.execute_input":"2022-02-18T09:04:09.033472Z","iopub.status.idle":"2022-02-18T09:04:15.201055Z","shell.execute_reply.started":"2022-02-18T09:04:09.033383Z","shell.execute_reply":"2022-02-18T09:04:15.20019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Paramaters\ntaking only 10 samples from each folder to save time ","metadata":{}},{"cell_type":"code","source":"# Preprocessing parameters (it can be changed according to convenience)\nSEED = 42\nFRAC = 0.2     # Validation fraction\nSR = 44100     # sampling rate\nMAXLEN= 60    # seconds\nN_MELS = 128\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\nseed_everything(SEED)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T09:04:15.203143Z","iopub.execute_input":"2022-02-18T09:04:15.203398Z","iopub.status.idle":"2022-02-18T09:04:15.209394Z","shell.execute_reply.started":"2022-02-18T09:04:15.203363Z","shell.execute_reply":"2022-02-18T09:04:15.207914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Processing and encoding labels","metadata":{}},{"cell_type":"code","source":"ROOT = \"/kaggle/input/birdsong-recognition/\"\n\n# Read train\ndf = pd.read_csv(os.path.join(ROOT, 'train.csv'))[['ebird_code', 'filename', 'duration']]\ndf['path'] = ROOT+'train_audio/' + df['ebird_code'] + \"/\" + df['filename']\n\nclasses = set(random.sample(df['ebird_code'].unique().tolist(), 15)) \n\n\ndf = df[df.ebird_code.apply(lambda x: x in classes)].reset_index(drop=True)\nkeys = set(df.ebird_code)\nvalues = np.arange(0, len(keys))\ncode_dict = dict(zip(sorted(keys), values))\ndf['label'] = df['ebird_code'].apply(lambda x: code_dict[x])","metadata":{"execution":{"iopub.status.busy":"2022-02-18T09:04:15.211121Z","iopub.execute_input":"2022-02-18T09:04:15.211397Z","iopub.status.idle":"2022-02-18T09:04:16.733433Z","shell.execute_reply.started":"2022-02-18T09:04:15.211363Z","shell.execute_reply":"2022-02-18T09:04:16.732572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing functions\n","metadata":{}},{"cell_type":"code","source":"samples = SR* 5\ndef read_audio(path):\n    '''\n    Reads in the audio file and returns\n    an array that we can turn into a melspectogram\n    '''\n    y, _ = librosa.core.load(path, sr=44100)\n    # trim silence\n    if 0 < len(y): # workaround: 0 length causes error\n        y, _ = librosa.effects.trim(y)\n    if len(y) > samples: # long enough\n        y = y[0:0+samples]\n    else: # pad blank\n        padding = samples - len(y)\n        offset = padding // 2\n        y = np.pad(y, (offset, samples - len(y) - offset), 'constant')\n    return y\n\ndef audio_to_melspectrogram(audio):\n    '''\n    Convert to melspectrogram after audio is read in\n    '''\n    spectrogram = librosa.feature.melspectrogram(audio, \n                                                 sr=SR,\n                                                 n_mels=N_MELS,\n                                                 hop_length=347,\n                                                 n_fft=N_MELS,\n                                                 fmin=20,\n                                                 fmax=SR//2)\n    return librosa.power_to_db(spectrogram).astype(np.float32)\n\ndef read_as_melspectrogram(path):\n    '''\n    Convert audio into a melspectrogram \n    so we can use machine learning\n    '''\n    mels = audio_to_melspectrogram(read_audio(path))\n    return mels\n\ndef convert_wav_to_image(df, path):\n    X = []\n    for _,row in tqdm_notebook(df.iterrows()):\n        if row['filename'] != 'XC195038.mp3' :\n            x = read_as_melspectrogram('{}/{}/{}'.format(path[0],str(row['ebird_code']) ,str(row['filename'])))\n            X.append(x.transpose())\n    return X\n\ndef normalize(img):\n    '''\n    Normalizes an array \n    (subtract mean and divide by standard deviation)\n    '''\n    eps = 0.001\n    if np.std(img) != 0:\n        img = (img - np.mean(img)) / np.std(img)\n    else:\n        img = (img - np.mean(img)) / eps\n    return img\n\ndef normalize_dataset(X):\n    '''\n    Normalizes list of arrays\n    (subtract mean and divide by standard deviation)\n    '''\n    normalized_dataset = []\n    for img in X:\n        normalized = normalize(img)\n        normalized_dataset.append(normalized)\n    return normalized_dataset","metadata":{"execution":{"iopub.status.busy":"2022-02-18T09:04:19.331175Z","iopub.execute_input":"2022-02-18T09:04:19.33172Z","iopub.status.idle":"2022-02-18T09:04:19.344669Z","shell.execute_reply.started":"2022-02-18T09:04:19.331676Z","shell.execute_reply":"2022-02-18T09:04:19.343998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = np.array(convert_wav_to_image(df, ['../input/birdsong-recognition/train_audio']))\nnormalized_x = normalize_dataset(X)\nreshape_x = np.array(normalized_x)\nreshape_x = np.reshape(reshape_x, (-1,636,128,1))","metadata":{"execution":{"iopub.status.busy":"2022-02-18T09:04:20.450158Z","iopub.execute_input":"2022-02-18T09:04:20.450676Z","iopub.status.idle":"2022-02-18T09:15:27.466199Z","shell.execute_reply.started":"2022-02-18T09:04:20.450635Z","shell.execute_reply":"2022-02-18T09:15:27.465324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# One-hot encoding the labels","metadata":{}},{"cell_type":"code","source":"y = df['label'].values\nlabels = to_categorical(y, num_classes=15)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T09:16:38.620314Z","iopub.execute_input":"2022-02-18T09:16:38.621012Z","iopub.status.idle":"2022-02-18T09:16:38.630181Z","shell.execute_reply.started":"2022-02-18T09:16:38.620971Z","shell.execute_reply":"2022-02-18T09:16:38.628792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize the spectrograms","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15,10))\nplt.imshow(X[0]);\nprint(y[0])","metadata":{"execution":{"iopub.status.busy":"2022-02-18T09:16:53.453224Z","iopub.execute_input":"2022-02-18T09:16:53.4539Z","iopub.status.idle":"2022-02-18T09:16:53.721939Z","shell.execute_reply.started":"2022-02-18T09:16:53.453863Z","shell.execute_reply":"2022-02-18T09:16:53.721258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Divide into train and test","metadata":{}},{"cell_type":"code","source":"X_train = reshape_x[:-333]\nX_test = reshape_x[:333]\ny_train = labels[:-333]\ny_test = labels[:333]","metadata":{"execution":{"iopub.status.busy":"2022-02-18T09:21:28.248474Z","iopub.execute_input":"2022-02-18T09:21:28.248728Z","iopub.status.idle":"2022-02-18T09:21:28.558097Z","shell.execute_reply.started":"2022-02-18T09:21:28.248697Z","shell.execute_reply":"2022-02-18T09:21:28.557231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Shape of X_train:\", X_train.shape)\nprint(\"Shape of y_train:\", y_train.shape)\nprint(\"Shape of X_test:\", X_test.shape)\nprint(\"Shape of y_test:\", y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T09:21:29.22742Z","iopub.execute_input":"2022-02-18T09:21:29.227942Z","iopub.status.idle":"2022-02-18T09:21:29.234834Z","shell.execute_reply.started":"2022-02-18T09:21:29.227888Z","shell.execute_reply":"2022-02-18T09:21:29.234175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Funtions for feature creation\nThe following functions can be used to convert a value to a type compatible which takes a scalar input values and returns a tf.train.Feature.","metadata":{}},{"cell_type":"code","source":"def _bytes_feature(value):\n    if isinstance(value, type(tf.constant(0))): # if value ist tensor\n        value = value.numpy() # get value of tensor\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n    return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _int64_feature(value):\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\ndef serialize_array(array):\n    array = tf.io.serialize_tensor(array)\n    return array","metadata":{"execution":{"iopub.status.busy":"2022-02-18T09:21:33.988877Z","iopub.execute_input":"2022-02-18T09:21:33.989589Z","iopub.status.idle":"2022-02-18T09:21:33.996555Z","shell.execute_reply.started":"2022-02-18T09:21:33.989549Z","shell.execute_reply":"2022-02-18T09:21:33.995809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Writing and Converting to TFRecord\nNow, we'll create a dictionary to store the actual image, height, width and depth of the image and the label where we first serialize the array and then convert it to a bytes_feature. All these key:value mappings make up the features for one Example.","metadata":{}},{"cell_type":"code","source":"def parse_single_image(image, label=None):\n    if label is None:\n            data = {\n                'height' : _int64_feature(image.shape[0]),\n                'width' : _int64_feature(image.shape[1]),\n                'depth' : _int64_feature(image.shape[2]),\n                'raw_image' : _bytes_feature(serialize_array(image))\n    }\n    else:\n        data = {\n                'height' : _int64_feature(image.shape[0]),\n                'width' : _int64_feature(image.shape[1]),\n                'depth' : _int64_feature(image.shape[2]),\n                'raw_image' : _bytes_feature(serialize_array(image)),\n                'label' : _bytes_feature(serialize_array(label))\n        }\n    out = tf.train.Example(features=tf.train.Features(feature=data))\n\n    return out","metadata":{"execution":{"iopub.status.busy":"2022-02-18T09:21:35.385729Z","iopub.execute_input":"2022-02-18T09:21:35.38627Z","iopub.status.idle":"2022-02-18T09:21:35.392986Z","shell.execute_reply.started":"2022-02-18T09:21:35.386229Z","shell.execute_reply":"2022-02-18T09:21:35.392021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tqdm\ndef write_images_to_tfr(images, filename, labels=None):\n    max_files=500\n    out_dir=\"./\"\n    splits = (len(images)//max_files) + 1 #determine how many shards are needed\n    if len(images)%max_files == 0:\n        splits-=1\n    print(f\"\\nUsing {splits} shard(s) for {len(images)} files, with up to {max_files} samples per shard\")\n    \n    file_count = 0\n\n    for i in tqdm.tqdm(range(splits)):\n        current_shard_name = \"{}{}_{}{}.tfrecords\".format(out_dir, i+1, splits, filename)\n        writer = tf.io.TFRecordWriter(current_shard_name)\n\n        current_shard_count = 0\n        while current_shard_count < max_files: \n            index = i*max_files+current_shard_count\n            if index == len(images): \n                break\n            if labels is None:  \n                current_image = images[index]\n                out = parse_single_image(image=current_image)\n\n            else:\n                current_image = images[index]\n                current_label = labels[index]\n                out = parse_single_image(image=current_image, label=current_label)\n    \n            writer.write(out.SerializeToString())\n            current_shard_count+=1\n            file_count += 1\n\n        writer.close()\n    print(f\"\\nWrote {file_count} elements to TFRecord\")\n    return file_count","metadata":{"execution":{"iopub.status.busy":"2022-02-18T09:21:36.16584Z","iopub.execute_input":"2022-02-18T09:21:36.166403Z","iopub.status.idle":"2022-02-18T09:21:36.175106Z","shell.execute_reply.started":"2022-02-18T09:21:36.166365Z","shell.execute_reply":"2022-02-18T09:21:36.174349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"write_images_to_tfr(X_train, \"train_tfrecord\", y_train)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T09:21:36.992121Z","iopub.execute_input":"2022-02-18T09:21:36.992763Z","iopub.status.idle":"2022-02-18T09:21:39.660501Z","shell.execute_reply.started":"2022-02-18T09:21:36.992724Z","shell.execute_reply":"2022-02-18T09:21:39.659804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"write_images_to_tfr(X_test, \"test_tfrecord\", y_test)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T09:21:39.662074Z","iopub.execute_input":"2022-02-18T09:21:39.66248Z","iopub.status.idle":"2022-02-18T09:21:40.131699Z","shell.execute_reply.started":"2022-02-18T09:21:39.662436Z","shell.execute_reply":"2022-02-18T09:21:40.131021Z"},"trusted":true},"execution_count":null,"outputs":[]}]}