{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Audio Augmentation \nThis notebook is using Google specaugment techniques for audio augmentation, which was used in speech recognition and some primary audio augmentation, just for experimenting purpose. It is interesting as it is the first time dealing with Audio in deep learning, decided to do some audio augmentation. We have taken a single mp3, but this can be done to the whole dataset.\nAugmentation done :\n* Time Shift\n* Speed Rate  Manipulation\n* Frequency Masking\n* Time Masking","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Importing Libraries","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import random\nimport librosa\nimport scipy\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport IPython.display as ipd\nimport cv2\nimport torch\nimport torchaudio\nfrom torchaudio import transforms\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"loading Audio file via Librosa","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"file_path = '../input/birdsong-recognition/train_audio/aldfly/XC134874.mp3'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"wav, sr = librosa.load(file_path, sr=None)\nprint(wav.shape, wav.max(), wav.min())\nipd.Audio(file_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_spectrogram(wav):\n    plotter = librosa.stft(wav, n_fft=480, hop_length=160,win_length=480, window='hamming')\n    spect, phase = librosa.magphase(plotter)\n    return spect","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Showing spectogram via log **","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"log_spect = np.log(show_spectrogram(wav))\nprint('spectrogram shape:', log_spect.shape)\nplt.imshow(log_spect, aspect='auto', origin='lower',)\nplt.title('spectrogram of origin audio')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Time Shifting  \n> Basic Time shifted according ratio taken by you","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"start_ = int(np.random.uniform(-18000,18000))\nprint('time shift: ',start_)\nif start_ >= 0:\n    wav_time_shift = np.r_[wav[start_:], np.random.uniform(-0.01,0.01, start_)]\nelse:\n    wav_time_shift = np.r_[np.random.uniform(-0.01,0.01, -start_), wav[:start_]]\nipd.Audio(wav_time_shift, rate=sr)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Output","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"EPS = 1e-8\nlog_spect = np.log(show_spectrogram(wav_time_shift)+EPS)\nprint('spectrogram shape:', log_spect.shape)\nplt.imshow(log_spect, aspect='auto', origin='lower',)\nplt.title('spectrogram of time shifted audio')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Speed Rate","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"speed_rate = np.random.uniform(0.7,1.3)\nwav_speed_tune = cv2.resize(wav, (1, int(len(wav) * speed_rate))).squeeze()\nprint('speed rate: %.3f' % speed_rate, '(lower is faster)')\nif len(wav_speed_tune) < 1223424:\n    pad_len = 1223424 - len(wav_speed_tune)\n    wav_speed_tune = np.r_[np.random.uniform(-0.001,0.001,int(pad_len/2)),wav_speed_tune,np.random.uniform(-0.001,0.001,int(np.ceil(pad_len/2)))]\nelse: \n    cut_len = len(wav_speed_tune) - 1223424\n    wav_speed_tune = wav_speed_tune[int(cut_len/2):int(cut_len/2)+1223424]\nprint('wav length: ', wav_speed_tune.shape[0])\nipd.Audio(wav_speed_tune, rate=sr)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Output","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"log_spect = np.log(show_spectrogram(wav_speed_tune)+EPS)\nprint('spectrogram shape:', log_spect.shape)\nplt.imshow(log_spect, aspect='auto', origin='lower',)\nplt.title('spectrogram of speed tuned audio')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loading Audio on pytorch","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"audio,sr = torchaudio.load(file_path)\nsample=(audio,sr)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Converting audio into Melspectrogram","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef tfm_spectro(ad, sr=16000, to_db_scale=False, n_fft=1024, \n                ws=None, hop=None, f_min=0.0, f_max=-80, pad=0, n_mels=128):\n    # We must reshape signal for torchaudio to generate the spectrogram.\n    mel = transforms.MelSpectrogram(sample_rate=ad[1], n_mels=n_mels, n_fft=n_fft, hop_length=hop, \n                                    f_min=f_min, f_max=f_max, pad=pad,)(ad[0].reshape(1, -1))\n    mel = mel.permute(0,2,1) # swap dimension, mostly to look sane to a human.\n    if to_db_scale: mel = transforms.AmplitudeToDB(stype='magnitude', top_db=f_max)(mel)\n    return mel\n\nspectro = tfm_spectro(sample, ws=512, hop=256, n_mels=128, to_db_scale=True, f_max=8000, f_min=-80)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#displaying\ndef tensor_to_img(spectrogram): \n    plt.imshow(spectrogram[0],aspect='auto', origin='lower')\n    plt.show();\n    display(spectrogram.shape)\ntensor_to_img(spectro)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Frequency Masking\nApply masking to a spectrogram in the frequency domain.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def freq_mask(spec, F=250, num_masks=1):\n    test = spec.clone()\n    num_mel_channels = test.shape[1]\n    for i in range(0, num_masks):        \n        freq = random.randrange(0, F)\n        zero = random.randrange(0, num_mel_channels - freq)\n        # avoids randrange error if values are equal and range is empty\n        if (zero == zero + freq): return test\n        mask_end = random.randrange(zero, zero + freq) \n        test[0][zero:mask_end] = test.mean()\n    return test","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Output","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def test_freq_mask():\n    print('Original')\n    tensor_to_img(spectro)\n    print('5 masks')\n    tensor_to_img(freq_mask(spectro, num_masks=5))\ntest_freq_mask()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Time Masking\nApplying masking in time domain","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def time_mask(spec, time=40, num_masks=1):\n    test = spec.clone()\n    length = test.shape[2]\n    for i in range(0, num_masks):\n        t = random.randrange(0, time)\n        zero = random.randrange(0, length - t)\n        if (zero == zero + t): return cloned\n        mask_end = random.randrange(zero, zero + t)\n        test[0][:,zero:mask_end] = test.mean()\n    return test","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Output","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def test_time_mask():\n    print('One Mask')\n    tensor_to_img(time_mask(spectro))\n    print('Two Mask')\n    tensor_to_img(time_mask(spectro, num_masks=2))\ntest_time_mask()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I have many changes left to do this was my first attempt and was inspired by SpecAugment. Please leave a like and any recommendation in comments.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}