{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":19596,"databundleVersionId":1292430,"sourceType":"competition"},{"sourceId":21669,"databundleVersionId":1692278,"sourceType":"competition"},{"sourceId":91844,"databundleVersionId":11361821,"sourceType":"competition"}],"dockerImageVersionId":29962,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Audio Albumentations\n\nIn this competition we needed similar Audio tool for creating nice training pipeline with augmentations for audio\n\n\n\n### Acknowledgement\n\n- [NLP Albumentations](https://www.kaggle.com/shonenkov/nlp-albumentations) - by [@Alex Shonenkov](https://www.kaggle.com/shonenkov)\n\n- [Data Augmentation for Audio](https://medium.com/@makcedward/data-augmentation-for-audio-76912b01fdf6) - by [Edward Ma](https://medium.com/@makcedward)","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport random\nimport numpy as np\nimport pandas as pd\nimport librosa\nimport matplotlib.pyplot as plt\nimport IPython.display as ipd\n\n\nfrom tqdm import tqdm\nfrom pydub import AudioSegment\n\nimport albumentations\nfrom albumentations.core.transforms_interface import DualTransform, BasicTransform\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"path = f\"../input/birdsong-recognition/train_audio/aldfly/XC134874.mp3\"\nsample_rate = 16000\nsound = AudioSegment.from_mp3(path)\nsound = sound.set_frame_rate(sample_rate)\n\ndata = np.array(sound.get_array_of_samples(), dtype=np.float32), sample_rate","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class AudioTransform(BasicTransform):\n    \"\"\"Transform for Audio task\"\"\"\n\n    @property\n    def targets(self):\n        return {\"data\": self.apply}\n    \n    def update_params(self, params, **kwargs):\n        if hasattr(self, \"interpolation\"):\n            params[\"interpolation\"] = self.interpolation\n        if hasattr(self, \"fill_value\"):\n            params[\"fill_value\"] = self.fill_value\n        return params","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Noise Injection\n\nIt simply add some random value into data by using numpy.","metadata":{}},{"cell_type":"code","source":"class NoiseInjection(AudioTransform):\n    \"\"\"It simply add some random value into data by using numpy\"\"\"\n    def __init__(self, always_apply=False, p=0.5):\n        super(NoiseInjection, self).__init__(always_apply, p)\n    \n    def apply(self, data, noise_levels=(0, 0.5), **params):\n        sound, sr = data\n        noise_level = np.random.uniform(*noise_levels)\n        noise = np.random.randn(len(sound))\n        augmented_sound = sound + noise_level * noise\n        # Cast back to same data type\n        augmented_sound = augmented_sound.astype(type(sound[0]))\n\n        return augmented_sound, sr","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transform = NoiseInjection(p=1.0)\nsound_aug, sr = transform(data=data)['data']\n\nplt.plot(data[0])\nplt.plot(sound_aug)\nplt.show()\n\ndisplay(ipd.Audio(data[0], rate=sr))\ndisplay(ipd.Audio(sound_aug, rate=sr))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Shifting Time\n\nThe idea of shifting time is very simple. It just shift audio to left/right with a random second. If shifting audio to left (fast forward) with x seconds, first x seconds will mark as 0 (i.e. silence). If shifting audio to right (back forward) with x seconds, last x seconds will mark as 0 (i.e. silence).","metadata":{}},{"cell_type":"code","source":"class ShiftingTime(AudioTransform):\n    \"\"\"Shifting time axis\"\"\"\n    def __init__(self, always_apply=False, p=0.5):\n        super(ShiftingTime, self).__init__(always_apply, p)\n    \n    def apply(self, data, **params):\n        sound, sr = data\n\n        shift_max = np.random.randint((len(sound)/sr)/2)\n        shift = np.random.randint(sr * shift_max)\n        direction = np.random.randint(0,2)\n        if direction == 1:\n            shift = -shift\n\n        augmented_sound = np.roll(sound, shift)\n        # Set to silence for heading/ tailing\n        if shift > 0:\n            augmented_sound[:shift] = 0\n        else:\n            augmented_sound[shift:] = 0\n\n        return augmented_sound, sr","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transform = ShiftingTime(p=1.0)\nsound_aug, sr = transform(data=data)['data']\n\nplt.plot(data[0])\nplt.plot(sound_aug)\nplt.show()\n\ndisplay(ipd.Audio(data[0], rate=sr))\ndisplay(ipd.Audio(sound_aug, rate=sr))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### PitchShift\n\nThis augmentation is a wrapper of librosa function. It change pitch randomly\n","metadata":{}},{"cell_type":"code","source":"class PitchShift(AudioTransform):\n    \"\"\"Shifting time axis\"\"\"\n    def __init__(self, always_apply=False, p=0.5):\n        super(PitchShift, self).__init__(always_apply, p)\n    \n    def apply(self, data, **params):\n        sound, sr = data\n\n        n_steps = np.random.randint(-10, 10)\n        augmented_sound = librosa.effects.pitch_shift(sound, sr, n_steps)\n\n        return augmented_sound, sr","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transform = PitchShift(p=1.0)\nsound_aug, sr = transform(data=data)['data']\n\nplt.plot(data[0])\nplt.plot(sound_aug)\nplt.show()\n\ndisplay(ipd.Audio(data[0], rate=sr))\ndisplay(ipd.Audio(sound_aug, rate=sr))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### TimeStretch\n\nSame as changing pitch, this augmentation is performed by librosa function. It stretches times series by a fixed rate.","metadata":{}},{"cell_type":"code","source":"class TimeStretch(AudioTransform):\n    \"\"\"Shifting time axis\"\"\"\n    def __init__(self, always_apply=False, p=0.5):\n        super(TimeStretch, self).__init__(always_apply, p)\n    \n    def apply(self, data, **params):\n        sound, sr = data\n\n        rate = np.random.uniform(0, 2)\n        augmented_sound = librosa.effects.time_stretch(sound, rate)\n\n        return augmented_sound, sr","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transform = TimeStretch(p=1.0)\nsound_aug, sr = transform(data=data)['data']\n\nplt.plot(data[0])\nplt.plot(sound_aug)\nplt.show()\n\ndisplay(ipd.Audio(data[0], rate=sr))\ndisplay(ipd.Audio(sound_aug, rate=sr))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### RandomAudio","metadata":{}},{"cell_type":"code","source":"class RandomAudio(AudioTransform):\n    \"\"\"Shifting time axis\"\"\"\n    def __init__(self,  seconds=5, always_apply=False, p=0.5):\n        super(RandomAudio, self).__init__(always_apply, p)\n\n        self.seconds = seconds\n    \n    def apply(self, data, **params):\n        sound, sr = data\n\n        shift = np.random.randint(len(sound))\n        trim_sound = np.roll(sound, shift)\n\n        min_samples = int(sr * self.seconds)\n\n        if len(trim_sound) < min_samples:\n            padding = min_samples - len(trim_sound)\n            offset = padding // 2\n            trim_sound = np.pad(trim_sound, (offset, padding - offset), \"constant\")\n        else:\n            trim_sound = trim_sound[:min_samples]\n\n        return trim_sound, sr","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transform = RandomAudio(p=1.0)\n\nsound_aug, sr = transform(data=data)['data']\n\nplt.plot(data[0])\nplt.plot(sound_aug)\nplt.show()\n\ndisplay(ipd.Audio(data[0], rate=sr))\ndisplay(ipd.Audio(sound_aug, rate=sr))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### MelSpectrogram\n\nComputes the Mel-scaled power spectrogram of an input signal.","metadata":{}},{"cell_type":"code","source":"class MelSpectrogram(AudioTransform):\n    \"\"\"Shifting time axis\"\"\"\n    def __init__(self, parameters, always_apply=False, p=0.5):\n        super(MelSpectrogram, self).__init__(always_apply, p)\n\n        self.parameters = parameters\n    \n    def apply(self, data, **params):\n        sound, sr = data\n\n        melspec = librosa.feature.melspectrogram(sound, sr=sr, **self.parameters)\n        melspec = librosa.power_to_db(melspec)\n        melspec = melspec.astype(np.float32)\n\n        return melspec, sr","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"melspectrogram_parameters = {\n        \"n_mels\": 128,\n        \"fmin\": 20,\n        \"fmax\": 32000\n    }\n\ntransform = MelSpectrogram(parameters=melspectrogram_parameters, p=1.0)\n\nmelspec, sr = transform(data=data)['data']\n\nplt.figure(figsize=(20,10))\nplt.imshow(melspec)\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### SpecAugment","metadata":{}},{"cell_type":"code","source":"class SpecAugment(AudioTransform):\n    \"\"\"Shifting time axis\"\"\"\n    def __init__(self, num_mask=2, freq_masking=0.15, time_masking=0.20, always_apply=False, p=0.5):\n        super(SpecAugment, self).__init__(always_apply, p)\n\n        self.num_mask = num_mask\n        self.freq_masking = freq_masking\n        self.time_masking = time_masking\n    \n    def apply(self, data, **params):\n        melspec, sr = data\n\n        spec_aug = self.spec_augment(melspec, \n                                     self.num_mask,\n                                     self.freq_masking,\n                                     self.time_masking,\n                                     melspec.min())\n        \n\n\n        return spec_aug, sr\n    \n    # Source: https://www.kaggle.com/davids1992/specaugment-quick-implementation\n    def spec_augment(self, \n                    spec: np.ndarray,\n                    num_mask=2,\n                    freq_masking=0.15,\n                    time_masking=0.20,\n                    value=0):\n        spec = spec.copy()\n        num_mask = random.randint(1, num_mask)\n        for i in range(num_mask):\n            all_freqs_num, all_frames_num  = spec.shape\n            freq_percentage = random.uniform(0.0, freq_masking)\n\n            num_freqs_to_mask = int(freq_percentage * all_freqs_num)\n            f0 = np.random.uniform(low=0.0, high=all_freqs_num - num_freqs_to_mask)\n            f0 = int(f0)\n            spec[f0:f0 + num_freqs_to_mask, :] = value\n\n            time_percentage = random.uniform(0.0, time_masking)\n\n            num_frames_to_mask = int(time_percentage * all_frames_num)\n            t0 = np.random.uniform(low=0.0, high=all_frames_num - num_frames_to_mask)\n            t0 = int(t0)\n            spec[:, t0:t0 + num_frames_to_mask] = value\n\n        return spec","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transform = SpecAugment(p=1.0)\ndata = melspec, sr\n\nspecAug, sr = transform(data=data)['data']\n\nplt.figure(figsize=(20,10))\nplt.imshow(specAug)\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### SpectToImage","metadata":{}},{"cell_type":"code","source":"class SpectToImage(AudioTransform):\n\n    def __init__(self, always_apply=False, p=0.5):\n        super(SpectToImage, self).__init__(always_apply, p)\n    \n    def apply(self, data, **params):\n        image, sr = data\n        delta = librosa.feature.delta(image)\n        accelerate = librosa.feature.delta(image, order=2)\n        image = np.stack([image, delta, accelerate], axis=-1)\n        image = image.astype(np.float32) / 100.0\n\n        return image","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transform = SpectToImage(p=1.0)\ndata = specAug, sr\n\nimage = transform(data=data)['data']\n\nplt.figure(figsize=(20,10))\nplt.imshow(image)\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### All in one","metadata":{}},{"cell_type":"code","source":"audio_augmentation = albumentations.Compose([\n     RandomAudio(always_apply=True),\n     NoiseInjection(p=1),\n     MelSpectrogram(parameters=melspectrogram_parameters,always_apply=True),\n     SpecAugment(p=1),\n     SpectToImage(always_apply=True)\n])\n\ndata = np.array(sound.get_array_of_samples(), dtype=np.float32), sample_rate\nimage = audio_augmentation(data=data)['data']\n\nplt.imshow(image)\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Thank you for reading my kernel\n### More to come stay tuned","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}