{"cells":[{"metadata":{},"cell_type":"markdown","source":"# librosaの勉強\n\n+ https://librosa.org/doc/latest/tutorial.html","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Library ","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport librosa\nimport librosa.display\nfrom IPython.display import Audio, IFrame, display # jupyterで再生につかう","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(librosa.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 1. 音声ファイルのpathを設定\n#filename  = librosa.example('nutcracker')\nfilename ='../input/birdsong-resampled-train-audio-04/wooscj2/XC67042.wav'\n\n# 2. `y`：波形\n#    `sr`：サンプリングレート\n# デフォルトだとモノラル、sr=22050\ny, sr = librosa.load(filename )\n# type(y) -> numpy.ndarray\n# y.shape -> (144577,) \n# sr      -> 22050\n\n# 2-1. サンプリングレートを指定できる\ny, sr = librosa.load(filename, sr=32000)\n# y.shape -> (209816,)\n# sr      -> 32000\n\n# 3. デフォルトのビートトラッカーを実行(bpm推定)\ntempo, beat_frames = librosa.beat.beat_track(y=y, sr=sr)\n# type(tempo) -> numpy.float64\n# tempo       -> 113.63636363636364\n# beat_frames -> array([  3,  38,  71, 101, 132, 165, 196])\n\n# 4. ビートイベントのフレームインデックスをタイムスタンプに変換\nbeat_times = librosa.frames_to_time(beat_frames, sr=sr)\n# beat_times -> array([0.048, 0.608, 1.136, 1.616, 2.112, 2.64 , 3.136])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 表示\nlibrosa.display.waveplot(y, sr=sr)\nAudio(y, rate=sr)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 調波打楽器音分離 (HPSS: Hermonic/Percussive Source Seperation)\n\n打楽器の音と非打楽器（調波楽器）の音を分離する  \nhttps://www.wizard-notes.com/entry/music-analysis/hpss#f-4f2dfd09","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Separate harmonics and percussives into two waveforms\ny_harmonic, y_percussive = librosa.effects.hpss(y)\n\n# Plot\nplt.subplot(3,1,1)\nplt.plot(y)\nplt.title(\"Original signal\")\n\nplt.subplot(3,1,2)\nplt.plot(y_harmonic)\nplt.title(\"Harmonic signal\")\n\nplt.subplot(3,1,3)\nplt.plot(y_percussive)\nplt.title(\"Percussive signal\")\n\nplt.tight_layout()\nplt.show()\n\nprint('y_harmonic')\ndisplay(Audio(y_harmonic,rate=sr))\nprint('y_percussive')\ndisplay(Audio(y_percussive,rate=sr))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### メル周波数ケプストラム係数(MFCC:Mel-Frequency Cepstrum Coefficients)\n\n+ https://www.wizard-notes.com/entry/music-analysis/insts-timbre-with-mfcc\n  + n_mfcc:分析や機械学習で使う場合は12～24くらいの次元数が良く使われます。\n\n```メル尺度（メルしゃくど、英語: mel scale）は、音高の知覚的尺度である。メル尺度の差が同じであれば、人間が感じる音高の差が同じになることを意図している。```  \n人間の聴覚特性に合わせた変換\n\n### log-melspectrogram\n\n```MFCCの離散コサイン変換無いバージョンです。```\n+ [機械学習のための音声の特徴量ざっくりメモ (Librosa ,numpy)](https://qiita.com/yutalfa/items/dbd172138db60d461a56)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"mfcc = librosa.feature.mfcc(y=y, sr=sr,n_mfcc=24)\n#mfcc.shape -> (24, 410)\nlibrosa.display.specshow(mfcc, sr=sr)\nplt.colorbar()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"melspec = librosa.feature.melspectrogram(y, sr=sr, n_mels=128, fmin=20, fmax=16000)\n# デシベル変換\nmelspec = librosa.power_to_db(melspec).astype(np.float32)\n# melspec.shape -> (128, 410)\nlibrosa.display.specshow(melspec, sr=sr)\nplt.colorbar()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 入力の列間の（平滑化された）一次差を計算．\n# mfccと同じ形状のものを出力\nmfcc_delta = librosa.feature.delta(mfcc)\n#mfcc_delta.shape -> (24, 410)\nlibrosa.display.specshow(mfcc_delta, sr=sr)\nplt.colorbar()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"その他処理は下記\nhttps://librosa.org/doc/latest/tutorial.html","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### sfとlibrosaの比較","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import soundfile as sf\nfilename ='../input/birdsong-resampled-train-audio-04/wooscj2/XC67042.wav'\n\ny, sr = librosa.load(filename , sr=32000)\ny_1, sr_1 = sf.read(filename)\n\nprint(f\"librosa \\n shape{y.shape},\\n type{type(y[0])}\\n size{y.__sizeof__()}\")\nprint(\"-\"*50)\nprint(f\"soundfile \\n shape{y_1.shape},\\n type{type(y_1[0])}\\n size{y_1.__sizeof__()}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# ノイズ消す実験","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# trainは下記コード(pip)でもいいがtestでは使えないので下記の長文のコードをそのままはるのがいいか？\n#!pip install noisereduce","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n#import noisereduce as nr\n\nimport soundfile as sf\nimport librosa\nimport librosa.display\nfrom IPython.display import Audio, IFrame, display # jupyterで再生につかう\n\nfrom scipy.ndimage import maximum_filter1d","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://github.com/timsainb/noisereduce\nimport scipy.signal\nimport numpy as np\nimport librosa\nfrom tqdm.autonotebook import tqdm\nimport warnings\n\ndef _stft(y, n_fft, hop_length, win_length, use_tensorflow=False):\n    if use_tensorflow:\n        # return librosa.stft(y=y, n_fft=n_fft, hop_length=hop_length, win_length=win_length, center=True)\n        return _stft_tensorflow(y, n_fft, hop_length, win_length)\n    else:\n        return librosa.stft(\n            y=y, n_fft=n_fft, hop_length=hop_length, win_length=win_length, center=True\n        )\n\n\ndef _istft(y, n_fft, hop_length, win_length, use_tensorflow=False):\n    if use_tensorflow:\n        # return librosa.istft(y, hop_length, win_length)\n        return _istft_tensorflow(y.T, n_fft, hop_length, win_length)\n    else:\n        return librosa.istft(y, hop_length, win_length)\n\n\ndef _stft_librosa(y, n_fft, hop_length, win_length):\n    return librosa.stft(\n        y=y, n_fft=n_fft, hop_length=hop_length, win_length=win_length, center=True\n    )\n\n\ndef _istft_librosa(y, hop_length, win_length):\n    return librosa.istft(y, hop_length, win_length)\n\n\ndef _stft_tensorflow(y, n_fft, hop_length, win_length):\n    return (\n        tf.signal.stft(\n            y,\n            win_length,\n            hop_length,\n            n_fft,\n            pad_end=True,\n            window_fn=tf.signal.hann_window,\n        )\n        .numpy()\n        .T\n    )\n\n\ndef _istft_tensorflow(y, n_fft, hop_length, win_length):\n    return tf.signal.inverse_stft(\n        y.astype(np.complex64), win_length, hop_length, n_fft\n    ).numpy()\n\n\ndef _amp_to_db(x):\n    return librosa.core.amplitude_to_db(x, ref=1.0, amin=1e-20, top_db=80.0)\n\n\ndef _db_to_amp(x,):\n    return librosa.core.db_to_amplitude(x, ref=1.0)\n\n\ndef update_pbar(pbar, message):\n    \"\"\" writes to progress bar\n    \"\"\"\n    if pbar is not None:\n        pbar.set_description(message)\n        pbar.update(1)\n\n\ndef _smoothing_filter(n_grad_freq, n_grad_time):\n    \"\"\"Generates a filter to smooth the mask for the spectrogram\n        \n    Arguments:\n        n_grad_freq {[type]} -- [how many frequency channels to smooth over with the mask.]\n        n_grad_time {[type]} -- [how many time channels to smooth over with the mask.]\n    \"\"\"\n\n    smoothing_filter = np.outer(\n        np.concatenate(\n            [\n                np.linspace(0, 1, n_grad_freq + 1, endpoint=False),\n                np.linspace(1, 0, n_grad_freq + 2),\n            ]\n        )[1:-1],\n        np.concatenate(\n            [\n                np.linspace(0, 1, n_grad_time + 1, endpoint=False),\n                np.linspace(1, 0, n_grad_time + 2),\n            ]\n        )[1:-1],\n    )\n    smoothing_filter = smoothing_filter / np.sum(smoothing_filter)\n    return smoothing_filter\n\n\ndef mask_signal(sig_stft, sig_mask):\n    \"\"\" Reduces amplitude of time/frequency regions of a spectrogram based upon a mask \n        \n    Arguments:\n        sig_stft {[type]} -- spectrogram of signal\n        sig_mask {[type]} -- mask to apply to signal\n    \n    Returns:\n        sig_stft_amp [type] -- masked signal\n    \"\"\"\n    sig_stft_amp = sig_stft * (1 - sig_mask)\n    return sig_stft_amp\n\n\ndef convolve_gaussian(sig_mask, smoothing_filter, use_tensorflow=False):\n    \"\"\" Convolves a gaussian filter with a mask (or any image)\n    \n    Arguments:\n        sig_mask {[type]} -- The signal mask\n        smoothing_filter {[type]} -- the filter to convolve\n    \n    Keyword Arguments:\n        use_tensorflow {bool} -- use tensorflow.signal or scipy.signal (default: {False})\n    \"\"\"\n    if use_tensorflow:\n        smoothing_filter = smoothing_filter * (\n            (np.shape(smoothing_filter)[1] - 1) / 2 + 1\n        )\n        smoothing_filter = smoothing_filter[:, :, tf.newaxis, tf.newaxis].astype(\n            \"float32\"\n        )\n        img = sig_mask[:, :, tf.newaxis, tf.newaxis].astype(\"float32\")\n        return (\n            tf.nn.conv2d(img, smoothing_filter, strides=[1, 1, 1, 1], padding=\"SAME\")\n            .numpy()\n            .squeeze()\n        )\n    else:\n        return scipy.signal.fftconvolve(sig_mask, smoothing_filter, mode=\"same\")\n\n\ndef load_tensorflow(verbose=False):\n    \"\"\"loads tensorflow if it is available\n    Used as a backend for fft and convolution\n    \n    Returns:\n        bool -- whether to use tensorflow\n    \"\"\"\n    try:\n        # import tensorflow as tf\n        globals()[\"tf\"] = __import__(\"tensorflow\")\n\n        if verbose:\n            available_gpus = tf.config.experimental.list_physical_devices(\"GPU\")\n            print(\"GPUs available: {}\".format(available_gpus))\n        if int(tf.__version__[0]) < 2:\n            warnings.warn(\n                \"Tensorflow version is below 2.0, reverting to non-tensorflow backend\"\n            )\n            return False\n    except:\n        warnings.warn(\n            \"Tensorflow is not installed, reverting to non-tensorflow backend\"\n        )\n        return False\n    return True\n\n\ndef reduce_noise(\n    audio_clip,\n    noise_clip,\n    n_grad_freq=2,\n    n_grad_time=4,\n    n_fft=2048,\n    win_length=2048,\n    hop_length=512,\n    n_std_thresh=1.5,\n    prop_decrease=1.0,\n    pad_clipping=True,\n    use_tensorflow=False,\n    verbose=False,\n):\n    \"\"\"Remove noise from audio based upon a clip containing only noise\n    Args:\n        audio_clip (array): The first parameter.\n        noise_clip (array): The second parameter.\n        n_grad_freq (int): how many frequency channels to smooth over with the mask.\n        n_grad_time (int): how many time channels to smooth over with the mask.\n        n_fft (int): number audio of frames between STFT columns.\n        win_length (int): Each frame of audio is windowed by `window()`. The window will be of length `win_length` and then padded with zeros to match `n_fft`..\n        hop_length (int):number audio of frames between STFT columns.\n        n_std_thresh (int): how many standard deviations louder than the mean dB of the noise (at each frequency level) to be considered signal\n        prop_decrease (float): To what extent should you decrease noise (1 = all, 0 = none)\n        pad_clipping (bool): Pad the signals with zeros to ensure that the reconstructed data is equal length to the data\n        use_tensorflow (bool): Use tensorflow as a backend for convolution and fft to speed up computation\n        verbose (bool): Whether to plot the steps of the algorithm\n    Returns:\n        array: The recovered signal with noise subtracted\n    \"\"\"\n    # load tensorflow if you are using it as a backend\n    if use_tensorflow:\n        use_tensorflow = load_tensorflow(verbose)\n\n    if verbose:\n        pbar = tqdm(total=7)\n    else:\n        pbar = None\n\n    update_pbar(pbar, \"STFT on noise\")\n    # STFT over noise\n    noise_stft = _stft(\n        noise_clip, n_fft, hop_length, win_length, use_tensorflow=use_tensorflow\n    )\n    noise_stft_db = _amp_to_db(np.abs(noise_stft))  # convert to dB\n    # Calculate statistics over noise\n    update_pbar(pbar, \"STFT on signal\")\n    mean_freq_noise = np.mean(noise_stft_db, axis=1)\n    std_freq_noise = np.std(noise_stft_db, axis=1)\n    noise_thresh = mean_freq_noise + std_freq_noise * n_std_thresh\n    # STFT over signal\n    update_pbar(pbar, \"STFT on signal\")\n\n    # pad signal with zeros to avoid extra frames being clipped if desired\n    if pad_clipping:\n        nsamp = len(audio_clip)\n        audio_clip = np.pad(audio_clip, [0, hop_length], mode=\"constant\")\n\n    sig_stft = _stft(\n        audio_clip, n_fft, hop_length, win_length, use_tensorflow=use_tensorflow\n    )\n    # spectrogram of signal in dB\n    sig_stft_db = _amp_to_db(np.abs(sig_stft))\n    update_pbar(pbar, \"Generate mask\")\n\n    # calculate the threshold for each frequency/time bin\n    db_thresh = np.repeat(\n        np.reshape(noise_thresh, [1, len(mean_freq_noise)]),\n        np.shape(sig_stft_db)[1],\n        axis=0,\n    ).T\n    # mask if the signal is above the threshold\n    sig_mask = sig_stft_db < db_thresh\n    update_pbar(pbar, \"Smooth mask\")\n    # Create a smoothing filter for the mask in time and frequency\n    smoothing_filter = _smoothing_filter(n_grad_freq, n_grad_time)\n\n    # convolve the mask with a smoothing filter\n    sig_mask = convolve_gaussian(sig_mask, smoothing_filter, use_tensorflow)\n\n    sig_mask = sig_mask * prop_decrease\n    update_pbar(pbar, \"Apply mask\")\n    # mask the signal\n\n    sig_stft_amp = mask_signal(sig_stft, sig_mask)\n\n    update_pbar(pbar, \"Recover signal\")\n    # recover the signal\n    recovered_signal = _istft(\n        sig_stft_amp, n_fft, hop_length, win_length, use_tensorflow=use_tensorflow\n    )\n    # fix the recovered signal length if padding signal\n    if pad_clipping:\n        recovered_signal = librosa.util.fix_length(recovered_signal, nsamp)\n\n    recovered_spec = _amp_to_db(\n        np.abs(\n            _stft(\n                recovered_signal,\n                n_fft,\n                hop_length,\n                win_length,\n                use_tensorflow=use_tensorflow,\n            )\n        )\n    )\n\n    return recovered_signal","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def envelope(y, rate, threshold):\n    mask = []\n    y_mean = maximum_filter1d(np.abs(y), mode=\"constant\", size=rate//20)\n    for mean in y_mean:\n        if mean > threshold:\n            mask.append(True)\n        else:\n            mask.append(False)\n    return mask, y_mean","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ノイズ消すやつ\nfilename ='../input/birdsong-resampled-train-audio-04/sagthr/XC112650.wav' # 長い\n#filename ='../input/birdsong-resampled-train-audio-04/wooscj2/XC67042.wav' # 短い\nfilename = '../input/birdsong-recognition/train_audio/bkcchi/XC114073.mp3'\n#y, sr = sf.read(filename)\ny, sr = librosa.load(filename,sr=32000,mono=True,res_type=\"kaiser_fast\")\nprint(\"sampling rate:\", sr)\nplt.plot(y)\nplt.show()\n\nAudio(data=y, rate=sr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"thr = 0.25\nmask, env = envelope(y, sr, thr)\n\nplt.plot(y[mask], label=\"birdcall\")\nplt.plot(y[np.logical_not(mask)], label=\"noise\")\nplt.legend(bbox_to_anchor=(1, 1), loc='upper right')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_denoise = reduce_noise(audio_clip=y, noise_clip=y[np.logical_not(mask)], verbose=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'{y.shape}{y_denoise.shape}')\nprint('y')\ndisplay(Audio(y,rate=sr))\nprint('y_denoise')\ndisplay(Audio(y_denoise,rate=sr))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"melspec = librosa.feature.melspectrogram(y, sr=sr, n_mels=128, fmin=20, fmax=16000)\n# デシベル変換\nmelspec = librosa.power_to_db(melspec).astype(np.float32)\n# melspec.shape -> (128, 410)\nlibrosa.display.specshow(melspec, sr=sr)\nplt.colorbar()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"melspec_denoise = librosa.feature.melspectrogram(y_denoise, sr=sr, n_mels=128, fmin=20, fmax=16000)\n# デシベル変換\nmelspec_denoise = librosa.power_to_db(melspec_denoise).astype(np.float32)\n# melspec.shape -> (128, 410)\nlibrosa.display.specshow(melspec_denoise, sr=sr)\nplt.colorbar()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## SpecAugment試験","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport librosa\nimport librosa.display\nfrom IPython.display import Audio, IFrame, display # jupyterで再生につかう\nimport soundfile as sf\nimport random","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"filename ='../input/birdsong-resampled-train-audio-04/wooscj2/XC67042.wav'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class SpecAugmentTransform(object):\n    def __init__(self,\n                 do_time_warp=False, #未完成なのでFalse\n                 do_freq_mask=True, \n                 do_time_mask=True,\n                 num_freq_mask=1,\n                 F=27,\n                 num_time_mask=1,\n                 time=100):\n        self.do_time_warp = do_time_warp\n        self.do_freq_mask = do_freq_mask\n        self.do_time_mask = do_time_mask\n        self.num_freq_mask = num_freq_mask\n        self.F = F\n        self.num_time_mask = num_time_mask\n        self.time = time\n\n    # コールされる関数\n    def __call__(self, spec):\n        return self.transform(spec)\n\n    # 1文をidに変換する\n    def transform(self, spec):\n        # 指定があれば始まりと終わりの記号を追加する\n        if self.do_time_warp:\n            spec = self.time_warp(spec=spec, W=5)\n        if self.do_freq_mask:\n            spec = self.freq_mask(spec=spec, F=self.F, num_masks=self.num_freq_mask)\n        if self.do_time_mask:\n            spec = self.time_mask(spec=spec, time=self.time, num_masks=self.num_time_mask)\n\n        return spec\n\n    #未完成\n    def time_warp(self, spec, W=5):\n        num_rows = spec.shape[1]\n        spec_len = spec.shape[2]\n        device = spec.device\n\n        y = num_rows//2\n        horizontal_line_at_ctr = spec[0][y]\n        assert len(horizontal_line_at_ctr) == spec_len\n\n        point_to_warp = horizontal_line_at_ctr[random.randrange(W, spec_len - W)]\n        assert isinstance(point_to_warp, torch.Tensor)\n\n        # Uniform distribution from (0,W) with chance to be up to W negative\n        dist_to_warp = random.randrange(-W, W)\n        src_pts, dest_pts = (torch.tensor([[[y, point_to_warp]]], device=device), \n                             torch.tensor([[[y, point_to_warp + dist_to_warp]]], device=device))\n        warped_spectro, dense_flows = sparse_image_warp(spec, src_pts, dest_pts)\n        return warped_spectro.squeeze(3)\n    \n    #ランダムな周波数帯をランダムなサイズ分マスク\n    def freq_mask(self, spec, F, num_masks):\n        test = spec.copy()\n        num_mel_channels = test.shape[0]\n        for i in range(0, num_masks):        \n            freq = random.randrange(0, F)\n            zero = random.randrange(0, num_mel_channels - freq)\n            # avoids randrange error if values are equal and range is empty\n            if (zero == zero + freq): continue\n            mask_end = random.randrange(zero, zero + freq) \n            test[zero:mask_end] = test.mean()\n        return test\n    \n    #ランダムは時間帯をランダムなサイズ分マスク\n    def time_mask(self, spec, time, num_masks):\n        test = spec.copy()\n        length = test.shape[1]\n        for i in range(0, num_masks):\n            t = random.randrange(0, time)\n            zero = random.randrange(0, length - t)\n            if (zero == zero + t): continue\n            mask_end = random.randrange(zero, zero + t)\n            test[:,zero:mask_end] = test.mean()\n        return test","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### SpecAugment用のtransformerを定義","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"transform = SpecAugmentTransform(do_time_warp=False, \n                                 do_freq_mask=True, \n                                 do_time_mask=True,\n                                 num_freq_mask=3,\n                                 num_time_mask=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PERIOD = 5\ndef test_spectrogram(file_path, code, spectrogram_transforms=None):\n    wav_path = file_path\n    ebird_code = code\n    \n    y, sr = sf.read(wav_path)\n\n    \n    len_y = len(y)\n    effective_length = sr * PERIOD\n    if len_y < effective_length:\n        new_y = np.zeros(effective_length, dtype=y.dtype)\n        start = np.random.randint(effective_length - len_y)\n        new_y[start:start + len_y] = y\n        y = new_y.astype(np.float32)\n    elif len_y > effective_length:\n        start = np.random.randint(len_y - effective_length)\n        y = y[start:start + effective_length].astype(np.float32)\n    else:\n        y = y.astype(np.float32)\n\n    melspec = librosa.feature.melspectrogram(y, sr=sr)\n    melspec = librosa.power_to_db(melspec).astype(np.float32)\n\n    if spectrogram_transforms:\n        melspec = spectrogram_transforms(melspec)\n    else:\n        pass\n\n    return melspec, sr","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### SpecAugmentなし","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"melspec, sr = test_spectrogram(file_path=filename, code=\"aaa\", spectrogram_transforms=None)\nlibrosa.display.specshow(melspec, sr=sr, x_axis='time', y_axis='mel')\nplt.colorbar()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## SpecAugmentあり","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"auged_melspec, sr = test_spectrogram(file_path=filename, code=\"aaa\", spectrogram_transforms=transform)\nlibrosa.display.specshow(auged_melspec, sr=sr, x_axis='time', y_axis='mel')\nplt.colorbar()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## MixUp","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## 手法の超ざっくり説明\n * ランダムに抽出した２つのデータをミックスして１つにする\n * ラベルデータもミックスして１つにする\n\n\n### ハイパーパラメータαについて\n * 0に近いほど、λは0か1に数値となる（Beta分布が鍋型）。\n * 1に近いほど、λは0～1の間のランダムな数値となる（Beta分布が一様分布）。\n * 1より大きいほど、λは0.5に近い数値となる（Beta分布が釣り鐘型）。","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport librosa\nimport librosa.display\nfrom IPython.display import Audio, IFrame, display # jupyterで再生につかう\nimport soundfile as sf\nimport random","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PERIOD = 5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BIRD_CODE = {\n    'aldfly': 0, 'ameavo': 1, 'amebit': 2, 'amecro': 3, 'amegfi': 4,\n    'amekes': 5, 'amepip': 6, 'amered': 7, 'amerob': 8, 'amewig': 9,\n    'amewoo': 10, 'amtspa': 11, 'annhum': 12, 'astfly': 13, 'baisan': 14,\n    'baleag': 15, 'balori': 16, 'banswa': 17, 'barswa': 18, 'bawwar': 19,\n    'belkin1': 20, 'belspa2': 21, 'bewwre': 22, 'bkbcuc': 23, 'bkbmag1': 24,\n    'bkbwar': 25, 'bkcchi': 26, 'bkchum': 27, 'bkhgro': 28, 'bkpwar': 29,\n    'bktspa': 30, 'blkpho': 31, 'blugrb1': 32, 'blujay': 33, 'bnhcow': 34,\n    'boboli': 35, 'bongul': 36, 'brdowl': 37, 'brebla': 38, 'brespa': 39,\n    'brncre': 40, 'brnthr': 41, 'brthum': 42, 'brwhaw': 43, 'btbwar': 44,\n    'btnwar': 45, 'btywar': 46, 'buffle': 47, 'buggna': 48, 'buhvir': 49,\n    'bulori': 50, 'bushti': 51, 'buwtea': 52, 'buwwar': 53, 'cacwre': 54,\n    'calgul': 55, 'calqua': 56, 'camwar': 57, 'cangoo': 58, 'canwar': 59,\n    'canwre': 60, 'carwre': 61, 'casfin': 62, 'caster1': 63, 'casvir': 64,\n    'cedwax': 65, 'chispa': 66, 'chiswi': 67, 'chswar': 68, 'chukar': 69,\n    'clanut': 70, 'cliswa': 71, 'comgol': 72, 'comgra': 73, 'comloo': 74,\n    'commer': 75, 'comnig': 76, 'comrav': 77, 'comred': 78, 'comter': 79,\n    'comyel': 80, 'coohaw': 81, 'coshum': 82, 'cowscj1': 83, 'daejun': 84,\n    'doccor': 85, 'dowwoo': 86, 'dusfly': 87, 'eargre': 88, 'easblu': 89,\n    'easkin': 90, 'easmea': 91, 'easpho': 92, 'eastow': 93, 'eawpew': 94,\n    'eucdov': 95, 'eursta': 96, 'evegro': 97, 'fiespa': 98, 'fiscro': 99,\n    'foxspa': 100, 'gadwal': 101, 'gcrfin': 102, 'gnttow': 103, 'gnwtea': 104,\n    'gockin': 105, 'gocspa': 106, 'goleag': 107, 'grbher3': 108, 'grcfly': 109,\n    'greegr': 110, 'greroa': 111, 'greyel': 112, 'grhowl': 113, 'grnher': 114,\n    'grtgra': 115, 'grycat': 116, 'gryfly': 117, 'haiwoo': 118, 'hamfly': 119,\n    'hergul': 120, 'herthr': 121, 'hoomer': 122, 'hoowar': 123, 'horgre': 124,\n    'horlar': 125, 'houfin': 126, 'houspa': 127, 'houwre': 128, 'indbun': 129,\n    'juntit1': 130, 'killde': 131, 'labwoo': 132, 'larspa': 133, 'lazbun': 134,\n    'leabit': 135, 'leafly': 136, 'leasan': 137, 'lecthr': 138, 'lesgol': 139,\n    'lesnig': 140, 'lesyel': 141, 'lewwoo': 142, 'linspa': 143, 'lobcur': 144,\n    'lobdow': 145, 'logshr': 146, 'lotduc': 147, 'louwat': 148, 'macwar': 149,\n    'magwar': 150, 'mallar3': 151, 'marwre': 152, 'merlin': 153, 'moublu': 154,\n    'mouchi': 155, 'moudov': 156, 'norcar': 157, 'norfli': 158, 'norhar2': 159,\n    'normoc': 160, 'norpar': 161, 'norpin': 162, 'norsho': 163, 'norwat': 164,\n    'nrwswa': 165, 'nutwoo': 166, 'olsfly': 167, 'orcwar': 168, 'osprey': 169,\n    'ovenbi1': 170, 'palwar': 171, 'pasfly': 172, 'pecsan': 173, 'perfal': 174,\n    'phaino': 175, 'pibgre': 176, 'pilwoo': 177, 'pingro': 178, 'pinjay': 179,\n    'pinsis': 180, 'pinwar': 181, 'plsvir': 182, 'prawar': 183, 'purfin': 184,\n    'pygnut': 185, 'rebmer': 186, 'rebnut': 187, 'rebsap': 188, 'rebwoo': 189,\n    'redcro': 190, 'redhea': 191, 'reevir1': 192, 'renpha': 193, 'reshaw': 194,\n    'rethaw': 195, 'rewbla': 196, 'ribgul': 197, 'rinduc': 198, 'robgro': 199,\n    'rocpig': 200, 'rocwre': 201, 'rthhum': 202, 'ruckin': 203, 'rudduc': 204,\n    'rufgro': 205, 'rufhum': 206, 'rusbla': 207, 'sagspa1': 208, 'sagthr': 209,\n    'savspa': 210, 'saypho': 211, 'scatan': 212, 'scoori': 213, 'semplo': 214,\n    'semsan': 215, 'sheowl': 216, 'shshaw': 217, 'snobun': 218, 'snogoo': 219,\n    'solsan': 220, 'sonspa': 221, 'sora': 222, 'sposan': 223, 'spotow': 224,\n    'stejay': 225, 'swahaw': 226, 'swaspa': 227, 'swathr': 228, 'treswa': 229,\n    'truswa': 230, 'tuftit': 231, 'tunswa': 232, 'veery': 233, 'vesspa': 234,\n    'vigswa': 235, 'warvir': 236, 'wesblu': 237, 'wesgre': 238, 'weskin': 239,\n    'wesmea': 240, 'wessan': 241, 'westan': 242, 'wewpew': 243, 'whbnut': 244,\n    'whcspa': 245, 'whfibi': 246, 'whtspa': 247, 'whtswi': 248, 'wilfly': 249,\n    'wilsni1': 250, 'wiltur': 251, 'winwre3': 252, 'wlswar': 253, 'wooduc': 254,\n    'wooscj2': 255, 'woothr': 256, 'y00475': 257, 'yebfly': 258, 'yebsap': 259,\n    'yehbla': 260, 'yelwar': 261, 'yerwar': 262, 'yetvir': 263\n}\n\nINV_BIRD_CODE = {v: k for k, v in BIRD_CODE.items()}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_file_list = [['/kaggle/input/birdsong-resampled-train-audio-00/aldfly/XC134874.wav','aldfly'],\n                    ['/kaggle/input/birdsong-resampled-train-audio-00/aldfly/XC135454.wav', 'aldfly']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MixUpTransform(object):\n    def __init__(self,\n                 file_list,\n                 alpha=0.4):\n        self.file_list = file_list\n        self.alpha = alpha\n        \n        \n    # コールされる関数\n    def __call__(self, idx, melspec, one_hot_label):\n        return self.transform(idx, melspec, one_hot_label)\n\n\n    def transform(self, idx, melspec, one_hot_label):\n        #ペアデータの用意\n        pair_data_idx = self._get_pair_index(idx)\n        pair_wav_path, pair_ebird_code = self.file_list[pair_data_idx]\n        pair_melspec = self._convert_mel_spectrogram(pair_wav_path)\n        pair_one_hot_label = self._convert_one_hot_label(pair_ebird_code)\n        \n        r = np.random.beta(self.alpha, self.alpha, 1)[0]\n        # 画像、ラベルを混ぜる（クリップしないと範囲外になることがある）\n        mixed_melspec = r*melspec + (1 - r)*pair_melspec\n        mixed_label = r*one_hot_label + (1 - r)*pair_one_hot_label\n#         mixed_melspec = np.clip(r*melspec + (1 - r)*pair_melspec, 0, 1)\n#         mixed_label = np.clip(r*one_hot_label + (1 - r)*pair_one_hot_label, 0, 1)\n        \n        return mixed_melspec, mixed_label\n\n    \n    def _convert_mel_spectrogram(self, wav_path):\n        y, sr = sf.read(wav_path)\n\n        len_y = len(y)\n        effective_length = sr * PERIOD\n        if len_y < effective_length:\n            new_y = np.zeros(effective_length, dtype=y.dtype)\n            start = np.random.randint(effective_length - len_y)\n            new_y[start:start + len_y] = y\n            y = new_y.astype(np.float32)\n        elif len_y > effective_length:\n            start = np.random.randint(len_y - effective_length)\n            y = y[start:start + effective_length].astype(np.float32)\n        else:\n            y = y.astype(np.float32)\n\n        melspec = librosa.feature.melspectrogram(y, sr=sr)\n        melspec = librosa.power_to_db(melspec).astype(np.float32)\n\n        return melspec\n    \n    \n    def _convert_one_hot_label(self, ebird_code):\n        labels = np.zeros(len(BIRD_CODE), dtype=\"f\")\n        labels[BIRD_CODE[ebird_code]] = 1\n\n        return labels\n    \n    \n    def _get_pair_index(self, idx):    \n        r = list(range(0, idx)) + list(range(idx+1, len(self.file_list)))\n        \n        return random.choice(r)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transform = MixUpTransform(sample_file_list, alpha=0.4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PERIOD = 5\ndef test_spectrogram(file_list, idx, mixup_transforms=None):\n    wav_path, ebird_code = file_list[idx]\n    y, sr = sf.read(wav_path)\n\n    len_y = len(y)\n    effective_length = sr * PERIOD\n    if len_y < effective_length:\n        new_y = np.zeros(effective_length, dtype=y.dtype)\n        start = np.random.randint(effective_length - len_y)\n        new_y[start:start + len_y] = y\n        y = new_y.astype(np.float32)\n    elif len_y > effective_length:\n        start = np.random.randint(len_y - effective_length)\n        y = y[start:start + effective_length].astype(np.float32)\n    else:\n        y = y.astype(np.float32)\n\n    melspec = librosa.feature.melspectrogram(y, sr=sr)\n    melspec = librosa.power_to_db(melspec).astype(np.float32)\n\n    label = np.zeros(len(BIRD_CODE), dtype=\"f\")\n    label[BIRD_CODE[ebird_code]] = 1\n    \n    if mixup_transforms:\n        melspec, label = mixup_transforms(idx, melspec, label)\n    else:\n        pass\n\n\n    return melspec, label, sr","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## mixupなし","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"melspec, label, sr = test_spectrogram(file_list=sample_file_list, idx=0, mixup_transforms=None)\nlibrosa.display.specshow(melspec, sr=sr, x_axis='time', y_axis='mel')\nplt.colorbar()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"melspec, label, sr = test_spectrogram(file_list=sample_file_list, idx=1, mixup_transforms=None)\nlibrosa.display.specshow(melspec, sr=sr, x_axis='time', y_axis='mel')\nplt.colorbar()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## mixupあり","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"auged_melspec, label, sr = test_spectrogram(file_list=sample_file_list, idx=0, mixup_transforms=transform)\nlibrosa.display.specshow(auged_melspec, sr=sr, x_axis='time', y_axis='mel')\nplt.colorbar()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"https://github.com/ebouteillon/freesound-audio-tagging-2019/blob/master/code/training-cnn-model1.ipynb\nここからαは0.4にした","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}