{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Here is an example of how to match the melspectrograms of librosa and torchaudio. <br>\nI do not know which parameter is more accurate, but it reduces the risk of using a different looking image in the training than the one analyzed by librosa.","metadata":{}},{"cell_type":"code","source":"from pathlib import Path\nimport numpy as np\nimport pandas as pd\nimport typing as tp\nimport yaml\nimport random\nimport os\nimport sys\nimport soundfile as sf\nimport librosa\nimport librosa.display\nimport cv2\nimport matplotlib.pyplot as plt\nimport time\nimport pickle\nimport glob\nfrom tqdm import tqdm\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.data as data\n\nimport torchaudio","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-03T08:37:37.063377Z","iopub.execute_input":"2022-06-03T08:37:37.064066Z","iopub.status.idle":"2022-06-03T08:37:42.508954Z","shell.execute_reply.started":"2022-06-03T08:37:37.063977Z","shell.execute_reply":"2022-06-03T08:37:42.508211Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    n_mels = 128\n    fmin = 20\n    fmax = 16000\n    n_fft = 2048\n    hop_length = 512","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_DIR = Path(\"../input/birdclef-2022\")\ntrain_wav_dir = DATA_DIR / 'train_audio'\ntrain_csv_path = DATA_DIR / 'train_metadata.csv'","metadata":{"execution":{"iopub.status.busy":"2022-06-03T08:44:17.492746Z","iopub.execute_input":"2022-06-03T08:44:17.493109Z","iopub.status.idle":"2022-06-03T08:44:17.498697Z","shell.execute_reply.started":"2022-06-03T08:44:17.493081Z","shell.execute_reply":"2022-06-03T08:44:17.497844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(train_csv_path)","metadata":{"execution":{"iopub.status.busy":"2022-06-03T08:44:20.639508Z","iopub.execute_input":"2022-06-03T08:44:20.639943Z","iopub.status.idle":"2022-06-03T08:44:20.695434Z","shell.execute_reply.started":"2022-06-03T08:44:20.639905Z","shell.execute_reply":"2022-06-03T08:44:20.694809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# idx = np.random.randint(len(train))\nidx = 100\nraw = train.iloc[idx]\n\nwav_name = raw[\"filename\"]\nebird_code = raw[\"primary_label\"]\ny, sr = sf.read(train_wav_dir / wav_name, always_2d=True)\ny = y[:, 0]","metadata":{"execution":{"iopub.status.busy":"2022-06-03T10:03:30.539251Z","iopub.execute_input":"2022-06-03T10:03:30.539669Z","iopub.status.idle":"2022-06-03T10:03:30.593529Z","shell.execute_reply.started":"2022-06-03T10:03:30.539639Z","shell.execute_reply":"2022-06-03T10:03:30.592502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compare_mels(mels, titles):\n    fig, ax = plt.subplots(len(mels), 1, figsize=(12, len(mels)*4))\n    for i in range(len(mels)):\n        librosa.display.specshow(mels[i], y_axis='mel', fmax=CFG.fmax, x_axis='time', sr=sr, ax=ax[i])\n        ax[i].set_title(titles[i])\n    plt.tight_layout()\n    plt.show()\n    \n    diff = np.max(np.abs(mels[1] - mels[0]))\n    print(f'max diff = {diff}')","metadata":{"execution":{"iopub.status.busy":"2022-06-03T10:03:30.810019Z","iopub.execute_input":"2022-06-03T10:03:30.810504Z","iopub.status.idle":"2022-06-03T10:03:30.820264Z","shell.execute_reply.started":"2022-06-03T10:03:30.810472Z","shell.execute_reply":"2022-06-03T10:03:30.818530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### common case","metadata":{}},{"cell_type":"code","source":"melspec = librosa.feature.melspectrogram(y=y, sr=sr, n_fft=CFG.n_fft, hop_length=CFG.hop_length, n_mels=CFG.n_mels, fmin=CFG.fmin, fmax=CFG.fmax)\nmelspec = librosa.power_to_db(melspec, ref=np.max).astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2022-06-03T10:03:31.167777Z","iopub.execute_input":"2022-06-03T10:03:31.168198Z","iopub.status.idle":"2022-06-03T10:03:31.293887Z","shell.execute_reply.started":"2022-06-03T10:03:31.168167Z","shell.execute_reply":"2022-06-03T10:03:31.293080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"logmelspec_extractor = nn.Sequential(torchaudio.transforms.MelSpectrogram(sample_rate=sr, \n                                                                     n_fft=CFG.n_fft, \n                                                                     win_length=CFG.n_fft, \n                                                                     hop_length=CFG.hop_length, \n                                                                     f_min=CFG.fmin, \n                                                                     f_max=CFG.fmax, \n                                                                     n_mels=CFG.n_mels),\n                                                 torchaudio.transforms.AmplitudeToDB(),\n                                                 )\ntorch_melspec = logmelspec_extractor(torch.tensor(y.reshape(1, -1)).float()).numpy()[0]","metadata":{"execution":{"iopub.status.busy":"2022-06-03T10:03:31.483281Z","iopub.execute_input":"2022-06-03T10:03:31.483715Z","iopub.status.idle":"2022-06-03T10:03:31.523006Z","shell.execute_reply.started":"2022-06-03T10:03:31.483685Z","shell.execute_reply":"2022-06-03T10:03:31.521912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mels = [melspec, torch_melspec]\ntitles = ['librosa', 'torch_audio']","metadata":{"execution":{"iopub.status.busy":"2022-06-03T10:03:31.696473Z","iopub.execute_input":"2022-06-03T10:03:31.696862Z","iopub.status.idle":"2022-06-03T10:03:31.701824Z","shell.execute_reply.started":"2022-06-03T10:03:31.696832Z","shell.execute_reply":"2022-06-03T10:03:31.700351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"compare_mels(mels, titles)","metadata":{"execution":{"iopub.status.busy":"2022-06-03T10:03:31.904892Z","iopub.execute_input":"2022-06-03T10:03:31.905548Z","iopub.status.idle":"2022-06-03T10:03:32.544347Z","shell.execute_reply.started":"2022-06-03T10:03:31.905515Z","shell.execute_reply":"2022-06-03T10:03:32.543795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## after correction\n\nThe fixes are as follows.\n\n1. calc melspectrogram\nCorrect the parameters to those listed [here](https://github.com/pytorch/audio/issues/1058).\n\n2. take the log\n「ref」 is not specified because this cannot be set in torchaudio. Also, 「top_db」 should be set to None.","metadata":{}},{"cell_type":"code","source":"melspec = librosa.feature.melspectrogram(y=y, \n                                         sr=sr, \n                                         n_fft=CFG.n_fft, \n                                         hop_length=CFG.hop_length, \n                                         n_mels=CFG.n_mels, \n                                         fmin=CFG.fmin, \n                                         fmax=CFG.fmax,\n                                         center=True,\n                                         pad_mode=\"reflect\",\n#                                          power=1.0,\n                                         norm='slaney',\n                                         htk=True,)\nmelspec = librosa.power_to_db(melspec, top_db=None).astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2022-06-03T10:03:33.185052Z","iopub.execute_input":"2022-06-03T10:03:33.185641Z","iopub.status.idle":"2022-06-03T10:03:33.287943Z","shell.execute_reply.started":"2022-06-03T10:03:33.185602Z","shell.execute_reply":"2022-06-03T10:03:33.286993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"logmelspec_extractor = nn.Sequential(torchaudio.transforms.MelSpectrogram(sample_rate=sr, \n                                                                     n_fft=CFG.n_fft, \n                                                                     win_length=CFG.n_fft, \n                                                                     hop_length=CFG.hop_length, \n                                                                     f_min=CFG.fmin, \n                                                                     f_max=CFG.fmax, \n                                                                     n_mels=CFG.n_mels,\n                                                                     center=True,\n                                                                     pad_mode=\"reflect\",\n                                                                     norm=\"slaney\",\n                                                                     onesided=True,\n                                                                     mel_scale=\"htk\"),\n                                                 torchaudio.transforms.AmplitudeToDB(),\n                                                 )\ntorch_melspec = logmelspec_extractor(torch.tensor(y.reshape(1, -1)).float()).numpy()[0]","metadata":{"execution":{"iopub.status.busy":"2022-06-03T10:03:33.840085Z","iopub.execute_input":"2022-06-03T10:03:33.840435Z","iopub.status.idle":"2022-06-03T10:03:33.879033Z","shell.execute_reply.started":"2022-06-03T10:03:33.840402Z","shell.execute_reply":"2022-06-03T10:03:33.878058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mels = [melspec, torch_melspec]\ntitles = ['librosa', 'torch_audio']","metadata":{"execution":{"iopub.status.busy":"2022-06-03T10:03:34.699811Z","iopub.execute_input":"2022-06-03T10:03:34.700427Z","iopub.status.idle":"2022-06-03T10:03:34.705761Z","shell.execute_reply.started":"2022-06-03T10:03:34.700391Z","shell.execute_reply":"2022-06-03T10:03:34.704714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"compare_mels(mels, titles)","metadata":{"execution":{"iopub.status.busy":"2022-06-03T10:03:34.882764Z","iopub.execute_input":"2022-06-03T10:03:34.883805Z","iopub.status.idle":"2022-06-03T10:03:35.714835Z","shell.execute_reply.started":"2022-06-03T10:03:34.883756Z","shell.execute_reply":"2022-06-03T10:03:35.713264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"If you know of a way to make the difference even smaller, please let me know.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}