{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":44224,"databundleVersionId":5188730,"sourceType":"competition"},{"sourceId":70203,"databundleVersionId":8068726,"sourceType":"competition"}],"dockerImageVersionId":30162,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Converting audio to 🧮 spectogram\n\nSee: **[Is bird presented all the time in training recordings?](https://www.kaggle.com/c/birdclef-2022/discussion/308861)**\n\nSo this kernel converts audio dataset to image dataset as in many cases a spectrogram is good representation of audio recording\n\n**NOTE: as we create the the image-dataset in this kernel home, it can be easily attached to your future training kernel...**\n\nSee: [Easy Kaggle Offline Submission With Chaining Kernels](https://towardsdatascience.com/easy-kaggle-offline-submission-with-chaining-kernels-30bba5ea5c4d)","metadata":{}},{"cell_type":"code","source":"!pip install -q noisereduce\n\nPATH_DATASET = \"/kaggle/input/birdclef-2024\"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-04-08T21:27:59.535424Z","iopub.execute_input":"2024-04-08T21:27:59.535785Z","iopub.status.idle":"2024-04-08T21:28:11.063449Z","shell.execute_reply.started":"2024-04-08T21:27:59.535688Z","shell.execute_reply":"2024-04-08T21:28:11.062223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip wheel -q https://github.com/Borda/kaggle_image-classify/archive/refs/heads/main.zip --wheel-dir frozen_packages\n# !pip wheel -q https://github.com/PyTorchLightning/lightning-flash/archive/refs/heads/feature/soft_targets.zip --wheel-dir frozen_packages\n!rm frozen_packages/torch*\n!ls -l frozen_packages | grep -e kaggle -e flash","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-04-08T21:28:11.065580Z","iopub.execute_input":"2024-04-08T21:28:11.065882Z","iopub.status.idle":"2024-04-08T21:30:04.244687Z","shell.execute_reply.started":"2024-04-08T21:28:11.065838Z","shell.execute_reply":"2024-04-08T21:30:04.243138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport pandas as pd\n\npath_csv = os.path.join(PATH_DATASET, \"train_metadata.csv\")\ntrain_meta = pd.read_csv(path_csv).sample(frac=1)\ndisplay(train_meta.head())","metadata":{"execution":{"iopub.status.busy":"2024-04-08T21:30:04.247441Z","iopub.execute_input":"2024-04-08T21:30:04.247794Z","iopub.status.idle":"2024-04-08T21:30:04.510157Z","shell.execute_reply.started":"2024-04-08T21:30:04.247754Z","shell.execute_reply":"2024-04-08T21:30:04.509004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Explore 🔍 audio lengths\n\nLet us check lenghth of particular recodings...","metadata":{}},{"cell_type":"code","source":"import torchaudio\nfrom tqdm.auto import tqdm\nfrom joblib import Parallel, delayed\n\ndef get_length(fn):\n    fp = os.path.join(PATH_DATASET, \"train_audio\", fn)\n    waveform, sample_rate = torchaudio.load(fp)\n    return waveform.size()[-1]\n\nsizes = Parallel(n_jobs=os.cpu_count())(delayed(get_length)(fn) for fn in tqdm(train_meta[\"filename\"]))","metadata":{"execution":{"iopub.status.busy":"2024-04-08T21:30:04.513596Z","iopub.execute_input":"2024-04-08T21:30:04.514333Z","iopub.status.idle":"2024-04-08T21:40:16.699189Z","shell.execute_reply.started":"2024-04-08T21:30:04.514280Z","shell.execute_reply":"2024-04-08T21:40:16.698069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.hist(sizes, bins=150)\nplt.gca().set_xscale('log')\nplt.gca().set_yscale('log')\nplt.grid()","metadata":{"execution":{"iopub.status.busy":"2024-04-08T21:40:16.701604Z","iopub.execute_input":"2024-04-08T21:40:16.703141Z","iopub.status.idle":"2024-04-08T21:40:18.483510Z","shell.execute_reply.started":"2024-04-08T21:40:16.703060Z","shell.execute_reply":"2024-04-08T21:40:18.482782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Convert 🗃️ audio to set of spectrograms","metadata":{}},{"cell_type":"code","source":"import torch\nimport torchaudio\nimport noisereduce\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport librosa\nfrom math import ceil\nfrom pprint import pprint\nfrom torch import Tensor\nfrom torch.utils.data import DataLoader\n\nSPECTROGRAM_PARAMS = dict(\n    sample_rate=32_000,\n    hop_length=640,\n    n_fft=800,\n    n_mels=128,\n    fmin=20,\n    fmax=16_000,\n    win_length=512\n)\nPCEN_PARAS = dict(\n    time_constant=0.06,\n    eps=1e-6,\n    gain=0.8,\n    power=0.25,\n    bias=10,\n)\n\n\n@torch.no_grad()\ndef create_spectrogram(\n    fname: str,\n    reduce_noise: bool = False,\n    frame_size: int = 5,\n    frame_step: int = 2,\n    spec_params: dict = SPECTROGRAM_PARAMS,\n) -> list:\n    waveform, sample_rate = librosa.core.load(fname, sr=spec_params[\"sample_rate\"], mono=True)\n    if reduce_noise:\n        waveform = noisereduce.reduce_noise(\n            y=waveform,\n            sr=sample_rate,\n            time_constant_s=float(frame_size),\n            time_mask_smooth_ms=250,\n            n_fft=spec_params[\"n_fft\"],\n            use_tqdm=False,\n            n_jobs=2,\n        )\n\n    step = int(frame_step * sample_rate)\n    size = int(frame_size * sample_rate)\n    count = ceil((len(waveform) - size) / float(step))\n    frames = []\n    for i in range(max(1, count)):\n        begin = i * step\n        frame = waveform[begin:begin + size]\n        if len(frame) < size:\n            if i == 0:\n                rep = round(float(size) / len(frame))\n                frame = frame.repeat(int(rep))\n            elif len(frame) < (size * 0.33):\n                continue\n            else:\n                frame = waveform[-size:]\n        frames.append(frame)\n\n    spectrograms = []\n    for frm in frames:\n        sg = librosa.feature.melspectrogram(\n            y=frm,\n            sr=sample_rate,\n            n_fft=spec_params[\"n_fft\"],\n            win_length=spec_params[\"win_length\"],\n            hop_length=spec_params[\"hop_length\"],\n            n_mels=spec_params[\"n_mels\"],\n            fmin=spec_params[\"fmin\"],\n            fmax=spec_params[\"fmax\"],\n            power=1,\n        )\n#         sg = librosa.pcen(sg, sr=sample_rate, hop_length=spec_params[\"hop_length\"], **PCEN_PARAS)\n        sg = librosa.amplitude_to_db(sg, ref=np.max)\n        spectrograms.append(np.nan_to_num(sg))\n    return spectrograms","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-04-08T21:40:18.485041Z","iopub.execute_input":"2024-04-08T21:40:18.485429Z","iopub.status.idle":"2024-04-08T21:40:20.764666Z","shell.execute_reply.started":"2024-04-08T21:40:18.485398Z","shell.execute_reply":"2024-04-08T21:40:20.763677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_audio = os.path.join(PATH_DATASET, \"train_audio\", \"asbfly/XC134896.ogg\")\n# path_audio = os.path.join(PATH_DATASET, \"train_audio\", \"amesun2/XC113074.ogg\")\n# path_audio = os.path.join(PATH_DATASET, \"train_audio\", \"elepai/XC27344.ogg\")\n# path_audio = os.path.join(PATH_DATASET, \"train_audio\", \"hawgoo/XC210217.ogg\")\nprint(path_audio)\nsgs = create_spectrogram(path_audio, reduce_noise=False)\n\n\nfig, axarr = plt.subplots(nrows=len(sgs), figsize=(8, 3 * len(sgs)))\nfor i, sg in enumerate(sgs):\n    print(np.min(sg), np.max(sg))\n    im = axarr[i].imshow(sg)  # librosa\n    plt.colorbar(im, ax=axarr[i])","metadata":{"execution":{"iopub.status.busy":"2024-04-08T21:40:20.765943Z","iopub.execute_input":"2024-04-08T21:40:20.766214Z","iopub.status.idle":"2024-04-08T21:40:24.268022Z","shell.execute_reply.started":"2024-04-08T21:40:20.766181Z","shell.execute_reply":"2024-04-08T21:40:24.266814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Export only frames from the recoding beginning and ending**","metadata":{}},{"cell_type":"code","source":"from PIL import Image\nfrom tqdm.auto import tqdm\nfrom functools import partial\nfrom joblib import Parallel, delayed\n\n\ndef convert_and_export(\n    fn: str, path_in: str, path_out: str,\n    reduce_noise: bool = False,\n    frame_size: int = 5, frame_step: int = 2,\n    img_extension=\".png\",\n) -> None:\n    path_audio = os.path.join(path_in, fn)\n    try:\n        sgs = create_spectrogram(\n            path_audio,\n            reduce_noise=reduce_noise,\n            frame_size=frame_size,\n            frame_step=frame_step,\n        )\n    except Exception as ex:\n        print(f\"Failed conversion for audio: {path_audio}\")\n        return\n    if not sgs:\n        print(f\"Too short audio for: {path_audio}\")\n        return\n    # see: https://www.kaggle.com/c/birdclef-2022/discussion/308861\n    # this is adjustment for window 5s and step 2s\n    nb = ceil((10 - frame_size) / frame_step) + 1\n    if len(sgs) > 2 * nb:\n        sgs = sgs[:nb] + sgs[-nb:]\n    path_npz = os.path.join(path_out, fn + '.npz')\n    os.makedirs(os.path.dirname(path_npz), exist_ok=True)\n    # np.savez_compressed(path_npz, np.array(sgs, dtype=np.float16))\n    for i, sg in enumerate(sgs):\n        path_img = os.path.join(path_out, fn + f\".{i:03}\" + img_extension)\n        try:\n            # plt.imsave(path_img, sg, vmin=-70, vmax=20)\n            sg = (sg + 80) / 80.0\n            sg = np.clip(sg, a_min=0, a_max=1) * 255\n            img = Image.fromarray(sg.astype(np.uint8))\n            img.resize((256,256)).save(path_img)\n        except Exception as ex:\n            print(f\"Failed exporting for image: {path_img}\")\n            continue","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-04-08T21:40:24.269949Z","iopub.execute_input":"2024-04-08T21:40:24.270979Z","iopub.status.idle":"2024-04-08T21:40:24.288127Z","shell.execute_reply.started":"2024-04-08T21:40:24.270925Z","shell.execute_reply":"2024-04-08T21:40:24.286563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Perform conversion 🏃 full dataset\n\nRunning conversion in pararlle and otionaly you can be using GPU","metadata":{}},{"cell_type":"code","source":"_convert_and_export = partial(\n    convert_and_export,\n    path_in=os.path.join(PATH_DATASET, \"train_audio\"),\n    path_out=\"train_images\",\n)\n\n_= Parallel(n_jobs=3)(delayed(_convert_and_export)(fn) for fn in tqdm(train_meta[\"filename\"]))\n# _= list(map(_convert_and_export, tqdm(train_meta[\"filename\"])))","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-04-08T21:40:24.289400Z","iopub.execute_input":"2024-04-08T21:40:24.289699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nfrom pprint import pprint\n\nprint(train_meta[\"filename\"][42])\nimgs = glob.glob(os.path.join(\"train_images\", train_meta[\"filename\"][1] + \".*.png\"))\npprint(sorted(imgs))\n\npath_img = imgs[0]\nprint(path_img)\nimg = plt.imread(path_img)\nprint(img.shape)\nplt.imshow(img)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Compute 🎟️ color normalizations","metadata":{}},{"cell_type":"code","source":"def _color_means(img_path):\n    img = plt.imread(img_path)\n    if np.max(img) > 1.5:\n        img = img / 255.0\n    clr_mean = np.mean(img) if img.ndim == 2 else {i: np.mean(img[..., i]) for i in range(3)}\n    clr_std = np.std(img) if img.ndim == 2 else {i: np.std(img[..., i]) for i in range(3)}\n    return clr_mean, clr_std\n\nimages = glob.glob(os.path.join(\"train_images\", \"*\", \"*.png\"))\nclr_mean_std = Parallel(n_jobs=os.cpu_count())(delayed(_color_means)(fn) for fn in tqdm(images))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_color_mean = pd.DataFrame([c[0] for c in clr_mean_std]).describe()\ndisplay(img_color_mean.T)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_color_std = pd.DataFrame([c[1] for c in clr_mean_std]).describe()\ndisplay(img_color_std.T)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_color_mean = list(img_color_mean.T[\"mean\"])\nimg_color_std = list(img_color_std.T[\"mean\"])\nprint(img_color_mean, img_color_std)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}