{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook is an adaptation of a wonderful last-year BirdCLEF competition notebook by kkiller:\nhttps://www.kaggle.com/kneroma/birdclef-mels-computer-public/\n\n**Please upvote it as well!**\n\n\nI just fixed a few things and reformatted it a little.\n\nThis notebook allows you to compute melspectrograms to speed up training of your models.","metadata":{}},{"cell_type":"markdown","source":"## Import packages","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport json\nimport soundfile as sf\nimport librosa as lb\nimport librosa.display as lbd\nfrom pathlib import Path\nfrom soundfile import SoundFile\nfrom joblib import Parallel, delayed\nfrom sklearn.model_selection import StratifiedKFold\nfrom tqdm.notebook import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-15T20:34:08.812093Z","iopub.execute_input":"2022-02-15T20:34:08.812899Z","iopub.status.idle":"2022-02-15T20:34:08.819588Z","shell.execute_reply.started":"2022-02-15T20:34:08.812851Z","shell.execute_reply":"2022-02-15T20:34:08.818617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Config","metadata":{}},{"cell_type":"code","source":"DATA_DIR = Path(\"../input/birdclef-2022\")\nSR = 32_000   # sample rate\nDURATION = 7  # max duration of sound\nSEED = 42\n\nTRAIN_AUDIO_DIR = DATA_DIR/\"train_audio\"\nTRAIN_AUDIO_IMAGES_SAVE_DIR = Path(\"audio_images\") # save the melspectrograms to here\nTRAIN_AUDIO_IMAGES_SAVE_DIR.mkdir(exist_ok=True, parents=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T20:34:08.821249Z","iopub.execute_input":"2022-02-15T20:34:08.821524Z","iopub.status.idle":"2022-02-15T20:34:08.837034Z","shell.execute_reply.started":"2022-02-15T20:34:08.821483Z","shell.execute_reply":"2022-02-15T20:34:08.835859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prepare training dataset","metadata":{}},{"cell_type":"code","source":"def get_audio_info(filepath):\n    \"\"\"Get some properties from  an audio file\"\"\"\n    with SoundFile(filepath) as f:\n        sr = f.samplerate\n        frames = f.frames\n        duration = float(frames)/sr\n    return {\"frames\": frames, \"sr\": sr, \"duration\": duration}","metadata":{"execution":{"iopub.status.busy":"2022-02-15T20:34:08.838475Z","iopub.execute_input":"2022-02-15T20:34:08.83907Z","iopub.status.idle":"2022-02-15T20:34:08.851602Z","shell.execute_reply.started":"2022-02-15T20:34:08.839025Z","shell.execute_reply":"2022-02-15T20:34:08.850908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_df(n_splits=5, seed=SEED, nrows=None):\n    df = pd.read_csv(DATA_DIR/\"train_metadata.csv\", nrows=nrows)\n    LABEL_IDS = {label: label_id for label_id,label in enumerate(sorted(df[\"primary_label\"].unique()))}\n    df[\"label_id\"] = df[\"primary_label\"].map(LABEL_IDS)\n    df[\"filepath\"] = [str(TRAIN_AUDIO_DIR/filename) for filename in df.filename]\n    pool = Parallel(-1)\n    mapper = delayed(get_audio_info)\n    tasks = [mapper(filepath) for filepath in df.filepath]\n\n    df = pd.concat([df, pd.DataFrame(pool(tqdm(tasks)))], axis=1, sort=False)\n    \n    skf = StratifiedKFold(n_splits=n_splits, random_state=seed, shuffle=True)\n    splits = skf.split(np.arange(len(df)), y=df.label_id.values)\n    df[\"fold\"] = -1\n    for fold, (train_set, val_set) in enumerate(splits):\n        df.loc[df.index[val_set], \"fold\"] = fold\n\n    return LABEL_IDS, df","metadata":{"execution":{"iopub.status.busy":"2022-02-15T20:34:08.853195Z","iopub.execute_input":"2022-02-15T20:34:08.854057Z","iopub.status.idle":"2022-02-15T20:34:08.868342Z","shell.execute_reply.started":"2022-02-15T20:34:08.854018Z","shell.execute_reply":"2022-02-15T20:34:08.867732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Process train dataframe and split it to folds","metadata":{}},{"cell_type":"code","source":"LABEL_IDS, df = make_df(nrows=None)\n\ndf.to_csv(\"rich_train_metadata.csv\", index=True)\nwith open(\"LABEL_IDS.json\", \"w\") as f:\n    json.dump(LABEL_IDS, f)\n\nprint(df.shape)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-15T20:34:08.870926Z","iopub.execute_input":"2022-02-15T20:34:08.871189Z","iopub.status.idle":"2022-02-15T20:34:48.786335Z","shell.execute_reply.started":"2022-02-15T20:34:08.871157Z","shell.execute_reply":"2022-02-15T20:34:48.784725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's see how many elements each fold has.","metadata":{}},{"cell_type":"code","source":"df[\"fold\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-02-15T20:34:48.788991Z","iopub.execute_input":"2022-02-15T20:34:48.789887Z","iopub.status.idle":"2022-02-15T20:34:48.801275Z","shell.execute_reply.started":"2022-02-15T20:34:48.789822Z","shell.execute_reply":"2022-02-15T20:34:48.80014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Helper functions","metadata":{}},{"cell_type":"code","source":"class MelSpecComputer:\n    def __init__(self, sr, n_mels, fmin, fmax, **kwargs):\n        self.sr = sr\n        self.n_mels = n_mels\n        self.fmin = fmin\n        self.fmax = fmax\n        kwargs[\"n_fft\"] = kwargs.get(\"n_fft\", self.sr//10)\n        kwargs[\"hop_length\"] = kwargs.get(\"hop_length\", self.sr//(10*4))\n        self.kwargs = kwargs\n\n    def __call__(self, y):\n\n        melspec = lb.feature.melspectrogram(\n            y=y, sr=self.sr, n_mels=self.n_mels, fmin=self.fmin, fmax=self.fmax, **self.kwargs,\n        )\n\n        melspec = lb.power_to_db(melspec).astype(np.float32)\n        return melspec","metadata":{"execution":{"iopub.status.busy":"2022-02-15T20:34:48.803588Z","iopub.execute_input":"2022-02-15T20:34:48.804485Z","iopub.status.idle":"2022-02-15T20:34:48.815752Z","shell.execute_reply.started":"2022-02-15T20:34:48.804406Z","shell.execute_reply":"2022-02-15T20:34:48.814491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mono_to_color(X, eps=1e-6, mean=None, std=None):\n    mean = mean or X.mean()\n    std = std or X.std()\n    X = (X - mean) / (std + eps)\n    \n    _min, _max = X.min(), X.max()\n\n    if (_max - _min) > eps:\n        V = np.clip(X, _min, _max)\n        V = 255 * (V - _min) / (_max - _min)\n        V = V.astype(np.uint8)\n    else:\n        V = np.zeros_like(X, dtype=np.uint8)\n\n    return V\n\ndef crop_or_pad(y, length, is_train=True, start=None):\n    if len(y) < length:\n        y = np.concatenate([y, np.zeros(length - len(y))])\n        \n        n_repeats = length // len(y)\n        epsilon = length % len(y)\n        \n        y = np.concatenate([y]*n_repeats + [y[:epsilon]])\n        \n    elif len(y) > length:\n        if not is_train:\n            start = start or 0\n        else:\n            start = start or np.random.randint(len(y) - length)\n\n        y = y[start:start + length]\n\n    return y","metadata":{"execution":{"iopub.status.busy":"2022-02-15T20:34:48.8177Z","iopub.execute_input":"2022-02-15T20:34:48.818183Z","iopub.status.idle":"2022-02-15T20:34:48.831056Z","shell.execute_reply.started":"2022-02-15T20:34:48.81803Z","shell.execute_reply":"2022-02-15T20:34:48.830065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AudioToImage:\n    def __init__(self, sr=SR, n_mels=128, fmin=0, fmax=None, \n                 duration=DURATION, step=None, res_type=\"kaiser_fast\", \n                 resample=True):\n        self.sr = sr\n        self.n_mels = n_mels\n        self.fmin = fmin\n        self.fmax = fmax or self.sr//2\n\n        self.duration = duration\n        self.audio_length = self.duration*self.sr\n        self.step = step or self.audio_length\n        \n        self.res_type = res_type\n        self.resample = resample\n\n        self.mel_spec_computer = MelSpecComputer(sr=self.sr, n_mels=self.n_mels, fmin=self.fmin,\n                                                 fmax=self.fmax)\n        \n    def audio_to_image(self, audio):\n        melspec = self.mel_spec_computer(audio) \n        image = mono_to_color(melspec)\n#         image = normalize(image, mean=None, std=None)\n        return image\n\n    def __call__(self, row, save=True):\n        max_audio_duration = 10*self.duration\n        init_audio_length = max_audio_duration*row.sr\n        \n        start = 0 if row.duration <  max_audio_duration else np.random.randint(row.frames - init_audio_length)\n    \n        audio, orig_sr = sf.read(row.filepath, start=start, stop=start+init_audio_length, dtype=\"float32\")\n        \n        # convert sound to mono if stereo\n        if len(audio.shape) > 1:\n            audio = lb.to_mono(audio)\n\n        if self.resample and orig_sr != self.sr:\n            audio = lb.resample(audio, orig_sr, self.sr, res_type=self.res_type)\n        \n        audios = [audio[i:i+self.audio_length] for i in range(0, max(1, len(audio) - self.audio_length + 1), self.step)]\n        audios[-1] = crop_or_pad(audios[-1] , length=self.audio_length)\n        images = np.stack([self.audio_to_image(audio) for audio in audios])\n        \n        if save:\n            path = TRAIN_AUDIO_IMAGES_SAVE_DIR/f\"{row.filename}.npy\"\n            path.parent.mkdir(exist_ok=True, parents=True)\n            np.save(str(path), images)\n        else:\n            return row.filename, images","metadata":{"execution":{"iopub.status.busy":"2022-02-15T20:45:35.422242Z","iopub.execute_input":"2022-02-15T20:45:35.423288Z","iopub.status.idle":"2022-02-15T20:45:35.440137Z","shell.execute_reply.started":"2022-02-15T20:45:35.423235Z","shell.execute_reply":"2022-02-15T20:45:35.439269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Convert audio to melspectrograms and save them as images","metadata":{}},{"cell_type":"code","source":"def get_audios_as_images(df):\n    pool = Parallel(-1) \n    converter = AudioToImage(step=int(DURATION*0.666*SR))\n    mapper = delayed(converter)\n    tasks = [mapper(row) for row in df.itertuples(False)]\n    pool(tqdm(tasks))","metadata":{"execution":{"iopub.status.busy":"2022-02-15T20:45:35.44158Z","iopub.execute_input":"2022-02-15T20:45:35.442273Z","iopub.status.idle":"2022-02-15T20:45:35.460918Z","shell.execute_reply.started":"2022-02-15T20:45:35.442237Z","shell.execute_reply":"2022-02-15T20:45:35.45978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_audios_as_images(df)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T20:58:01.137417Z","iopub.execute_input":"2022-02-15T20:58:01.137738Z","iopub.status.idle":"2022-02-15T20:58:05.643513Z","shell.execute_reply.started":"2022-02-15T20:58:01.137706Z","shell.execute_reply":"2022-02-15T20:58:05.642581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row = df.loc[df.duration.idxmax()]\nmels = np.load(str((TRAIN_AUDIO_IMAGES_SAVE_DIR/row.filename).as_posix() + \".npy\"))\nprint(mels.shape)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T20:58:05.645754Z","iopub.execute_input":"2022-02-15T20:58:05.646105Z","iopub.status.idle":"2022-02-15T20:58:05.656791Z","shell.execute_reply.started":"2022-02-15T20:58:05.646057Z","shell.execute_reply":"2022-02-15T20:58:05.65552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Show one melspectrogram","metadata":{}},{"cell_type":"markdown","source":"Let's show one melspectrogram as an example","metadata":{}},{"cell_type":"code","source":"lbd.specshow(mels[0])","metadata":{"execution":{"iopub.status.busy":"2022-02-15T20:58:05.658607Z","iopub.execute_input":"2022-02-15T20:58:05.659041Z","iopub.status.idle":"2022-02-15T20:58:05.837844Z","shell.execute_reply.started":"2022-02-15T20:58:05.658997Z","shell.execute_reply":"2022-02-15T20:58:05.837177Z"},"trusted":true},"execution_count":null,"outputs":[]}]}