{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"I have noticed some training notebooks that take a **random 5 sec** from the clips as training data. \n\nI think that may be a deceptive strategy because **the birds are not singing for the whole clip**. \nFurthemore we should be able to classify when there is no bird singing at all in a sequence so we need training data for that as well. \n\nThat means that being able to identify if there is or not a bird singing in a clip section is primordial to this competition\n\nThis notebook explores strategies of preprocessing that allow to split clips between silence (background noise) and real bird songs parts \n\n\n\nThere is too much data to process it on Kaggle so I took up a Google Cloud Engine to do it [https://github.com/LuisBlanche/kaggle-birdsong-split-silence](Code on github)\nI am now uploading the datasets to kaggle, you can find them here :\n\nhttps://www.kaggle.com/luisblanche/birdcall-singing-0\n\nhttps://www.kaggle.com/luisblanche/birdcall-singing-1\n\nhttps://www.kaggle.com/luisblanche/birdcall-singing-2\n\nhttps://www.kaggle.com/luisblanche/birdcall-singing-3\n\nhttps://www.kaggle.com/luisblanche/birdcall-singing-4\n\nhttps://www.kaggle.com/luisblanche/birdcall-background This one is 5sec samples of background sound for each clip of the training data","metadata":{}},{"cell_type":"markdown","source":"## Import modules and data","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\n\nimport librosa\nimport pandas as pd\nimport pandas_profiling\nimport matplotlib.pyplot as plt\nfrom IPython.display import Audio, IFrame, display\nimport librosa.display\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"../input/birdsong-recognition/train.csv\", parse_dates=['date'])","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Remove silences \n\nStrategy : \n* Select long clips that have more chances to have silence in it while the recorder wait for another birdcall\n* Use librosa to remove silent parts , using  **mean db - std db** as the threshold to consider silence","metadata":{}},{"cell_type":"code","source":"long_clips = train[train['duration'] > 60].index \nlen(long_clips)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path=\"../input/birdsong-resampled-train-audio-00\"\n\n\ndef get_audio_path(row_number):\n    \"\"\"Returns species and path for a row number of train\"\"\"\n    data = train.iloc[row_number]\n    species = data['species']\n    audio_path=os.path.join(path, data['ebird_code'], data['filename'].split('.')[0] + '.wav')\n    return species, audio_path\n    \ndef get_audio(row_number):\n    \"\"\"displays audio for a row number of train\n    \"\"\"\n    species, audio_path = get_audio_path(row_number)\n    print(species)\n    return display(Audio(audio_path))\n\nsound_variables = ['pitch', 'speed', 'number_of_notes', 'type', 'volume', 'length']\ndef describe_audio(row_number):\n    \"\"\"print audio with bird sound metadata \n    \"\"\"\n    audio = get_audio(row_number)\n    metadata = train.iloc[row_number][sound_variables]\n    return audio, metadata\n\ndef plot_wave(row_number):\n    \"\"\"\n    \"\"\"\n    species, audio_path = get_audio_path(row_number)\n    x , sr = librosa.load(audio_path)\n    librosa.display.waveplot(x, sr=sr)\n    plt.gca().set_title(species)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Select an example clip","metadata":{}},{"cell_type":"code","source":"clip = long_clips[1]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_audio(0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"describe_audio(clip)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_wave(clip)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Listening to this clip and looking at the waveplot we understand that a big part of is just \"white noise\" with not bird singing. Let's try and use [librosa.effects.split](https://librosa.org/librosa/0.7.1/generated/librosa.effects.split.html#librosa-effects-split) to remove te silent parts and reduce the amount of information to process","metadata":{}},{"cell_type":"code","source":" def split_sound(row_number):\n    \"\"\"Returns the sound array, sample rate and\n    x_split = intervals where sound is louder than top db\n    \"\"\"\n    species, audio_path = get_audio_path(row_number)\n    x , sr = librosa.load(audio_path)\n    db = librosa.core.amplitude_to_db(x)\n    mean_db = np.abs(db).mean()\n    std_db = db.std()\n    x_split = librosa.effects.split(y=x, top_db = mean_db - std_db)\n    return x, sr, x_split","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def remove_silence(clip):\n    \"\"\"Removes silence from clip\n    \"\"\"\n    sound, sr, intervals = split_sound(clip)\n    silence_removed = []\n    for inter in intervals:\n        silence_removed.extend(sound[inter[0]:inter[1]])\n    silence_removed = np.array(silence_removed)\n    return silence_removed, sr","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"silence_removed, sr = remove_silence(clip)\ndisplay(Audio(silence_removed, rate=sr))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"librosa.display.waveplot(silence_removed, sr=sr)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see that the clip has been reduced from 2min 7s to 51s , and that we still have all the bird sounds from the initial clip, we merely remove parts of the clip where we could hear waves in the background. ","metadata":{}},{"cell_type":"markdown","source":"## Create Silent/Noise clips ","metadata":{}},{"cell_type":"code","source":"x, sr, split = split_sound(clip)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gather_silence(clip):\n    sound, sr, intervals = split_sound(clip)\n    silence = sound[0:intervals[0][0]]\n    for i in range(len(intervals)-1):\n        silence = np.append(silence, sound[intervals[i][1]:intervals[i+1][0]])\n    silence = np.append(silence, sound[intervals[-1][1]:])\n    return silence, sr","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"silence, sr = gather_silence(clip)\ndisplay(Audio(silence, rate=sr))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"librosa.display.waveplot(silence, sr=sr)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Just like that we get the background noise, which could be use to train another model to recognize birds in the background (that could be very useful for prediction in clip 3 for instance) ","metadata":{}},{"cell_type":"markdown","source":"## Create Dataset","metadata":{"trusted":true}},{"cell_type":"code","source":"from pathlib import Path\nROOT = Path.cwd().parent\nINPUT_ROOT = ROOT / \"input\"\nRAW_DATA = INPUT_ROOT / \"birdsong-recognition\"\nTRAIN_AUDIO_DIR = RAW_DATA / \"train_audio\"\nTRAIN_RESAMPLED_AUDIO_DIRS = [\n  INPUT_ROOT / \"birdsong-resampled-train-audio-{:0>2}\".format(i)  for i in range(5)\n]\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_RESAMPLED_AUDIO_DIRS","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_SINGING_DIR = Path(\"processed_data/train_audio_singing\")\nTRAIN_BACKGROUND_DIR = Path(\"processed_data/train_audio_background\")\nTRAIN_SINGING_DIR.mkdir(parents=True, exist_ok=True)\nTRAIN_BACKGROUND_DIR.mkdir(parents=True, exist_ok=True)\nfor ebird_code in train.ebird_code.unique():\n    ebird_dir = TRAIN_SINGING_DIR / ebird_code\n    background_dir = TRAIN_BACKGROUND_DIR / ebird_code\n    ebird_dir.mkdir(exist_ok=True)\n    background_dir.mkdir(exist_ok=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\ndef split_sound(clip):\n    \"\"\"Returns the sound array, sample rate and\n    x_split = intervals where sound is louder than top db\n    \"\"\"\n    db = librosa.core.amplitude_to_db(clip)\n    mean_db = np.abs(db).mean()\n    std_db = db.std()\n    x_split = librosa.effects.split(y=clip, top_db = mean_db - std_db)\n    return x_split\n\ndef take_random_sample(clip, sample_len=5, sample_rate=32000):\n    if len(clip) > sample_len*sample_len:\n        idx = random.randint(0, len(clip) -  sample_len*sample_len)\n        sample = clip[idx:idx + sample_rate * sample_len]\n        return sample\n    else:\n        return clip\n    \n\ndef split_singing_background(clip):\n    \"\"\"Removes silence from clip\n    \"\"\"\n    intervals = split_sound(clip)\n    singing = []\n    background = clip[0:intervals[0][0]]\n    for i in range(len(intervals)-1):\n        background = np.append(background, clip[intervals[i][1]:intervals[i+1][0]])\n    background = np.append(background, clip[intervals[-1][1]:])\n    background = take_random_sample(background)\n    for inter in intervals:\n        singing.extend(clip[inter[0]:inter[1]])\n    singing = np.array(singing)\n    singing = take_random_sample(singing)\n    return singing , silence\n\n\ndef remove_silence_from_file(ebird_code: str, filename: str, source_dir: str, target_sr: int = 32000):\n    ebird_dir = TRAIN_SINGING_DIR / ebird_code\n    background_dir = TRAIN_BACKGROUND_DIR / ebird_code\n    filename = filename.replace('.mp3', '.wav')\n    try:\n        y, _ = librosa.load(\n            source_dir / ebird_code / filename,\n            sr=target_sr, mono=True, res_type=\"kaiser_fast\")\n        sound, background = split_singing_background(y)\n        sf.write(str(ebird_dir / filename), sound, target_sr)\n        sf.write(str(background_dir / filename), background, target_sr)\n    except Exception as e:\n        print(e)\n        with open(\"skipped.txt\", \"a\") as f:\n            file_path = str(source_dir / ebird_code / filename)\n            f.write(file_path + ' ' + str(e) + \"\\n\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_list = [train[train['ebird_code'].str.startswith(('a'))],  # Remove 'b' to save space\n              train[train['ebird_code'].str.startswith(('c', 'd', 'e', 'f'))],\n              train[train['ebird_code'].str.startswith(('g' 'h', 'i', 'j', 'k', 'l', 'm'))],\n              train[train['ebird_code'].str.startswith(('n', 'o', 'p', 'q', 'r'))],\n              train[train['ebird_code'].str.startswith(('s', 't', 'u', 'v', 'w', 'x', 'y', 'z'))]\n             ]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from joblib import delayed, Parallel\nimport soundfile as sf\nwarnings.simplefilter(\"ignore\")\nfor i in range(1): ## Change to 5 for complete dataset (does not work on kaggle because HDD is too small)\n    train_audio_infos = train_list[i][[\"ebird_code\", \"filename\"]].values.tolist()\n    source_dir = TRAIN_RESAMPLED_AUDIO_DIRS[i]\n    #Parallel(n_jobs=-1, verbose=5)(\n    #     delayed(remove_silence_from_file)(ebird_code, file_name, source_dir) for ebird_code, file_name in train_audio_infos)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}