{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Install and Load Packages","metadata":{"papermill":{"duration":0.051306,"end_time":"2021-04-20T20:49:10.707537","exception":false,"start_time":"2021-04-20T20:49:10.656231","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!jupyter nbextension enable --py widgetsnbextension","metadata":{"papermill":{"duration":104.771648,"end_time":"2021-04-20T20:50:55.528942","exception":false,"start_time":"2021-04-20T20:49:10.757294","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfrom os import listdir\nfrom os.path import isfile, join\n\nfrom tqdm.auto import tqdm\nimport joblib\nimport gc\nimport time\nimport matplotlib.pyplot as plt\n\nimport numpy as np\nimport pandas as pd\nimport scipy as sp\n\nimport librosa\nimport librosa.display\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score\n\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nfrom tensorflow.keras.utils import Progbar\n\nfrom IPython.core.display import display, HTML\nimport IPython.display as ipd\nfrom functools import partial\n\nfrom imblearn.over_sampling import RandomOverSampler\n\ndisplay(HTML(\"<style>.container { width:100% !important; }</style>\"))\nnp.set_printoptions(threshold=100000)","metadata":{"papermill":{"duration":4.171531,"end_time":"2021-04-20T20:50:59.752319","exception":false,"start_time":"2021-04-20T20:50:55.580788","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"librosa.__version__","metadata":{"papermill":{"duration":0.058964,"end_time":"2021-04-20T20:50:59.972534","exception":false,"start_time":"2021-04-20T20:50:59.91357","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.__version__","metadata":{"papermill":{"duration":0.058853,"end_time":"2021-04-20T20:51:00.083678","exception":false,"start_time":"2021-04-20T20:51:00.024825","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.getcwd()","metadata":{"papermill":{"duration":0.059445,"end_time":"2021-04-20T20:51:00.195288","exception":false,"start_time":"2021-04-20T20:51:00.135843","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define global variables","metadata":{"papermill":{"duration":0.052381,"end_time":"2021-04-20T20:51:00.300107","exception":false,"start_time":"2021-04-20T20:51:00.247726","status":"completed"},"tags":[]}},{"cell_type":"code","source":"ON_KAGGLE = True\nMODEL_FOLDER_NAME = 'resnet50/'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train folder\nTRAIN_FOLDER = \"../input/birdsong-recognition/train_audio/\"\n\nif (ON_KAGGLE): # if on kaggle\n    MODEL_PATH = '../input/birdcall-models/'  # load model from memory\n    metric_log = joblib.load(open(MODEL_PATH + \"metric_log.pkl\", \"rb\")) # load metric_log from memory\n    NOCALL_TRAIN_PATH = \"../input/bird-backgrounds/\" # nocall audio\nelse: # if not on kaggle, pick up from last epoch\n    MODEL_PATH = '/Users/longy/Documents/checkpoints/birdcall/' + MODEL_FOLDER_NAME\n    if os.path.isfile(MODEL_PATH + \"metric_log.pkl\"):\n        metric_log = joblib.load(open(MODEL_PATH + \"metric_log.pkl\", \"rb\"))\n        start_epoch = len(metric_log[0])\n    NOCALL_TRAIN_PATH = \"../input/nocall/\"","metadata":{"papermill":{"duration":0.071533,"end_time":"2021-04-20T20:51:00.424078","exception":false,"start_time":"2021-04-20T20:51:00.352545","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# set default sampling rate to 32kHz\nDEFAULT_SR = 32000\n\n# frequency range\nF_MIN, F_MAX = 20, DEFAULT_SR/2\n\n# number of threads\nNUM_THREADS = 8\n\n# number of samples in window\nN_FFT=2048\n\n# step in samples\nHOP_LENGTH = 512\n\n# number of mel bins\nN_MELS=128\n\n# number of decibels below reference value to preserve in log-melspectrogram\nTOP_DB=80\n\n# clip length to train and predict on\nNUM_SECONDS=5\n\n# percentage of training data vs. validation data\nTRAIN_SIZE=0.8","metadata":{"papermill":{"duration":0.060074,"end_time":"2021-04-20T20:51:00.536764","exception":false,"start_time":"2021-04-20T20:51:00.47669","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Explore Training Data","metadata":{"papermill":{"duration":0.0532,"end_time":"2021-04-20T20:51:00.642562","exception":false,"start_time":"2021-04-20T20:51:00.589362","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train_info = pd.read_csv(\"../input/birdsong-recognition/train.csv\").drop_duplicates()\nprint(train_info.shape)\nprint(len(train_info.ebird_code.unique()))\nprint(train_info.columns)\ntrain_info.head()","metadata":{"papermill":{"duration":0.544249,"end_time":"2021-04-20T20:51:01.239325","exception":false,"start_time":"2021-04-20T20:51:00.695076","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## create code for each bird, add code for nocall","metadata":{"papermill":{"duration":0.053943,"end_time":"2021-04-20T20:51:01.740652","exception":false,"start_time":"2021-04-20T20:51:01.686709","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# number of birds to make prediction for, adding 'nocall'\nnum_birds = len(train_info.ebird_code.unique())+1\n\n# categorize ebird_code\ntrain_info['ebird_code_cat'] = train_info.ebird_code.astype('category').cat.codes","metadata":{"papermill":{"duration":0.066506,"end_time":"2021-04-20T20:51:01.861135","exception":false,"start_time":"2021-04-20T20:51:01.794629","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## create lookup between bird name and bird code","metadata":{"papermill":{"duration":0.054221,"end_time":"2021-04-20T20:51:01.970046","exception":false,"start_time":"2021-04-20T20:51:01.915825","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# create lookup for abbreviated name\nnocall_label='nocall'\nname_lookup = dict(zip(train_info.ebird_code.astype('category').cat.codes, \n                       train_info.ebird_code.astype('category')))\n\nnocall_code = np.max(train_info.ebird_code.astype('category').cat.codes.unique())+1\nname_lookup[nocall_code]=nocall_label\n\n# create reverse lookup for code (from abbreviated name)\ncode_lookup={v:k for k,v in name_lookup.items()}\n\n# create lookup for sampling rate\nsr_lookup = dict(zip(train_info.filename, train_info.sampling_rate))","metadata":{"papermill":{"duration":0.081212,"end_time":"2021-04-20T20:51:02.105942","exception":false,"start_time":"2021-04-20T20:51:02.02473","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Examine one audio file","metadata":{"papermill":{"duration":0.056174,"end_time":"2021-04-20T20:51:02.219843","exception":false,"start_time":"2021-04-20T20:51:02.163669","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# example\nexample = train_info.iloc[0,:]\n\n# filename\nfilename = example.filename\n\n# ebird\nbird = example.ebird_code\n\n# sampling rate\nsr = example.sampling_rate\n\n# duration of clip\nduration = example.duration\n\nprint(\"#ebird code: {}\\n\".format(bird))\nprint(\"#label: {}\\n\".format(example.primary_label))\nprint(\"#secondary labels: {}\\n\".format(example.secondary_labels))\nprint(\"#description:\\n {}\\n\".format(example.description))\nprint(\"#type: {}\\n\".format(example.type))\nprint(\"#saw bird: {}\\n\".format(example.bird_seen))\nprint(\"#sampling rate: {} Hz\\n\".format(sr))\nprint(\"#recording length: {} seconds\\n\".format(duration))","metadata":{"papermill":{"duration":0.068388,"end_time":"2021-04-20T20:51:02.343799","exception":false,"start_time":"2021-04-20T20:51:02.275411","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load sound clip","metadata":{"papermill":{"duration":0.05471,"end_time":"2021-04-20T20:51:02.453696","exception":false,"start_time":"2021-04-20T20:51:02.398986","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# shared in discussion: https://www.kaggle.com/c/birdsong-recognition/discussion/179592\ndef load_clip(path):\n    clip, sr_native = librosa.core.audio.__audioread_load(path, offset=0.0, duration=None, dtype=np.float32)\n    clip = librosa.to_mono(clip)\n    sr = DEFAULT_SR\n    if sr_native > 0:\n        clip = librosa.resample(clip, sr_native, sr, res_type='kaiser_fast')\n    return clip, sr","metadata":{"papermill":{"duration":0.062753,"end_time":"2021-04-20T20:51:02.570999","exception":false,"start_time":"2021-04-20T20:51:02.508246","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sound_clip, sr = load_clip(TRAIN_FOLDER + bird + '/' + filename)","metadata":{"papermill":{"duration":1.23904,"end_time":"2021-04-20T20:51:03.865908","exception":false,"start_time":"2021-04-20T20:51:02.626868","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ipd.Audio(TRAIN_FOLDER + bird + '/' + filename)","metadata":{"papermill":{"duration":0.098593,"end_time":"2021-04-20T20:51:04.025214","exception":false,"start_time":"2021-04-20T20:51:03.926621","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Show Timbre\nTimbre is the quality of sound that distinguishes the tone of different instruments and voices even if the sounds have the same pitch and loudness.","metadata":{"papermill":{"duration":0.06961,"end_time":"2021-04-20T20:51:04.16509","exception":false,"start_time":"2021-04-20T20:51:04.09548","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"**mel-spectrogram**","metadata":{"papermill":{"duration":0.070104,"end_time":"2021-04-20T20:51:04.305294","exception":false,"start_time":"2021-04-20T20:51:04.23519","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Definition: short-time-fourier-transform (stft) in each window to represent frequencies in mel-scale (such that equal distances in pitch sounded equally distant to the listener)","metadata":{"papermill":{"duration":0.069925,"end_time":"2021-04-20T20:51:04.445408","exception":false,"start_time":"2021-04-20T20:51:04.375483","status":"completed"},"tags":[]}},{"cell_type":"code","source":"melspectrogram = librosa.feature.melspectrogram(sound_clip, sr=sr, fmin=F_MIN, fmax=F_MAX)\nprint(\"In this case, melspectrogram computed {} mel-frequency spectrogram coefficients over {} frames.\".format(melspectrogram.shape[0], melspectrogram.shape[1]))\nmelspectrogram = librosa.power_to_db(melspectrogram).astype(np.float32)\nplt.figure(figsize=(20,3))\nlibrosa.display.specshow(melspectrogram, sr=sr, x_axis='time', y_axis='mel', fmin=F_MIN, fmax=F_MAX)\nplt.colorbar(format='%+2.0f dB')\n\nprint(melspectrogram.shape)\nprint(\"no white noise:\")\nipd.Audio(sound_clip, rate=sr)","metadata":{"papermill":{"duration":0.824896,"end_time":"2021-04-20T20:51:05.341727","exception":false,"start_time":"2021-04-20T20:51:04.516831","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Decibel-Frequency chart","metadata":{"papermill":{"duration":0.157926,"end_time":"2021-04-20T20:51:05.691997","exception":false,"start_time":"2021-04-20T20:51:05.534071","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Definition: Plot x=Frequency in Hz and y=Decibel for the entire clip. Useful to show the effect of adding noise to clip.","metadata":{"papermill":{"duration":0.169068,"end_time":"2021-04-20T20:51:06.035562","exception":false,"start_time":"2021-04-20T20:51:05.866494","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def plot_db_freq(clip):\n    S = librosa.stft(clip)\n    D = librosa.amplitude_to_db(np.abs(S))\n    D_AVG = np.mean(D, axis=1)\n\n    x_ticks_positions = [n for n in range(0, N_FFT // 2, N_FFT // 16)]\n    x_ticks_labels = [str(sr / N_FFT * n) + 'Hz' for n in x_ticks_positions]\n\n    plt.figure(figsize=(10,5))\n    plt.plot(D_AVG)\n    plt.xticks(x_ticks_positions, x_ticks_labels)\n    plt.xlabel('Frequency')\n    plt.ylabel('dB')\n    plt.show()","metadata":{"papermill":{"duration":0.125706,"end_time":"2021-04-20T20:51:06.277817","exception":false,"start_time":"2021-04-20T20:51:06.152111","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_db_freq(sound_clip)\nipd.Audio(sound_clip, rate=sr)","metadata":{"papermill":{"duration":0.409473,"end_time":"2021-04-20T20:51:06.802361","exception":false,"start_time":"2021-04-20T20:51:06.392888","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### add white noise","metadata":{"papermill":{"duration":0.153634,"end_time":"2021-04-20T20:51:07.110515","exception":false,"start_time":"2021-04-20T20:51:06.956881","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def add_white_noise(clip, wn_rate):\n    return clip + wn_rate * np.random.randn(len(clip)) # randn: standard normal distribution","metadata":{"papermill":{"duration":0.165504,"end_time":"2021-04-20T20:51:07.433298","exception":false,"start_time":"2021-04-20T20:51:07.267794","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wn_rate = 0.01\nsound_clip_with_wn = add_white_noise(sound_clip, wn_rate)\nplot_db_freq(sound_clip_with_wn)","metadata":{"papermill":{"duration":0.418607,"end_time":"2021-04-20T20:51:08.005846","exception":false,"start_time":"2021-04-20T20:51:07.587239","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### add pink noise","metadata":{"papermill":{"duration":0.155006,"end_time":"2021-04-20T20:51:08.317168","exception":false,"start_time":"2021-04-20T20:51:08.162162","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def add_pink_noise(clip, freq_floor,freq_ceil, n_freq):\n    freq = np.linspace(freq_floor,freq_ceil,n_freq)\n    noise = np.zeros(len(clip))\n    for f in freq:\n        amp = 1/f**1\n        noise = noise + amp*np.sin(2*np.pi*f*clip+np.random.rand(1)*2*np.pi) # rand: uniform distribution\n    return noise","metadata":{"papermill":{"duration":0.174296,"end_time":"2021-04-20T20:51:08.649135","exception":false,"start_time":"2021-04-20T20:51:08.474839","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sound_clip_with_pn = add_pink_noise(sound_clip,1,100,30)\nplot_db_freq(sound_clip_with_pn)","metadata":{"papermill":{"duration":1.009393,"end_time":"2021-04-20T20:51:09.813527","exception":false,"start_time":"2021-04-20T20:51:08.804134","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### experiment with different augmentation parameters","metadata":{"papermill":{"duration":0.189793,"end_time":"2021-04-20T20:51:10.159755","exception":false,"start_time":"2021-04-20T20:51:09.969962","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"#### useful functions","metadata":{"papermill":{"duration":0.250552,"end_time":"2021-04-20T20:51:10.663638","exception":false,"start_time":"2021-04-20T20:51:10.413086","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# get melspectrogram\ndef librosa_get_melspec(sound_clip, sr):\n    melspectrogram = librosa.feature.melspectrogram(sound_clip, \n                                                    n_fft=N_FFT, \n                                                    win_length=N_FFT, \n                                                    center=False, \n                                                    sr=sr, \n                                                    fmin=F_MIN, \n                                                    fmax=F_MAX)\n    melspectrogram = librosa.power_to_db(melspectrogram).astype(np.float32)\n    return melspectrogram\n\n# standardize 2D image, convert to grayscale: https://www.kaggle.com/daisukelab/cnn-2d-basic-solution-powered-by-fast-ai\ndef np_to_grayscale(X, mean=None, std=None, norm_max=None, norm_min=None, eps=1e-6):\n    X = standardize(X)\n    X = stack(X)\n    return X\n\ndef standardize(X, mean=None, std=None, norm_max=None, norm_min=None, eps=1e-6):\n    mean = mean or X.mean()\n    std = std or X.std()\n    Xstd = (X - mean) / (std + eps)\n    _min, _max = Xstd.min(), Xstd.max()\n    norm_max = norm_max or _max\n    norm_min = norm_min or _min\n    if (_max - _min) > eps:\n        V = Xstd\n        V[V < norm_min] = norm_min\n        V[V > norm_max] = norm_max\n        V = 255 * (V - norm_min) / (norm_max - norm_min)\n        V = V.astype(np.uint8)\n    else:\n        V = np.zeros_like(Xstd, dtype=np.uint8)\n    return V\n\ndef stack(X):\n    return np.stack([X,X,X],axis=-1)\n\n# crop 5 seconds\ndef crop_image(img, sr, random=True, num_seconds=NUM_SECONDS, hop_length=HOP_LENGTH):\n    height, width = img.shape\n    duration = width*hop_length/sr\n    if duration>num_seconds: \n        if random: #randomly crop 5 seconds\n            end_second = np.random.uniform(low=num_seconds,high=duration, size=1)[0]\n            start_second = end_second-num_seconds\n        else: #crop first 5 seconds\n            end_second = num_seconds\n            start_second = end_second-num_seconds\n    else:\n        end_second = duration\n        start_second = 0\n    \n    start_frame = int(np.floor(start_second*sr/hop_length))\n    end_frame = int(np.round(end_second*sr/hop_length))\n    \n    return img[:, start_frame:end_frame].astype(np.float32)","metadata":{"papermill":{"duration":0.37441,"end_time":"2021-04-20T20:51:11.298328","exception":false,"start_time":"2021-04-20T20:51:10.923918","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### white noise","metadata":{"papermill":{"duration":0.157693,"end_time":"2021-04-20T20:51:11.613106","exception":false,"start_time":"2021-04-20T20:51:11.455413","status":"completed"},"tags":[]}},{"cell_type":"code","source":"n_col = 3\nn_img = 9\n\nfig, axs = plt.subplots(n_img//n_col,n_col,figsize=(5*n_col,5*(n_img//n_col-1)))\nfor i in range(0,n_img):\n    x = i//n_col\n    y = i-x*n_col\n    ax = axs[x][y]\n    \n    sc = add_white_noise(sound_clip, i*0.005)\n    melspec = librosa_get_melspec(sc, sr)\n    cropped_melspec = crop_image(melspec,sr,random=False)\n    cropped_img = np_to_grayscale(cropped_melspec)\n\n    ax.imshow(cropped_img)\n    ax.set_title(\"White Noise Rate: \" + str(i*0.005))","metadata":{"papermill":{"duration":2.605441,"end_time":"2021-04-20T20:51:14.391989","exception":false,"start_time":"2021-04-20T20:51:11.786548","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Pink Noise","metadata":{"papermill":{"duration":0.164893,"end_time":"2021-04-20T20:51:14.723003","exception":false,"start_time":"2021-04-20T20:51:14.55811","status":"completed"},"tags":[]}},{"cell_type":"code","source":"n_col = 3\nn_img = 9\n\nfig, axs = plt.subplots(n_img//n_col,n_col,figsize=(5*n_col,5*(n_img//n_col-1)))\nfor i in range(0,n_img):\n    x = i//n_col\n    y = i-x*n_col\n    ax = axs[x][y]\n    \n    sc = add_pink_noise(sound_clip,1,100*(x+1),30*(y+1))\n    melspec = librosa_get_melspec(sc, sr)\n    cropped_melspec = crop_image(melspec,sr,random=False)\n    cropped_img = np_to_grayscale(cropped_melspec)\n\n    ax.imshow(cropped_img)\n    ax.set_title(\"Pink Noise: {}-{} Hz in a total of {} steps\".format(1,100*(x+1),30*(y+1)))","metadata":{"papermill":{"duration":14.516046,"end_time":"2021-04-20T20:51:29.409286","exception":false,"start_time":"2021-04-20T20:51:14.89324","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get nocall data","metadata":{"papermill":{"duration":0.171868,"end_time":"2021-04-20T20:51:29.755984","exception":false,"start_time":"2021-04-20T20:51:29.584116","status":"completed"},"tags":[]}},{"cell_type":"code","source":"nocall_info = [(nocall_code, join(NOCALL_TRAIN_PATH, f)) for f in listdir(NOCALL_TRAIN_PATH) if isfile(join(NOCALL_TRAIN_PATH, f))]\nnocall_info = pd.DataFrame(nocall_info, columns=['ebird_code_cat', 'filepath'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nocall_audio,sr = load_clip(nocall_info.iloc[0].filepath)\nipd.Audio(nocall_audio,rate=sr)","metadata":{"papermill":{"duration":0.420791,"end_time":"2021-04-20T20:51:31.0825","exception":false,"start_time":"2021-04-20T20:51:30.661709","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nocall_melspec = librosa_get_melspec(nocall_audio, sr)\nprint(nocall_melspec.shape)\nplt.figure(figsize=(20,3))\nplt.imshow(np_to_grayscale(nocall_melspec))","metadata":{"papermill":{"duration":0.453368,"end_time":"2021-04-20T20:51:31.724178","exception":false,"start_time":"2021-04-20T20:51:31.27081","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build Pipeline","metadata":{"papermill":{"duration":0.191713,"end_time":"2021-04-20T20:51:32.109694","exception":false,"start_time":"2021-04-20T20:51:31.917981","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train_info['filepath'] = TRAIN_FOLDER+train_info[\"ebird_code\"]+\"/\"+train_info[\"filename\"]\ntrain_info.ebird_code_cat = train_info.ebird_code_cat.astype('int32')\ntrain_info.filepath = train_info.filepath.astype('string')\ntrain_info[['ebird_code_cat', 'filepath']].head(1)","metadata":{"papermill":{"duration":0.233348,"end_time":"2021-04-20T20:51:32.540011","exception":false,"start_time":"2021-04-20T20:51:32.306663","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nocall_info.ebird_code_cat = nocall_info.ebird_code_cat.astype('int32')\nnocall_info.head(1)","metadata":{"papermill":{"duration":0.231662,"end_time":"2021-04-20T20:51:32.965679","exception":false,"start_time":"2021-04-20T20:51:32.734017","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Extract","metadata":{"papermill":{"duration":0.196923,"end_time":"2021-04-20T20:51:33.357371","exception":false,"start_time":"2021-04-20T20:51:33.160448","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def get_birdcall_audio(filepath, label):\n    audio, sr = load_clip(filepath.numpy())\n    return audio, tf.cast(sr,tf.float32), label\n\nextract = lambda x,y: tf.py_function(get_birdcall_audio,[x,y], [tf.float32,tf.float32,tf.int32])","metadata":{"papermill":{"duration":0.208044,"end_time":"2021-04-20T20:51:33.767037","exception":false,"start_time":"2021-04-20T20:51:33.558993","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Transform","metadata":{"papermill":{"duration":0.193202,"end_time":"2021-04-20T20:51:34.15508","exception":false,"start_time":"2021-04-20T20:51:33.961878","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"1. randomly crop 5 seconds from clip","metadata":{"papermill":{"duration":0.194824,"end_time":"2021-04-20T20:51:35.747447","exception":false,"start_time":"2021-04-20T20:51:35.552623","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def crop_and_pad(sound_clip,sr,label):\n    length = tf.size(sound_clip,out_type=tf.int32)\n    length = tf.cast(length,tf.float32)\n    duration = length/sr\n    \n    if (duration>=NUM_SECONDS):\n        # randomly select end second\n        end_second = tf.random.uniform(shape=[],\n                                       minval=NUM_SECONDS,\n                                       maxval=duration,\n                                       dtype=tf.float32)\n        # transform second to sample\n        cut_max = end_second*sr\n        cut_min = (end_second-NUM_SECONDS)*sr\n\n        # cast to integer\n        cut_min = tf.cast(cut_min, tf.int32)\n        cut_max = tf.cast(cut_max, tf.int32)\n\n        # cut clip\n        sound_clip = sound_clip[cut_min:cut_max]\n    else:\n        zero_padding_len = tf.cast((NUM_SECONDS-duration)*sr, tf.int32)\n        sound_clip = tf.concat([sound_clip,tf.zeros(zero_padding_len)],axis=0)\n    \n    return sound_clip,sr,label","metadata":{"papermill":{"duration":0.205621,"end_time":"2021-04-20T20:51:36.147542","exception":false,"start_time":"2021-04-20T20:51:35.941921","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"2. augment data by adding white noise or pink noise","metadata":{"papermill":{"duration":0.198078,"end_time":"2021-04-20T20:51:36.540648","exception":false,"start_time":"2021-04-20T20:51:36.34257","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"- choice 1: no augmentation\n- choice 2: white noise up to 0.5\n- choice 3: pink noise (200 or 300),30","metadata":{"papermill":{"duration":0.197268,"end_time":"2021-04-20T20:51:36.933846","exception":false,"start_time":"2021-04-20T20:51:36.736578","status":"completed"},"tags":[]}},{"cell_type":"code","source":"augment_choices = tf.range(0,3,dtype=tf.float32)\nwr_ceil = 0.05\nfreq_ceil_choices = 100. * tf.range(2,4,dtype=tf.float32) # 200, 300\nn_freq = 30","metadata":{"papermill":{"duration":1.106991,"end_time":"2021-04-20T20:51:38.238487","exception":false,"start_time":"2021-04-20T20:51:37.131496","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def augment(sound_clip,sr,label):\n    choice = tf.random.shuffle(augment_choices)[0]\n    length = tf.size(sound_clip, out_type=tf.int32)\n    \n    if tf.math.equal(choice,1): # white noise\n        wr_rate = tf.random.uniform([],0,wr_ceil)\n        sound_clip = sound_clip + wr_rate * tf.random.normal([length])\n    elif tf.math.equal(choice,2): # pink noise\n        freq_ceil = tf.random.shuffle(freq_ceil_choices)[0]\n        freqs = tf.linspace(1.,freq_ceil,n_freq)\n\n        i0 = tf.constant(0)\n        s0 = tf.zeros(shape=length)\n        \n        c = lambda i,s: i < n_freq\n        b = lambda i,s: [i+1, \n                         s + 1/freqs[i] * tf.math.sin(2*np.pi*(1/freqs[i])*sound_clip+tf.random.uniform(shape=[],minval=0,maxval=1,dtype=tf.float32)*2*np.pi)]\n        \n        _,sound_clip = tf.while_loop(c,b,loop_vars=[i0,s0],shape_invariants=[i0.get_shape(),sound_clip.get_shape()])\n    \n    return sound_clip,sr,label","metadata":{"papermill":{"duration":0.215356,"end_time":"2021-04-20T20:51:38.654107","exception":false,"start_time":"2021-04-20T20:51:38.438751","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"3. get mel-spectrogram","metadata":{"papermill":{"duration":0.196445,"end_time":"2021-04-20T20:51:39.046251","exception":false,"start_time":"2021-04-20T20:51:38.849806","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def log10(x):\n    numerator = tf.math.log(x)\n    denominator = tf.math.log(tf.constant(10, dtype=numerator.dtype))\n    return tf.divide(numerator, denominator)\n\ndef get_melspec(sound_clip,sr,label):\n    spec = tf.signal.stft(sound_clip,frame_length=N_FFT,frame_step=HOP_LENGTH)\n    spec = tf.abs(spec)\n    linear_to_mel_weight_matrix = tf.signal.linear_to_mel_weight_matrix(num_mel_bins=N_MELS, \n                                                                        num_spectrogram_bins=N_FFT//2+1, \n                                                                        sample_rate=sr, \n                                                                        lower_edge_hertz=F_MIN, \n                                                                        upper_edge_hertz=F_MAX,\n                                                                        dtype=tf.dtypes.float32)\n    melspec = tf.matmul(spec, linear_to_mel_weight_matrix)\n    log_melspec = 10*log10(tf.transpose(melspec)**2)\n    ref = tf.math.reduce_max(log_melspec)-TOP_DB\n    log_melspec = tf.where(log_melspec<ref,ref,log_melspec)\n    \n    return log_melspec,label","metadata":{"papermill":{"duration":0.209694,"end_time":"2021-04-20T20:51:39.453151","exception":false,"start_time":"2021-04-20T20:51:39.243457","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"4. standardize and stack to rgb(#,#,#)","metadata":{"papermill":{"duration":0.196481,"end_time":"2021-04-20T20:51:39.845147","exception":false,"start_time":"2021-04-20T20:51:39.648666","status":"completed"},"tags":[]}},{"cell_type":"code","source":"eps=1e-6\ndef to_grayscale(melspec, label):\n    mean = tf.math.reduce_mean(melspec)\n    std = tf.math.reduce_std(melspec)\n    xstd = (melspec-mean)/(std + eps)\n    norm_min = tf.math.reduce_min(xstd)\n    norm_max = tf.math.reduce_max(xstd)\n    \n    if (norm_max-norm_min>eps):\n        v = 255*(xstd-norm_min)/(norm_max-norm_min)\n        v = tf.cast(v, tf.uint8)\n    else:\n        v = tf.zeros_like(xstd, dtype=tf.uint8)\n    return tf.stack([v,v,v],axis=2), label","metadata":{"papermill":{"duration":0.213826,"end_time":"2021-04-20T20:51:40.512266","exception":false,"start_time":"2021-04-20T20:51:40.29844","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"5. create training and validation","metadata":{"papermill":{"duration":0.195288,"end_time":"2021-04-20T20:51:40.902693","exception":false,"start_time":"2021-04-20T20:51:40.707405","status":"completed"},"tags":[]}},{"cell_type":"code","source":"birdcall_train, birdcall_val = train_test_split(train_info,\n                                                stratify=train_info.ebird_code_cat, \n                                                train_size=TRAIN_SIZE)\nnocall_train, nocall_val = train_test_split(nocall_info,train_size=TRAIN_SIZE)\n\ntrain, val = (pd.concat([birdcall_train[[\"filepath\", \"ebird_code_cat\"]],nocall_train[[\"filepath\", \"ebird_code_cat\"]]],axis=0), \n              pd.concat([birdcall_val[[\"filepath\", \"ebird_code_cat\"]],nocall_val[[\"filepath\", \"ebird_code_cat\"]]],axis=0))","metadata":{"papermill":{"duration":0.251253,"end_time":"2021-04-20T20:51:42.196857","exception":false,"start_time":"2021-04-20T20:51:41.945604","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"balance dataset by oversampling on the minority classes","metadata":{"papermill":{"duration":0.225585,"end_time":"2021-04-20T20:51:42.618304","exception":false,"start_time":"2021-04-20T20:51:42.392719","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def overresample(df):\n    ros = RandomOverSampler()\n    resampled_df, _ = ros.fit_resample(df, df.ebird_code_cat)\n    resampled_df, _ = train_test_split(resampled_df, \n                                       stratify = resampled_df.ebird_code_cat, \n                                       train_size=int(np.mean(df.ebird_code_cat.value_counts().values))/int(np.max(df.ebird_code_cat.value_counts().values)))\n    return resampled_df ","metadata":{"papermill":{"duration":0.315005,"end_time":"2021-04-20T20:51:43.240064","exception":false,"start_time":"2021-04-20T20:51:42.925059","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resampled_train = overresample(train)\nresampled_train.ebird_code_cat.value_counts()","metadata":{"papermill":{"duration":1.444652,"end_time":"2021-04-20T20:51:45.015099","exception":false,"start_time":"2021-04-20T20:51:43.570447","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Pull things together","metadata":{"papermill":{"duration":0.394718,"end_time":"2021-04-20T20:51:45.614016","exception":false,"start_time":"2021-04-20T20:51:45.219298","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def stream_files(df, train=True):\n    if train:\n        ds = (tf.data.Dataset.from_tensor_slices((df.filepath.values,df.ebird_code_cat.values)).\n              map(extract,num_parallel_calls=NUM_THREADS).\n              map(crop_and_pad, num_parallel_calls=NUM_THREADS).\n              map(augment, num_parallel_calls=NUM_THREADS).\n              map(get_melspec, num_parallel_calls=NUM_THREADS))\n    else:\n        ds = (tf.data.Dataset.from_tensor_slices((df.filepath.values,df.ebird_code_cat.values)).\n              map(extract,num_parallel_calls=NUM_THREADS).\n              map(crop_and_pad, num_parallel_calls=NUM_THREADS).\n              map(get_melspec, num_parallel_calls=NUM_THREADS))\n\n    return ds","metadata":{"papermill":{"duration":0.209027,"end_time":"2021-04-20T20:51:46.022816","exception":false,"start_time":"2021-04-20T20:51:45.813789","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files = (stream_files(resampled_train))","metadata":{"papermill":{"duration":1.336236,"end_time":"2021-04-20T20:51:47.555679","exception":false,"start_time":"2021-04-20T20:51:46.219443","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can take a peek at the mel-spectrograms from training data:","metadata":{"papermill":{"duration":0.201227,"end_time":"2021-04-20T20:51:47.954359","exception":false,"start_time":"2021-04-20T20:51:47.753132","status":"completed"},"tags":[]}},{"cell_type":"code","source":"n_col = 4\nn_img = 16\n\nfig, axs = plt.subplots(n_img//n_col,n_col,figsize=(5*n_col,5*(n_img//n_col-1)))\nfor i, (melspec,label) in enumerate(train_files.take(n_img)):\n    x = i//n_col\n    y = i-x*n_col\n    ax = axs[x][y]\n    ax.imshow(melspec)\n    ax.set_title(name_lookup[label.numpy()])","metadata":{"papermill":{"duration":37.790639,"end_time":"2021-04-20T20:52:25.943326","exception":false,"start_time":"2021-04-20T20:51:48.152687","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### get image width and height","metadata":{"papermill":{"duration":0.224174,"end_time":"2021-04-20T20:52:26.38794","exception":false,"start_time":"2021-04-20T20:52:26.163766","status":"completed"},"tags":[]}},{"cell_type":"code","source":"for i, (melspec,label) in enumerate(train_files.take(1)):\n    img_height = melspec.numpy().shape[0]\n    img_width = melspec.numpy().shape[1]\n\nprint(\"image width: {}, height: {}\".format(img_width, img_height))","metadata":{"papermill":{"duration":24.945847,"end_time":"2021-04-20T20:52:51.568775","exception":false,"start_time":"2021-04-20T20:52:26.622928","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load","metadata":{"papermill":{"duration":0.214505,"end_time":"2021-04-20T20:52:52.002152","exception":false,"start_time":"2021-04-20T20:52:51.787647","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### set up params","metadata":{"papermill":{"duration":0.217793,"end_time":"2021-04-20T20:52:52.438546","exception":false,"start_time":"2021-04-20T20:52:52.220753","status":"completed"},"tags":[]}},{"cell_type":"code","source":"epochs = 100\nbatch_size = 64\nlr=.001\nearly_stopping=20\nthreshold=0.5","metadata":{"papermill":{"duration":0.232584,"end_time":"2021-04-20T20:52:52.890426","exception":false,"start_time":"2021-04-20T20:52:52.657842","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### compile model","metadata":{"papermill":{"duration":0.217725,"end_time":"2021-04-20T20:52:53.325419","exception":false,"start_time":"2021-04-20T20:52:53.107694","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"resnet50","metadata":{"papermill":{"duration":0.220573,"end_time":"2021-04-20T20:52:53.765168","exception":false,"start_time":"2021-04-20T20:52:53.544595","status":"completed"},"tags":[]}},{"cell_type":"code","source":"img_resize = [img_height, img_width]\n\nmodel = tf.keras.Sequential()\nresnet50 = tf.keras.applications.ResNet50(include_top=False, \n                                          pooling='avg', \n                                          weights=None,\n                                          input_shape=(img_height, img_width, 3))\n\nmodel.add(resnet50)\nmodel.add(tf.keras.layers.Dense(num_birds, activation='sigmoid'))","metadata":{"papermill":{"duration":2.648656,"end_time":"2021-04-20T20:52:56.638211","exception":false,"start_time":"2021-04-20T20:52:53.989555","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Adam\noptimizer = tf.keras.optimizers.Adam(learning_rate=lr)\n\n# cross entropy loss function\nloss_fn = tf.keras.losses.BinaryCrossentropy(reduction=tf.keras.losses.Reduction.SUM)\n\n# metrics\nf1_train = tfa.metrics.F1Score(num_birds, 'micro')\nf1_val = tfa.metrics.F1Score(num_birds, 'micro')\n\n# for logging results\nmetric_name = [\"loss\", \"f1\", \"val-f1\"]\n\nbest_f1_val = None\nbest_epoch = None\n\nstart_epoch = 0\nepoch_num = []\nmetric_log_val_f1 = []\nmetric_log_f1 = []\n\n# compile\nmodel.compile(optimizer=optimizer,loss=loss_fn)\n\n# summary\nmodel.summary()","metadata":{"papermill":{"duration":0.270468,"end_time":"2021-04-20T20:52:57.216094","exception":false,"start_time":"2021-04-20T20:52:56.945626","status":"completed"},"scrolled":true,"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### train step","metadata":{"papermill":{"duration":0.21959,"end_time":"2021-04-20T20:52:57.65514","exception":false,"start_time":"2021-04-20T20:52:57.43555","status":"completed"},"tags":[]}},{"cell_type":"code","source":"@tf.function\ndef train_step(x,y): \n    with tf.GradientTape() as tape:\n        x,y = tf.vectorized_map(lambda a: to_grayscale(a[0],a[1]),(x,y))\n        \n        # predict\n        y_pred = model(x)\n        y_truth = tf.one_hot(y, num_birds)\n\n        # calculate loss\n        loss = loss_fn(y_truth, y_pred)\n\n    # calculate gradients\n    grads = tape.gradient(loss,model.trainable_weights)\n    optimizer.apply_gradients(zip(grads,model.trainable_weights))\n\n    # compute metrics\n    f1_train.update_state(y_truth,y_pred)\n    \n    return loss","metadata":{"papermill":{"duration":0.238759,"end_time":"2021-04-20T20:52:58.113537","exception":false,"start_time":"2021-04-20T20:52:57.874778","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### validation step","metadata":{"papermill":{"duration":0.219358,"end_time":"2021-04-20T20:52:58.554441","exception":false,"start_time":"2021-04-20T20:52:58.335083","status":"completed"},"tags":[]}},{"cell_type":"code","source":"@tf.function\ndef val_step(x,y):\n    x,y = tf.vectorized_map(lambda a: to_grayscale(a[0],a[1]),(x,y))\n    \n    y_pred = model(x)\n    y_truth = tf.one_hot(y, num_birds)\n    \n    f1_val.update_state(y_truth,y_pred)","metadata":{"papermill":{"duration":0.22998,"end_time":"2021-04-20T20:52:59.005907","exception":false,"start_time":"2021-04-20T20:52:58.775927","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### start training","metadata":{"papermill":{"duration":0.218918,"end_time":"2021-04-20T20:52:59.44308","exception":false,"start_time":"2021-04-20T20:52:59.224162","status":"completed"},"tags":[]}},{"cell_type":"code","source":"resampled_train = overresample(train)\nprint(resampled_train.shape)","metadata":{"papermill":{"duration":1.428381,"end_time":"2021-04-20T20:53:01.092311","exception":false,"start_time":"2021-04-20T20:52:59.66393","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_ds = stream_files(val, train=False).batch(batch_size)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"steps_per_epoch = int(np.ceil(len(resampled_train)/batch_size))\nsteps_per_epoch","metadata":{"papermill":{"duration":0.231212,"end_time":"2021-04-20T20:53:01.5456","exception":false,"start_time":"2021-04-20T20:53:01.314388","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if (ON_KAGGLE==False):\n    with np.errstate(all='raise'):\n        # for each epoch\n        for epoch in range(start_epoch, epochs):\n            print(\"epoch %d\" % (epoch,))\n\n            # reset states\n            f1_train.reset_states()\n            f1_val.reset_states()\n            \n            # set up progress bar\n            pb_i = Progbar(target=steps_per_epoch,stateful_metrics=metric_name)\n\n            # get balanced data from train, shuffle, batch\n            resampled_train = overresample(train)\n            train_files = stream_files(resampled_train).batch(batch_size).prefetch(2)\n\n            for (x_batch, y_batch) in train_files:\n                # get loss\n                loss = train_step(x_batch, y_batch)\n                # update progress\n                pb_i.add(1, values=[('loss', loss), ('f1', f1_train.result()), ('val-f1', f1_val.result())])\n\n            # validate\n            for (x_batch_val, y_batch_val) in val_ds:\n                val_step(x_batch_val, y_batch_val)\n\n            # update f1 for validation set\n            pb_i.update(steps_per_epoch, \n                        values=[('loss', loss), ('f1', f1_train.result()),('val-f1', f1_val.result())],\n                        finalize=True)\n\n            # log results\n            epoch_num.append(epoch)\n            metric_log_f1.append(f1_train.result().numpy())\n            metric_log_val_f1.append(f1_val.result().numpy())\n\n            # save checkpoint after each epoch\n            model.save_weights(MODEL_PATH + 'model')\n            joblib.dump([epoch_num,metric_log_val_f1,metric_log_f1], open(MODEL_PATH + \"metric_log.pkl\", \"wb\"))\n\n            # early stopping if f1 score hasn't improved on validation set\n            if best_f1_val is None or f1_val.result()>best_f1_val:\n                best_f1_val,best_epoch=f1_val.result(),epoch\n\n            if best_epoch<epoch-early_stopping:\n                model.stop_training=True\n                print(\"training stopped early at epoch %d\" % epoch)\n                break\nelse:\n    model.load_weights(MODEL_PATH + 'model')\n    metric_log_val_f1 = metric_log[1]\n    metric_log_f1 = metric_log[2]","metadata":{"papermill":{"duration":6.50384,"end_time":"2021-04-20T20:53:08.268922","exception":false,"start_time":"2021-04-20T20:53:01.765082","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Plot F1 Score","metadata":{"papermill":{"duration":0.221004,"end_time":"2021-04-20T20:53:08.709709","exception":false,"start_time":"2021-04-20T20:53:08.488705","status":"completed"},"tags":[]}},{"cell_type":"code","source":"plt.plot(metric_log_val_f1, label='validation f1')\nplt.plot(metric_log_f1, label='training f1')\nplt.xlabel('Epoch')\nplt.ylabel('F1 Score')\nplt.legend(loc='lower right')","metadata":{"papermill":{"duration":0.365112,"end_time":"2021-04-20T20:53:09.294486","exception":false,"start_time":"2021-04-20T20:53:08.929374","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission to evaluate on test data","metadata":{"papermill":{"duration":0.221024,"end_time":"2021-04-20T20:53:09.738034","exception":false,"start_time":"2021-04-20T20:53:09.51701","status":"completed"},"tags":[]}},{"cell_type":"code","source":"BASE_TEST_DIR = '../input/birdsong-recognition' if os.path.exists('../input/birdsong-recognition/test_audio') else '../input/birdcall-check'\nTEST_FOLDER = f'{BASE_TEST_DIR}/test_audio/'\nTEST_FOLDER","metadata":{"papermill":{"duration":0.238809,"end_time":"2021-04-20T20:53:10.199732","exception":false,"start_time":"2021-04-20T20:53:09.960923","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_csv(f'{BASE_TEST_DIR}/test.csv')\ndf_test[\"filepath\"] = TEST_FOLDER + df_test.audio_id + '.mp3'\ndf_test.head()","metadata":{"papermill":{"duration":0.252724,"end_time":"2021-04-20T20:53:10.674392","exception":false,"start_time":"2021-04-20T20:53:10.421668","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def postprocess(y_prob, top = 3):\n    y_candidate = tf.where(tf.where(y_prob > threshold, 1.0, 0.0) * y_prob > 0)\n    y_top = tf.argsort(y_candidate, direction='DESCENDING')\n    if len(y_top)>0: return [name_lookup[code.numpy()] for code in y_top[0][:top]]\n    return []","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_prediction(audio, site):\n    sound_clip,sr=audio\n    \n    if (site=='site_1' or site=='site_2'):\n        melspec = librosa_get_melspec(sound_clip, sr=sr)\n        x = tf.expand_dims(np_to_grayscale(melspec).astype(np.uint8), axis=0)\n        y_prob = model(x)\n        y = postprocess(y_prob)\n    else:\n        duration = librosa.get_duration(sound_clip,sr)\n        if duration<NUM_SECONDS:\n            padded_clip = np.concatenate([sound_clip, np.zeros(int(sr*(NUM_SECONDS-duration)))])\n            melspec = librosa_get_melspec(padded_clip, sr=sr)\n            x = tf.expand_dims(np_to_grayscale(melspec).astype(np.uint8), axis=0)\n            y_prob = model(x)\n            y = postprocess(y_prob)\n        else:\n            start_second, end_second = 0, NUM_SECONDS\n            y = []\n            \n            # predict for each 5 seconds\n            while end_second<=duration:\n                clip = sound_clip[start_second*sr:end_second*sr]\n                melspec = librosa_get_melspec(clip, sr=sr)\n                x = tf.expand_dims(np_to_grayscale(melspec).astype(np.uint8), axis=0)\n                clip_prob = model(x)\n                clip_y = postprocess(clip_prob)\n                y.extend(clip_y)\n                start_second += NUM_SECONDS\n                end_second += NUM_SECONDS\n                \n            # predict for remaining time: at least 1 second\n            if end_second-duration>=1:\n                last_clip = np.concatenate([sound_clip[start_second*sr:], np.zeros(int(sr*(end_second-duration)))])\n                melspec = librosa_get_melspec(last_clip, sr=sr)\n                x = tf.expand_dims(np_to_grayscale(melspec).astype(np.uint8), axis=0)\n                clip_prob = model(x)\n                clip_y = postprocess(clip_prob)\n                y.extend(clip_y)\n    return y","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# remove redundancy from memory\ndel train_info, birdcall_train, birdcall_val, nocall_info, nocall_train, nocall_val, resampled_train\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cached_audio = dict()\nstart = time.time()\n\npreds = []\nfor i in tqdm(df_test.index):\n    row = df_test.iloc[i]\n    start_time = row.seconds-NUM_SECONDS\n    row_id, site, filepath = row.row_id, row.site, row.filepath\n    \n    if (filepath in cached_audio):\n        loaded_clip, sr = cached_audio[filepath]\n    else:\n        loaded_clip, sr = load_clip(filepath)\n        cached_audio[filepath] = (loaded_clip, sr)\n        \n    if site=='site_1' or site=='site_2':\n        audio = loaded_clip[int(start_time*sr):int((start_time+NUM_SECONDS)*sr)], sr\n    else:\n        audio = loaded_clip, sr\n        \n    pred = make_prediction(audio, site)\n    pred = ' '.join([bird for bird in np.unique(pred) if bird!=nocall_label])\n    if (len(pred)==0): pred=nocall_label\n    preds.append([row_id, pred])\n    \nprint(\"prediction finished in %d seconds\" % ((time.time() - start)))\npreds = pd.DataFrame(preds, columns=['row_id', 'birds'])\npreds.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds.to_csv('submission.csv', index=False)","metadata":{"papermill":{"duration":0.336668,"end_time":"2021-04-20T20:54:30.0694","exception":false,"start_time":"2021-04-20T20:54:29.732732","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]}]}