{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Intro\n\nWelcome to the [BirdCLEF 2021 - Birdcall Identification](https://www.kaggle.com/c/birdclef-2021/overview) compedition.\n\n![](https://storage.googleapis.com/kaggle-competitions/kaggle/25954/logos/header.png)\n\nWe will give you first a short introduction to start with your work. The nex step is to show a short analysis befor definen a model with keras.\n\nWe recommend [this notebook](https://www.kaggle.com/drcapa/recognizesongapp-fromscratch-tutorial) for handling audio data tutorial.\n\n<span style=\"color: royalblue;\">Please vote the notebook up if it helps you. Feel free to leave a comment above the notebook. Thank you. </span>","metadata":{}},{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport matplotlib.pyplot as plt\nimport soundfile as sf\nimport librosa\nimport librosa.display\nimport IPython.display as display\n\nfrom sklearn.model_selection import train_test_split\n\nfrom keras.utils import Sequence\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Conv1D, MaxPool1D, BatchNormalization\nfrom keras.optimizers import RMSprop,Adam\nfrom keras.applications import VGG19, VGG16, ResNet50\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Path","metadata":{}},{"cell_type":"code","source":"path = '/kaggle/input/birdclef-2021/'\nos.listdir(path)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Functions\nWe define some helper functions.","metadata":{}},{"cell_type":"code","source":"def read_ogg_file(path, file):\n    \"\"\" Read ogg audio file and return numpay array and samplerate\"\"\"\n    \n    data, samplerate = sf.read(path+file)\n    return data, samplerate\n\n\ndef plot_audio_file(data, samplerate):\n    \"\"\" Plot the audio data\"\"\"\n    \n    sr = samplerate\n    fig = plt.figure(figsize=(8, 4))\n    x = range(len(data))\n    y = data\n    plt.plot(x, y)\n    plt.plot(x, y, color='red')\n    plt.legend(loc='upper center')\n    plt.grid()\n    \n    \ndef plot_spectrogram(data, samplerate):\n    \"\"\" Plot spectrogram with mel scaling \"\"\"\n    \n    sr = samplerate\n    spectrogram = librosa.feature.melspectrogram(data, sr=sr)\n    log_spectrogram = librosa.power_to_db(spectrogram, ref=np.max)\n    librosa.display.specshow(log_spectrogram, sr=sr, x_axis='time', y_axis='mel')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Data","metadata":{}},{"cell_type":"code","source":"train_labels = pd.read_csv(path+'train_soundscape_labels.csv')\ntrain_meta = pd.read_csv(path+'train_metadata.csv')\ntest_data = pd.read_csv(path+'test.csv')\nsamp_subm = pd.read_csv(path+'sample_submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Overview","metadata":{}},{"cell_type":"code","source":"print('Number train label samples:', len(train_labels))\nprint('Number train meta samples:', len(train_meta))\nprint('Number train short folder:', len(os.listdir(path+'train_short_audio')))\nprint('Number train audios:', len(os.listdir(path+'train_soundscapes')))\nprint('Number test samples:', len(test_data))","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir(path+'train_short_audio/caltow')[:2]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_meta.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# A Sample File\nWe focus on the sample in the first row of the train meta data.","metadata":{}},{"cell_type":"code","source":"row = 0\ntrain_meta.iloc[row]","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We extract to features, the primary label which is the name of the folder where the audio file is stored and the filename:","metadata":{}},{"cell_type":"code","source":"label = train_meta.loc[row, 'primary_label']\nfilename = train_meta.loc[row, 'filename']\n\n# Check if the file is in the folder\nfilename in os.listdir(path+'train_short_audio/'+label)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Load the data and samplerate:","metadata":{}},{"cell_type":"code","source":"data, samplerate = sf.read(path+'train_short_audio/'+label+'/'+filename)\nprint(data[:8])\nprint(samplerate)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_audio_file(data, samplerate)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Plot [spectrogram](https://en.wikipedia.org/wiki/Spectrogram) with mel scaling:","metadata":{}},{"cell_type":"code","source":"plot_spectrogram(data, samplerate)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Display the audio of the file:","metadata":{}},{"cell_type":"code","source":"display.Audio(path+'train_short_audio/'+label+'/'+filename)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploratory Data Analysis\nOur challenge is to identify which birds are calling in **long** recordings.\n\nThere are 20 long audio files in the folder train_soundscapes. And there are also 20 unique audio ids: ","metadata":{}},{"cell_type":"code","source":"train_labels['audio_id'].unique()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Each audio file consists of 120 birds with a lenth of 5 seconds.","metadata":{}},{"cell_type":"code","source":"train_labels.groupby(by=['audio_id']).count()['birds'][:4]","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So we have to split the long audio into 120 small audio.","metadata":{}},{"cell_type":"markdown","source":"## Focus On Labels\nThe target label birds is a space delimited list of any bird songs present in the 5 second window. So we have to encode the labels. Therefor we look on an example with 3 different birds:","metadata":{}},{"cell_type":"code","source":"print('original label:', train_labels.loc[458, 'birds'])\nprint('split into list:', train_labels.loc[458, 'birds'].split(' '))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We extract all label of the train data:","metadata":{}},{"cell_type":"code","source":"labels = []\nfor row in train_labels.index:\n    labels.extend(train_labels.loc[row, 'birds'].split(' '))\nlabels = list(set(labels))\n\nprint('Number of unique bird labels:', len(labels))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We encode the labels and write them into a data frame:","metadata":{}},{"cell_type":"code","source":"df_labels_train = pd.DataFrame(index=train_labels.index, columns=labels)\nfor row in train_labels.index:\n    birds = train_labels.loc[row, 'birds'].split(' ')\n    for bird in birds:\n        df_labels_train.loc[row, bird] = 1\ndf_labels_train.fillna(0, inplace=True)\n\n# We set a dummy value for the target label in the test data because we will need for the Data Generator\ntest_data['birds'] = 'nocall'\n\ndf_labels_test = pd.DataFrame(index=test_data.index, columns=labels)\nfor row in test_data.index:\n    birds = test_data.loc[row, 'birds'].split(' ')\n    for bird in birds:\n        df_labels_test.loc[row, bird] = 1\ndf_labels_test.fillna(0, inplace=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This representation of the labels we can use for further analysis. In instance for the distribution of the bird labels. We show the top 10 of the most observations:","metadata":{}},{"cell_type":"code","source":"df_labels_train.sum().sort_values(ascending=False)[:10]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Finally we merge the labels with the original data:","metadata":{}},{"cell_type":"code","source":"train_labels = pd.concat([train_labels, df_labels_train], axis=1)\ntest_data = pd.concat([test_data, df_labels_test], axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Focus On Example","metadata":{}},{"cell_type":"markdown","source":"We focus on an example. The first audio file is named by","metadata":{}},{"cell_type":"code","source":"file = os.listdir(path+'train_soundscapes')[0]\nfile","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We load the data and samplerate:","metadata":{}},{"cell_type":"code","source":"data, samplerate = read_ogg_file(path+'train_soundscapes/', file)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The numpy array has a lenght of 19,200,000. So every sample consists of 160,000 values. These 160,000 values describes 5 seconds of the audio file.\n\nWe split the file name into the audio_id and site:","metadata":{}},{"cell_type":"code","source":"audio_id = file.split('_')[0]\nsite = file.split('_')[1]\nprint('audio_id:', audio_id, ', site:', site)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We focus on the samples with the label birds unequal to nocall. There are 4 samples","metadata":{}},{"cell_type":"code","source":"train_labels[(train_labels['audio_id']==int(audio_id)) & (train_labels['site']==site) & (train_labels['birds']!='nocall')]","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We want to extract the first example with the id 1771. This bird we can here from 455 seconds to 460 seconds.  ","metadata":{}},{"cell_type":"code","source":"sub_data = data[int(455/5)*160000:int(460/5)*160000]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Plot the audio array:","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(14, 5))\nlibrosa.display.waveplot(sub_data, sr=samplerate)\nplt.grid()\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Listen to the bird:","metadata":{}},{"cell_type":"code","source":"display.Audio(sub_data, rate=samplerate)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Parameter\nBased on the EDA we define some parameters:","metadata":{}},{"cell_type":"code","source":"data_lenght = 160000\naudio_lenght = 5\nnum_labels = len(labels)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For the Data Generator we want to define in the next step we need additional parameters:","metadata":{}},{"cell_type":"code","source":"batch_size = 16","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train, Val And Test Data","metadata":{}},{"cell_type":"code","source":"list_IDs_train, list_IDs_val = train_test_split(list(train_labels.index), test_size=0.33, random_state=2021)\nlist_IDs_test = list(samp_subm.index)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Audio Data Generator\nWe use a Data Generator to load the data on demand.","metadata":{}},{"cell_type":"code","source":"class DataGenerator(Sequence):\n    def __init__(self, path, list_IDs, data, batch_size):\n        self.path = path\n        self.list_IDs = list_IDs\n        self.data = data\n        self.batch_size = batch_size\n        self.indexes = np.arange(len(self.list_IDs))\n        \n    def __len__(self):\n        len_ = int(len(self.list_IDs)/self.batch_size)\n        if len_*self.batch_size < len(self.list_IDs):\n            len_ += 1\n        return len_\n    \n    def __getitem__(self, index):\n        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n        list_IDs_temp = [self.list_IDs[k] for k in indexes]\n        X, y = self.__data_generation(list_IDs_temp)\n        X = X.reshape((self.batch_size, 100, 1600//2))\n        return X, y\n    \n    def __data_generation(self, list_IDs_temp):\n        X = np.zeros((self.batch_size, data_lenght//2))\n        y = np.zeros((self.batch_size, num_labels))\n        for i, ID in enumerate(list_IDs_temp):\n            prefix = str(self.data.loc[ID, 'audio_id'])+'_'+self.data.loc[ID, 'site']\n            file_list = [s for s in os.listdir(self.path) if prefix in s]\n            if len(file_list) == 0:\n                # Dummy for missing test audio files\n                audio_file_fft = np.zeros((data_lenght//2))\n            else:\n                file = file_list[0]#[s for s in os.listdir(self.path) if prefix in s][0]\n                audio_file, audio_sr = read_ogg_file(self.path, file)\n                audio_file = audio_file[int((self.data.loc[ID, 'seconds']-5)/audio_lenght)*data_lenght:int(self.data.loc[ID, 'seconds']/audio_lenght)*data_lenght]\n                audio_file_fft = np.abs(np.fft.fft(audio_file)[: len(audio_file)//2])\n                # scale data\n                audio_file_fft = (audio_file_fft-audio_file_fft.mean())/audio_file_fft.std()\n            X[i, ] = audio_file_fft\n            y[i, ] = self.data.loc[ID, self.data.columns[5:]].values\n        return X, y","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Test the Data Generator","metadata":{}},{"cell_type":"code","source":"train_generator = DataGenerator(path+'train_soundscapes/', list_IDs_train, train_labels, batch_size)\nval_generator = DataGenerator(path+'train_soundscapes/', list_IDs_val, train_labels, batch_size)\ntest_generator = DataGenerator(path+'test_soundscapes/', list_IDs_test, test_data, batch_size)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define Model","metadata":{}},{"cell_type":"code","source":"epochs = 2\nlernrate = 2e-3","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Sequential()\nmodel.add(Conv1D(64, input_shape=(100, 1600//2,), kernel_size=5, strides=4, activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(MaxPool1D(pool_size=(4)))\nmodel.add(Conv1D(64, kernel_size=3, activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(Flatten())\nmodel.add(Dense(256, activation='relu'))\nmodel.add(Dense(num_labels, activation='sigmoid'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer = Adam(lr=lernrate),\n              loss='binary_crossentropy',\n              metrics=['binary_accuracy'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit_generator(generator=train_generator, validation_data=val_generator, epochs = epochs, workers=4)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Analyse Training","metadata":{}},{"cell_type":"code","source":"fig, axs = plt.subplots(1, 2, figsize=(16, 4))\nfig.subplots_adjust(hspace = .2, wspace=.2)\naxs = axs.ravel()\nloss = history.history['loss']\nloss_val = history.history['val_loss']\nepochs = range(1, len(loss)+1)\naxs[0].plot(epochs, loss, 'bo', label='loss_train')\naxs[0].plot(epochs, loss_val, 'ro', label='loss_val')\naxs[0].set_title('Value of the loss function')\naxs[0].set_xlabel('epochs')\naxs[0].set_ylabel('value of the loss function')\naxs[0].legend()\naxs[0].grid()\nacc = history.history['binary_accuracy']\nacc_val = history.history['val_binary_accuracy']\naxs[1].plot(epochs, acc, 'bo', label='accuracy_train')\naxs[1].plot(epochs, acc_val, 'ro', label='accuracy_val')\naxs[1].set_title('Accuracy')\naxs[1].set_xlabel('Epochs')\naxs[1].set_ylabel('Value of accuracy')\naxs[1].legend()\naxs[1].grid()\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict Test Data","metadata":{}},{"cell_type":"code","source":"y_pred = model.predict_generator(test_generator, verbose=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Set all values grather than 0.5 to 1:","metadata":{}},{"cell_type":"code","source":"y_test = np.where(y_pred > 0.5, 1, 0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Generate target label string:","metadata":{}},{"cell_type":"code","source":"for row in samp_subm.index:\n    string = ''\n    for col in range(len(y_test[row])):\n        if y_test[row][col] == 1:\n            if string == '':\n                string += labels[col]\n            else:\n                string += ' ' + labels[col]\n    if string == '':\n        string = 'nocall'\n    samp_subm.loc[row, 'birds'] = string","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Export","metadata":{}},{"cell_type":"code","source":"output = samp_subm\noutput.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}