{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport matplotlib.pyplot as plt\nimport soundfile as sf\nimport librosa\nimport librosa.display\nimport IPython.display as display\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport IPython.display as ipd\nimport librosa\nimport librosa.display\nimport os\nfrom tqdm import tqdm\nimport sklearn\nimport seaborn as sns\nimport plotly.express as px\n\n\nimport geopandas as gpd\nfrom shapely.geometry import Point, Polygon\n\nfrom sklearn.model_selection import train_test_split\n\nfrom keras.utils import Sequence\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Conv1D, MaxPool1D, BatchNormalization\nfrom keras.optimizers import RMSprop,Adam\nfrom keras.applications import VGG19, VGG16, ResNet50\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '/kaggle/input/birdclef-2021/'\nos.listdir(path)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_ogg_file(path, file):\n    \"\"\" Read ogg audio file and return numpay array and samplerate\"\"\"\n    \n    data, samplerate = sf.read(path+file)\n    return data, samplerate\n\n\ndef plot_audio_file(data, samplerate):\n    \"\"\" Plot the audio data\"\"\"\n    \n    sr = samplerate\n    fig = plt.figure(figsize=(8, 4))\n    x = range(len(data))\n    y = data\n    plt.plot(x, y)\n    plt.plot(x, y, color='red')\n    plt.legend(loc='upper center')\n    plt.grid()\n    \n    \ndef plot_spectrogram(data, samplerate):\n    \"\"\" Plot spectrogram with mel scaling \"\"\"\n    \n    sr = samplerate\n    spectrogram = librosa.feature.melspectrogram(data, sr=sr)\n    log_spectrogram = librosa.power_to_db(spectrogram, ref=np.max)\n    librosa.display.specshow(log_spectrogram, sr=sr, x_axis='time', y_axis='mel')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels = pd.read_csv(path+'train_soundscape_labels.csv')\ntrain_meta = pd.read_csv(path+'train_metadata.csv')\ntest_data = pd.read_csv(path+'test.csv')\nsamp_subm = pd.read_csv(path+'sample_submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Number train label samples:', len(train_labels))\nprint('Number train meta samples:', len(train_meta))\nprint('Number train short folder:', len(os.listdir(path+'train_short_audio')))\nprint('Number train audios:', len(os.listdir(path+'train_soundscapes')))\nprint('Number test samples:', len(test_data))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir(path+'train_short_audio/caltow')[:2]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_meta.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Data Size","metadata":{}},{"cell_type":"code","source":"print(f\"Training Dataset Shape: {(train_meta.shape)}\")\nprint(f\"Training Dataset Labels Shape: {(train_labels.shape)}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Column-wise unique values","metadata":{}},{"cell_type":"code","source":"print(\"Data: train\")\nprint(\"-----------\")\nfor col in train_meta.columns:\n    print(col + \":\" + (str(len(train_meta[col].unique()))))\n\nprint(\"\\nData: train_labels\")\nprint(\"-----------\")\nfor col in train_labels.columns:\n    print(col + \":\" + (str(len(train_labels[col].unique()))))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Time of the Recording","metadata":{}},{"cell_type":"code","source":"train_meta['year'] = train_meta['date'].apply(lambda x: x.split(\"-\")[0])\ntrain_meta['month'] = train_meta['date'].apply(lambda x: x.split(\"-\")[1])\nplt.figure(figsize=(16, 6))\nax = sns.countplot(train_meta['year'].sort_values(ascending=False), palette=\"hls\")\n\nplt.title(\"Audio Files Registration per Year Made\", fontsize=16)\nplt.xticks(rotation=90, fontsize=13)\nplt.yticks(fontsize=13)\nplt.ylabel(\"Frequency\", fontsize=14)\nplt.xlabel(\"\");","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(16, 6))\nax = sns.countplot(train_meta['month'].sort_values(ascending=False), palette=\"hls\")\n\nplt.title(\"Audio Files Registration per Month Made\", fontsize=16)\nplt.xticks(fontsize=13)\nplt.yticks(fontsize=13)\nplt.ylabel(\"Frequency\", fontsize=14)\nplt.xlabel(\"\");","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row = 0\ntrain_meta.iloc[row]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label = train_meta.loc[row, 'primary_label']\nfilename = train_meta.loc[row, 'filename']\n\n# Check if the file is in the folder\nfilename in os.listdir(path+'train_short_audio/'+label)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualizing the audio","metadata":{}},{"cell_type":"code","source":"filename = '../input/birdclef-2021/train_short_audio/acafly/XC11209.ogg'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(18, 5))\n\n# by default librosa.load returns a sample rate of 22050\n# librosa converts input to mono, hence always \ndata, sample_rate = librosa.load(filename)\nlibrosa.display.waveplot(data, sr=sample_rate)\nprint(\"Sample Rate: \", sample_rate)\nipd.Audio(filename)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Spectrogram\nA spectrogram is a visual representation of the spectrum of frequencies of a signal as it varies with time.","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(18, 5))\nX = librosa.stft(data)\nXdb = librosa.amplitude_to_db(abs(X))\nlibrosa.display.specshow(Xdb, sr=sample_rate, x_axis='time', y_axis='hz')\nplt.colorbar()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Spectral Centroid\nThe spectral centroid is a measure used in digital signal processing to characterise a spectrum. It indicates where the center of mass of the spectrum is located. ","metadata":{}},{"cell_type":"code","source":"spectral_centroids = librosa.feature.spectral_centroid(data, sr=sample_rate)[0]\nplt.figure(figsize=(25, 9))\nframes = range(len(spectral_centroids))\nt = librosa.frames_to_time(frames)\n\n# Normalising the spectral centroid for visualisation\ndef normalize(x, axis=0):\n    return sklearn.preprocessing.minmax_scale(x, axis=axis)\n\n#Plotting the Spectral Centroid along the waveform\nlibrosa.display.waveplot(data, sr=sample_rate, alpha=0.4)\nplt.plot(t, normalize(spectral_centroids), color='b')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Spectral Rolloff\nIt is a measure of the shape of the signal. It represents the frequency at which high frequencies decline to 0.","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(25, 9))\nspectral_rolloff = librosa.feature.spectral_rolloff(data+0.01, sr=sample_rate)[0]\nlibrosa.display.waveplot(data, sr=sample_rate, alpha=0.4)\nplt.plot(t, normalize(spectral_rolloff), color='r')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Spectral bandwidth\nThe spectral bandwidth is defined as the width of the band of light at one-half the peak maximum (or full width at half maximum [FWHM]) and is represented by the two vertical red lines and λSB on the wavelength axis.","metadata":{}},{"cell_type":"code","source":"spectral_bandwidth_2 = librosa.feature.spectral_bandwidth(data+0.01, sr=sample_rate)[0]\nspectral_bandwidth_3 = librosa.feature.spectral_bandwidth(data+0.01, sr=sample_rate, p=3)[0]\nspectral_bandwidth_4 = librosa.feature.spectral_bandwidth(data+0.01, sr=sample_rate, p=4)[0]\nplt.figure(figsize=(25, 9))\nlibrosa.display.waveplot(data, sr=sample_rate, alpha=0.4)\nplt.plot(t, normalize(spectral_bandwidth_2), color='r')\nplt.plot(t, normalize(spectral_bandwidth_3), color='g')\nplt.plot(t, normalize(spectral_bandwidth_4), color='y')\nplt.legend(('p = 2', 'p = 3', 'p = 4'))  # p: order of spectral bandwidth","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Zero-Crosing Rate\nThe zero-crossing rate (ZCR) is the rate at which a signal changes from positive to zero to negative or from negative to zero to positive.","metadata":{}},{"cell_type":"code","source":"#Plot the signal:\nplt.figure(figsize=(25, 9))\n# librosa.display.waveplot(data, sr=sample_rate)\n# Zooming in\nn0 = 9000\nn1 = 9100\n\nplt.plot(data[n0:n1])\nplt.grid()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"zero_crossings = librosa.zero_crossings(data[n0:n1], pad=False)\nprint(sum(zero_crossings)) #57","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are 31 zero crossing.","metadata":{}},{"cell_type":"markdown","source":"# Mel-Frequency Cepstral Coefficients (MFCCs)\nThe Mel frequency cepstral coefficients (MFCCs) of a signal are a small set of features (usually about 10–20) which concisely describe the overall shape of a spectral envelope.","metadata":{}},{"cell_type":"code","source":"mfccs = librosa.feature.mfcc(data, sr=sample_rate)\n\n#Displaying  the MFCCs:\nplt.figure(figsize=(15, 7))\nlibrosa.display.specshow(mfccs, sr=sample_rate, x_axis='time')\nplt.colorbar()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Chrome features\nA chroma feature or vector is typically a 12-element feature vector indicating how much energy of each pitch class, {C, C#, D, D#, E, …, B}, is present in the signal.","metadata":{}},{"cell_type":"code","source":"hop_length=512\nchromagram = librosa.feature.chroma_stft(data, sr=sample_rate, hop_length=hop_length)\nplt.figure(figsize=(20, 8))\nlibrosa.display.specshow(chromagram, x_axis='time', y_axis='chroma', hop_length=hop_length, cmap='coolwarm')\nplt.colorbar()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"There are total {} species\".format(train_meta['primary_label'].nunique()))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Top 100","metadata":{}},{"cell_type":"code","source":"def plotbar(series, pal):\n    plt.figure(figsize=(20, 9))\n    chart = sns.barplot(x=series.index, y=series.values, edgecolor=(0,0,0), linewidth=2, palette=(pal))\n    chart.set_xticklabels(chart.get_xticklabels(), rotation=45)\nspecies = train_meta['primary_label'].value_counts()[:100]\nplotbar(species, \"Blues_r\") # series, palette","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set(rc={'figure.figsize':(20,6)})\nsns.countplot(x='rating', data=train_meta, edgecolor=(0,0,0), linewidth=2, palette=('cubehelix'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are only few files with low ratings","metadata":{}},{"cell_type":"code","source":"authors = train_meta['author'].value_counts()[:10]\nplotbar(authors, \"YlOrBr_r\") # series, palette","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Richard E. Webster is author having maximum file entries.","metadata":{}},{"cell_type":"markdown","source":"# Top 100 training samples per species","metadata":{}},{"cell_type":"code","source":"print(\"Common Name\")\ncommon = train_meta['common_name'].value_counts()[:100]\nplotbar(authors, \"light:b_r\") # series, palette","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Scientific Name- Top-50\")\nscien = train_meta['scientific_name'].value_counts()[:50]\nplotbar(scien, \"Greens_r\") # series, palette","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filename = '../input/birdclef-2021/train_short_audio/acafly/XC11209.ogg'\nplt.figure(figsize=(18, 5))\n\n# by default librosa.load returns a sample rate of 22050\n# librosa converts input to mono, hence always \nsig, sample_rate = librosa.load(filename)\nlibrosa.display.waveplot(data, sr=sample_rate)\nprint(\"Sample Rate: \", sample_rate)\nipd.Audio(filename)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In the past that convolutional neural networks (CNN) perform particularly well for sound classification. But CNN need 2D inputs. Luckily, we can transform an audio signal into a 2D representation: a so-called spectrogram.","metadata":{}},{"cell_type":"code","source":"# First, compute the spectrogram using the \"short-time Fourier transform\" (stft)\nspec = librosa.stft(sig)\n\n# Scale the amplitudes according to the decibel scale\nspec_db = librosa.amplitude_to_db(spec, ref=np.max)\n\n# Plot the spectrogram\nplt.figure(figsize=(15, 5))\nlibrosa.display.specshow(spec_db, \n                         sr=32000, \n                         x_axis='time', \n                         y_axis='hz', \n                         cmap=plt.get_cmap('viridis'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('SPEC SHAPE:', spec_db.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"train_labels['audio_id'].unique()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels.groupby(by=['audio_id']).count()['birds'][:4]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('original label:', train_labels.loc[458, 'birds'])\nprint('split into list:', train_labels.loc[458, 'birds'].split(' '))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = []\nfor row in train_labels.index:\n    labels.extend(train_labels.loc[row, 'birds'].split(' '))\nlabels = list(set(labels))\n\nprint('Number of unique bird labels:', len(labels))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_labels_train = pd.DataFrame(index=train_labels.index, columns=labels)\nfor row in train_labels.index:\n    birds = train_labels.loc[row, 'birds'].split(' ')\n    for bird in birds:\n        df_labels_train.loc[row, bird] = 1\ndf_labels_train.fillna(0, inplace=True)\n\n# We set a dummy value for the target label in the test data because we will need for the Data Generator\ntest_data['birds'] = 'nocall'\n\ndf_labels_test = pd.DataFrame(index=test_data.index, columns=labels)\nfor row in test_data.index:\n    birds = test_data.loc[row, 'birds'].split(' ')\n    for bird in birds:\n        df_labels_test.loc[row, bird] = 1\ndf_labels_test.fillna(0, inplace=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_labels_train.sum().sort_values(ascending=False)[:10]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels = pd.concat([train_labels, df_labels_train], axis=1)\ntest_data = pd.concat([test_data, df_labels_test], axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file = os.listdir(path+'train_soundscapes')[0]\nfile","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data, samplerate = read_ogg_file(path+'train_soundscapes/', file)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"audio_id = file.split('_')[0]\nsite = file.split('_')[1]\nprint('audio_id:', audio_id, ', site:', site)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels[(train_labels['audio_id']==int(audio_id)) & (train_labels['site']==site) & (train_labels['birds']!='nocall')]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_data = data[int(455/5)*160000:int(460/5)*160000]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(14, 5))\nlibrosa.display.waveplot(sub_data, sr=samplerate)\nplt.grid()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display.Audio(sub_data, rate=samplerate)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_lenght = 160000\naudio_lenght = 5\nnum_labels = len(labels)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 16","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_IDs_train, list_IDs_val = train_test_split(list(train_labels.index), test_size=0.33, random_state=2021)\nlist_IDs_test = list(samp_subm.index)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DataGenerator(Sequence):\n    def __init__(self, path, list_IDs, data, batch_size):\n        self.path = path\n        self.list_IDs = list_IDs\n        self.data = data\n        self.batch_size = batch_size\n        self.indexes = np.arange(len(self.list_IDs))\n        \n    def __len__(self):\n        len_ = int(len(self.list_IDs)/self.batch_size)\n        if len_*self.batch_size < len(self.list_IDs):\n            len_ += 1\n        return len_\n    \n    def __getitem__(self, index):\n        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n        list_IDs_temp = [self.list_IDs[k] for k in indexes]\n        X, y = self.__data_generation(list_IDs_temp)\n        X = X.reshape((self.batch_size, 100, 1600//2))\n        return X, y\n    \n    def __data_generation(self, list_IDs_temp):\n        X = np.zeros((self.batch_size, data_lenght//2))\n        y = np.zeros((self.batch_size, num_labels))\n        for i, ID in enumerate(list_IDs_temp):\n            prefix = str(self.data.loc[ID, 'audio_id'])+'_'+self.data.loc[ID, 'site']\n            file_list = [s for s in os.listdir(self.path) if prefix in s]\n            if len(file_list) == 0:\n                # Dummy for missing test audio files\n                audio_file_fft = np.zeros((data_lenght//2))\n            else:\n                file = file_list[0]#[s for s in os.listdir(self.path) if prefix in s][0]\n                audio_file, audio_sr = read_ogg_file(self.path, file)\n                audio_file = audio_file[int((self.data.loc[ID, 'seconds']-5)/audio_lenght)*data_lenght:int(self.data.loc[ID, 'seconds']/audio_lenght)*data_lenght]\n                audio_file_fft = np.abs(np.fft.fft(audio_file)[: len(audio_file)//2])\n                # scale data\n                audio_file_fft = (audio_file_fft-audio_file_fft.mean())/audio_file_fft.std()\n            X[i, ] = audio_file_fft\n            y[i, ] = self.data.loc[ID, self.data.columns[5:]].values\n        return X, y","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_generator = DataGenerator(path+'train_soundscapes/', list_IDs_train, train_labels, batch_size)\nval_generator = DataGenerator(path+'train_soundscapes/', list_IDs_val, train_labels, batch_size)\ntest_generator = DataGenerator(path+'test_soundscapes/', list_IDs_test, test_data, batch_size)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 2\nlernrate = 1e-3","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_labels","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model 1 - FFT based 1D CNN Neural Network","metadata":{}},{"cell_type":"code","source":"model = Sequential()\nmodel.add(Conv1D(128, input_shape=(100, 1600//2,), kernel_size=5, strides=4, activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(MaxPool1D(pool_size=(4)))\nmodel.add(Conv1D(64, kernel_size=3, activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(Flatten())\nmodel.add(Dense(256, activation='relu'))\nmodel.add(Dense(num_labels, activation='sigmoid'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer = Adam(lr=lernrate),\n              loss='binary_crossentropy',\n              metrics=['binary_accuracy'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit_generator(generator=train_generator, validation_data=val_generator, epochs = epochs, workers=4)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axs = plt.subplots(1, 2, figsize=(16, 4))\nfig.subplots_adjust(hspace = .2, wspace=.2)\naxs = axs.ravel()\nloss = history.history['loss']\nloss_val = history.history['val_loss']\nepochs = range(1, len(loss)+1)\naxs[0].plot(epochs, loss, 'bo', label='loss_train')\naxs[0].plot(epochs, loss_val, 'ro', label='loss_val')\naxs[0].set_title('Value of the loss function')\naxs[0].set_xlabel('epochs')\naxs[0].set_ylabel('value of the loss function')\naxs[0].legend()\naxs[0].grid()\nacc = history.history['binary_accuracy']\nacc_val = history.history['val_binary_accuracy']\naxs[1].plot(epochs, acc, 'bo', label='accuracy_train')\naxs[1].plot(epochs, acc_val, 'ro', label='accuracy_val')\naxs[1].set_title('Accuracy')\naxs[1].set_xlabel('Epochs')\naxs[1].set_ylabel('Value of accuracy')\naxs[1].legend()\naxs[1].grid()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = model.predict_generator(test_generator, verbose=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test = np.where(y_pred > 0.5, 1, 0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for row in samp_subm.index:\n    string = ''\n    for col in range(len(y_test[row])):\n        if y_test[row][col] == 1:\n            if string == '':\n                string += labels[col]\n            else:\n                string += ' ' + labels[col]\n    if string == '':\n        string = 'nocall'\n    samp_subm.loc[row, 'birds'] = string","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output = samp_subm\noutput.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model 2 - Spectrums based 2D CNN","metadata":{}},{"cell_type":"code","source":"import os\nimport warnings\nimport shutil\nwarnings.filterwarnings(action='ignore')\n\nimport math\nimport pandas as pd\nimport librosa\nimport librosa.display\nimport numpy as np\nimport seaborn as sns; sns.set(style='whitegrid')\nimport matplotlib.pyplot as plt\n\nfrom sklearn.utils import shuffle\nfrom PIL import Image\nfrom tqdm import tqdm,tnrange,tqdm_notebook\nimport tensorflow as tf\nfrom tqdm.keras import TqdmCallback\nfrom keras.callbacks import ReduceLROnPlateau,EarlyStopping,ModelCheckpoint\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator \nfrom tensorflow.keras import applications as app\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten,AveragePooling2D\nfrom tensorflow.keras.layers import Dense,BatchNormalization,Dropout \nfrom tensorflow.keras.models import Sequential \nfrom tensorflow.keras.applications import EfficientNetB4, ResNet50,ResNet101, VGG16, MobileNet, InceptionV3","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Global Coefficients that can be modified\nclass coefs:\n    \n    # Generate Subset\n    rat_id = 4 # rating subset limiter \n    recs = 200 # each specie must have X recodings\n    max_files = 1500 # general last limit for rows\n    thresh = 0.25 # label probability selection threshold\n    submission = True # For Submission Only (Less Inference Output)\n    \n    # Global vars\n    seed = 1337\n    sr = 32000        # librosa sample rate input\n    sl = 5 # seconds   \n    sshape = (48,128) # height x width\n    fmin = 500      # spectrum min frequency\n    fmax = 12500    # spectrum max frequency\n    n_epoch = 100   # training epochs\n    cutoff = 15     # 3 sample spectogram (training) overwritten for inference\n\npath_switch = False\n\n# Helper Functions Stored Below","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot Keras Training History\ndef HistPlot():\n\n    fig,ax = plt.subplots(1,2,figsize=(12,4))\n    sns.despine(top=True,left=True,bottom=True)\n\n    ax[0].plot(history.history['accuracy'])\n    ax[0].plot(history.history['val_accuracy'])\n    ax[0].set_title('model accuracy')\n    ax[0].set_ylabel('accuracy')\n    ax[0].set_xlabel('epoch')\n    ax[0].grid(True,linestyle='--',alpha=0.5)\n    \n    ax[1].plot(history.history['loss'])\n    ax[1].plot(history.history['val_loss'])\n    ax[1].set_title('model loss')\n    ax[1].set_ylabel('loss')\n    ax[1].set_xlabel('epoch')\n    ax[1].legend(['train', 'test'], loc='upper left')\n    ax[1].grid(True,linestyle='--',alpha=0.5)\n    plt.show()\n\n# Split the Input signal into segments\ndef split_signal(sig):\n    sig_splits = []\n    for i in range(0, len(sig), int(coefs.sl * coefs.sr)):\n        split = sig[i:i + int(coefs.sl * coefs.sr)]\n        if len(split) < int(coefs.sl * coefs.sr):\n            break\n        sig_splits.append(split)\n    \n    return sig_splits\n\n# extracts spectrograms and saves them in a working directory\ndef get_spectrograms(filepath, primary_label, output_dir):\n\n    # duration is set from global variable\n    sig, rate = librosa.load(filepath, sr=coefs.sr, offset=None, duration=coefs.cutoff)\n    sig_splits = split_signal(sig) # split the signal into parts\n    \n    # Extract mel spectrograms for each audio chunk\n    s_cnt = 0\n    saved_samples = []\n    for chunk in sig_splits:\n        \n        hop_length = int(coefs.sl * coefs.sr / (coefs.sshape[1] - 1))\n        mel_spec = librosa.feature.melspectrogram(y=chunk, \n                                                  sr=coefs.sr, \n                                                  n_fft=1024, \n                                                  hop_length=hop_length, \n                                                  n_mels=coefs.sshape[0], \n                                                  fmin=coefs.fmin, \n                                                  fmax=coefs.fmax)\n    \n        mel_spec = librosa.power_to_db(mel_spec**2, ref=np.max) \n        \n        # Normalize\n        mel_spec -= mel_spec.min()\n        mel_spec /= mel_spec.max()\n        \n        # Save as image file\n        save_dir = os.path.join(output_dir, primary_label)\n        if not os.path.exists(save_dir):\n            os.makedirs(save_dir)\n        save_path = os.path.join(save_dir, filepath.rsplit(os.sep, 1)[-1].rsplit('.', 1)[0] + \n                                 '_' + str(s_cnt) + '.png')\n        im = Image.fromarray(mel_spec * 255.0).convert(\"L\")\n        im.save(save_path)\n        \n        saved_samples.append(save_path)\n        s_cnt += 1\n        \n    return saved_samples\n\n# https://stackoverflow.com/questions/1524126/how-to-print-a-list-more-nicely\ndef list_columns(obj, cols=4, columnwise=True, gap=4):\n    sobj = [str(item) for item in obj]\n    if cols > len(sobj): cols = len(sobj)\n    max_len = max([len(item) for item in sobj])\n    if columnwise: cols = int(math.ceil(float(len(sobj)) / float(cols)))\n    plist = [sobj[i: i+cols] for i in range(0, len(sobj), cols)]\n    if columnwise:\n        if not len(plist[-1]) == cols:\n            plist[-1].extend(['']*(len(sobj) - len(plist[-1])))\n        plist = zip(*plist)\n    printer = '\\n'.join([\n        ''.join([c.ljust(max_len + gap) for c in p])\n        for p in plist])\n    print (printer)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"''' CREATE A SUBSET OF THE DATA '''\nprint('STEP 1) CREATING A SUBSET OF DATASET:\\n')\n\nif(path_switch):\n    lpath = '.\\\\train_metadata.csv'\nelse:\n    lpath = '../input/birdclef-2021/train_metadata.csv'\ntrain = pd.read_csv(lpath)\nprint(f\"[DATASET]: {train.values.shape} : LABELS {len(train.primary_label.value_counts())}\")\n\n# subset filter 1 (rating)\ntemp_str = 'rating>='+str(coefs.rat_id)\ntrain = train.query(temp_str)\nprint('\\nRATING LIMITER APPLIED:')\nprint(f'[SUBSET]: {train.values.shape} : LABELS {len(train.primary_label.value_counts())}')\n\n# subset filter 2 (number of recordings per specie)\nbirds_count = {};\na = train.primary_label.unique() \na_val = train.groupby('primary_label')['primary_label'].count().values\nfor bird_species, count in zip(a,a_val):\n    birds_count[bird_species] = count\nto_model_spec = [key for key,value in birds_count.items() if value >= coefs.recs] \n\nprint(f'\\n {coefs.recs}+ RECORDINGS ONLY BIRDS LIMITED:')\nTRAIN = train.query('primary_label in @to_model_spec')\nLABELS = sorted(TRAIN.primary_label.unique())\nprint(f'[SUBSET]: {TRAIN.values.shape} : LABELS {len(LABELS)}')\n\nprint('\\n BIRD LABELS AVAILABLE AFTER FILTER:')\nlist_columns(to_model_spec, cols=4, columnwise=True, gap=4)\n\n# subset filter 3 (max audio files)\n\n# Shuffle the training data and limit the number of audio files to max_files\nprint('\\nLIMITING AUDIO FILES ...')\nTRAIN = shuffle(TRAIN, random_state=coefs.seed)[:coefs.max_files]\nLABELS = sorted(TRAIN.primary_label.unique())\nprint(f'[SUBSET]: {TRAIN.values.shape} : LABELS {len(LABELS)}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"''' CREATE & OUTPUT SPECTOGRAMS FOR TRAINING'''\n# we will use CNN approach \n\n# Parse audio files and extract training samples\nif(path_switch):\n    input_dir = '.\\\\train_short_audio\\\\'\n    output_dir = '.\\\\working\\\\melspectrogram_dataset\\\\'\nelse:\n    input_dir = '../input/birdclef-2021/train_short_audio/'\n    output_dir = './/working/melspectrogram_dataset/'\n\nsamples = []\nwith tqdm_notebook(total=len(TRAIN)) as pbar:\n    for idx, row in TRAIN.iterrows():\n        pbar.update(1)\n        \n        if row.primary_label in to_model_spec:\n            audio_file_path = os.path.join(input_dir, row.primary_label, row.filename)\n            samples += get_spectrograms(audio_file_path, row.primary_label, output_dir)\n            \nTRAIN_SPECS = shuffle(samples, random_state=coefs.seed)\nprint('SUCCESSFULLY EXTRACTED {} SPECTROGRAMS'.format(len(TRAIN_SPECS)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"''' DATALOADERS '''\n# Create Data Generators/Loader for Keras, images not to be deleted\n\ntrain_folder = './working/melspectrogram_dataset/'\nvalid_datagen = ImageDataGenerator(rescale=1./255, validation_split=0.2)\ntrain_datagen = ImageDataGenerator(rescale=1./255, validation_split=0.2,\\\n                                   shear_range=10,fill_mode='nearest')\n\ntrain_generator = train_datagen.flow_from_directory(train_folder, \n                        target_size=(coefs.sshape[0],coefs.sshape[1]),  # target size\n                        batch_size=32, \n                        seed=42,\n                        subset = \"training\",\n                        class_mode='categorical')    # batch size\nvalidation_generator = valid_datagen.flow_from_directory(train_folder, \n                        target_size=(coefs.sshape[0],coefs.sshape[1]),  # target size\n                        batch_size=32, \n                        seed=42,\n                        subset = \"validation\",\n                        class_mode='categorical')    # batch size","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.random.set_seed(coefs.seed)\nmodel = tf.keras.Sequential([\n    \n    # First conv block\n    tf.keras.layers.Conv2D(16, (3, 3), activation='relu', \n                           input_shape=(coefs.sshape[0], coefs.sshape[1],3)),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.MaxPooling2D((2, 2)),\n    \n    # Second conv block\n    tf.keras.layers.Conv2D(32, (3, 3), activation='relu'),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.MaxPooling2D((2, 2)), \n    \n    # Third conv block\n    tf.keras.layers.Conv2D(64, (3, 3), activation='relu'),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.MaxPooling2D((2, 2)), \n    \n    # Fourth conv block\n    tf.keras.layers.Conv2D(128, (3, 3), activation='relu'),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.MaxPooling2D((2, 2)),\n    \n    # Global pooling instead of flatten()\n    tf.keras.layers.GlobalAveragePooling2D(), \n    \n    # Dense block\n    tf.keras.layers.Dense(256, activation='relu'),   \n    tf.keras.layers.Dropout(0.5),  \n    tf.keras.layers.Dense(256, activation='relu'),   \n    tf.keras.layers.Dropout(0.5),\n    \n    # Classification layer\n    tf.keras.layers.Dense(len(LABELS), activation='softmax')\n])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transfer Learning","metadata":{}},{"cell_type":"code","source":"tf.random.set_seed(coefs.seed)\ndef pretrained_model(head_id):\n\n    # Define model with different applications\n    model = Sequential()\n\n    ''' Define Head Pretrained Models '''\n\n    if(head_id is 'vgg'):\n        model.add(VGG16(input_shape=(coefs.sshape[0],coefs.sshape[1],3),\n                            pooling='avg',\n                            classes=1000,\n                            include_top=False,\n                            weights='imagenet'))\n\n    elif(head_id is 'resnet'):\n        model.add(ResNet101(include_top=False,\n                               input_tensor=None,\n                               input_shape=(coefs.sshape[0],coefs.sshape[1],3),\n                               pooling='avg',\n                               classes=100,\n                               weights='imagenet'))\n\n    elif(head_id is 'mobilenet'):\n        model.add(MobileNet(alpha=1.0,\n                               depth_multiplier=1,\n                               dropout=0.001,\n                               include_top=False,\n                               weights=\"imagenet\",\n                               input_tensor=None,\n                               input_shape = (coefs.sshape[0],coefs.sshape[1],3),\n                               pooling=None,\n                               classes=1000))\n\n    elif(head_id is 'inception'):\n        # 75x75\n        model.add(InceptionV3(input_shape = (coefs.sshape[0],coefs.sshape[1],3), \n                                                    include_top = False, \n                                                    weights = 'imagenet'))\n\n    elif(head_id is 'efficientnet'):\n        model.add(EfficientNetB4(input_shape = (coefs.sshape[0],coefs.sshape[1],3), \n                                    include_top = False, \n                                    weights = 'imagenet'))\n\n    ''' Tail Model Part '''\n    model.add(Flatten())\n    model.add(Dense(1024,activation='relu'))\n    model.add(Dropout(0.01))\n    model.add(Dense(len(LABELS),activation='softmax'))\n\n    # # freeze main model coefficients\n    model.layers[0].trainable = False\n    model.summary()\n\n    return model\n\n# Select & Comment out above cell if used\nmodel = pretrained_model('resnet') # define the model\n# model = tf.keras.models.load_model('../input/birdclef-resnet101-1/best_model.h5') # Reload your model ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Compile the model and specify optimizer, loss and metric\nmodel.compile(optimizer=tf.keras.optimizers.Adam(lr=0.001),\n              loss=tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.01),\n              metrics=['accuracy'])\n\n# Add callbacks to reduce the learning rate if needed, early stopping, and checkpoint saving\ncallbacks = [ReduceLROnPlateau(monitor='val_loss',patience=50,verbose=1,factor=0.5),\n             EarlyStopping(monitor='val_loss',verbose=1,patience=5),\n             ModelCheckpoint(filepath='best_model.h5',monitor='val_loss',verbose=0,save_best_only=True),\n             TqdmCallback(verbose=0)\n            ]\nmodel.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(train_generator,\n                    validation_data = validation_generator,\n                    verbose = 0,\n                    callbacks=callbacks,\n                    epochs=coefs.n_epoch)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plots","metadata":{}},{"cell_type":"code","source":"# Plot Training \nHistPlot() # plot accuracy metric & loss function","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"''' I. HELPER FUNCTIONS'''\n# Define Path for Training & Test Soundscape Data\n# If competition reruns notebook, test data folder will be loaded\n\n# Print Available Soundscape Files\ndef print_available(verbose = False):\n    \n    def list_files(path):\n        return [os.path.join(path, f) for f in os.listdir(path) if f.rsplit('.', 1)[-1] in ['ogg']]\n    test_audio = list_files('../input/birdclef-2021/test_soundscapes')\n    if len(test_audio) == 0:\n        test_audio = list_files('../input/birdclef-2021/train_soundscapes')\n\n    if(verbose):\n        print('AVAILABLE SOUNDSCAPES:')\n        print('{} FILES IN TEST SET.'.format(len(test_audio)))\n        print('')\n    \n        ii=-1\n        for i in test_audio:\n            ii+=1\n            print(ii,i)\n        \n    return test_audio\n\n# Get the labels that will be used to train the model\ndef print_label_species():\n    list_columns(to_model_spec, cols=4, columnwise=True, gap=2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submissions","metadata":{}},{"cell_type":"code","source":"pd.set_option('display.max_rows', None)\n# Function to evaluate inference on given recording\ndef soundscape_records(path,model,submission=False):\n    \n    # before prediction clear read folder if it exists\n    if(os.path.exists('.//working/mel_soundscape/')):\n        shutil.rmtree('.//working/mel_soundscape/')\n#         os.listdir('.//working//')\n\n    # General Output / Submission DataFrame Structure  \n    if(submission is False):\n        data = {'row_id': [], 'prediction': [], 'score': []}\n    else:\n        data = {'row_id': [], 'birds': []}\n    \n    print('*** READING NEW FILE & STARTING PREDICTION... ***')\n    print(f'Reading File: {path}')\n    \n    ''' 1. CREATE SOUNDSCAPE FILES AND SAVE THEM '''\n    # for each spectogram input, call get_spectogram, which cuts the entire\n    # soundscape into chunks of 5 seconds\n    \n    coefs.cutoff = 600 # change to get all 5s segments in soundscape; should make 120 files\n    get_spectrograms(path,'soundscape','.//working/mel_soundscape/')\n#     print(len(os.listdir('.//working/mel_soundscape/soundscape'))) # should be 120\n    \n    ''' 2. LOAD IMAGE FILES & DATALOADER '''\n    # soundscape recording folder\n    soundscape_folder = './/working/mel_soundscape/'   \n    # image augmentation & generate dataloader\n    gen_datagen = ImageDataGenerator(rescale=1./255)\n    gen_test = gen_datagen.flow_from_directory(soundscape_folder,\n                        target_size=(coefs.sshape[0],coefs.sshape[1]),\n                        batch_size=32,\n                        class_mode='categorical')\n    \n    ''' 3. MAKE MODEL PREDICTION '''\n    # for each class predict probability for all images simulaneously\n    scores = model.predict(gen_test, verbose=1)\n    \n    # For each soundcape -> create X chunks + predict each \n    \n    time_id=0\n    for i in range(len(scores)):\n    \n        time_id+=5 # update segment time interval \n        idx = scores[i].argmax()      # possibly not best choice\n        species = LABELS[idx]\n        score = scores[i][idx]\n        \n        data['row_id'].append(path.split(os.sep)[-1].rsplit('_', 1)[0] + '_' + str(time_id))\n        \n        ''' *DECIDE IF PREDICTION PROBABILITY SHOULD EXCEED THRESHOLD '''\n        if score > coefs.thresh:\n            if(submission is False):\n                data['prediction'].append(species)\n            else:\n                data['birds'].append(species)\n        else:\n            if(submission is False):\n                data['prediction'].append('nocall')\n            else:\n                data['birds'].append('nocall')\n        \n         # store score\n        if(submission is False):\n            data['score'].append(score) # Add the confidence score as well\n        \n    # COMBINE & SHOW RESULTS\n    if(submission is False):\n        results = pd.DataFrame(data, columns = ['row_id', 'prediction', 'score'])\n    else:\n        results = pd.DataFrame(data, columns = ['row_id', 'birds'])\n    \n    if(submission is False):\n        gt = pd.read_csv('../input/birdclef-2021/train_soundscape_labels.csv')\n        results = pd.merge(gt, results, on='row_id') # merge only at available rows\n        results['outcome'] = results.birds == results.prediction\n        intersection_set = list(set(LABELS) & set(results.birds.to_list()))\n\n        print('1A. Before Prediction:')\n        list_columns(LABELS, cols=8, columnwise=True, gap=4)\n        print('1B. Birds Present')\n        list_columns(results.birds.unique())\n        print(f'bird overlap: {len(intersection_set)}/{len(results.birds.unique())} are even present')\n\n        print('\\n 2. All Predictions:')\n        print(results.outcome.value_counts())\n        print('')\n\n        print('3. Bird Predictions Only:')\n        df_bird = results[results.birds!='nocall']\n        print(df_bird.outcome.value_counts())\n        print('\\n\\n')\n        return 0\n    else:\n        return results # return one soundscape inference\n    \n''' MAIN INFERENCE OPTIONS '''\n# Use model to evaluate on soundscape segments in one or many files\n    \n# A. Get Pathways to Soundscapes \ntest_audio = print_available()\n    \n# B. Inference on all Soundscape, tlist stores all soundscape individual results\nii=-1;tlist = []\nfor i in test_audio:\n#     model = tf.keras.models.load_model('best_model.h5') # load external model\n    ii+=1;df_infer = soundscape_records(test_audio[ii],model,coefs.submission)\n    if(coefs.submission):\n        tlist.append(df_infer)\n\n# combine all soundscape inference results\nif(coefs.submission):    \n    df_allres = pd.concat(tlist)\n    df_allres.to_csv(\"submission1.csv\", index=False)\n    \n# C. Inference for one soundscape\n# model = tf.keras.models.load_model('best_model.h5')\n# soundscape_records(test_audio[0],model,coefs.submission)\n\n# Remove Training Spectrums to not show them in output\nshutil.rmtree('.//working/melspectrogram_dataset/')\nos.listdir('.//working//')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# NOTE: \n## If you find this notebook useful, do give me an upvote, it helps to keep up my motivation.","metadata":{}}]}