{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%matplotlib inline\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\npd.set_option('display.max_columns', None)\npd.set_option('display.max_colwidth', None)\n\nfrom glob import glob\nimport gc\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom IPython.display import display\n\nfrom tqdm import tqdm, tqdm_notebook\ntqdm.pandas()\n\nfrom sklearn.model_selection import train_test_split\nimport librosa\nimport librosa.display\nfrom IPython.display import Audio\nimport wave\nfrom scipy.io import wavfile\n\nimport plotly.express as px #Plotly Express\n\nfrom plotly.offline import iplot\n#to link plotly to pandas\nimport cufflinks as cf\ncf.go_offline()\ncf.set_config_file(offline = False, world_readable = True)\n\nplt.rcParams[\"figure.figsize\"] = (12, 8)\nplt.rcParams['axes.titlesize'] = 16\nplt.style.use('seaborn-whitegrid')\nsns.set_palette('Set3')\n\nimport tensorflow as tf\n\nimport os\nprint(os.listdir('../input/birdclef-2021/'))\n\nimport warnings\nwarnings.simplefilter('ignore')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_dir = '../input/birdclef-2021/'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_meta = pd.read_csv(base_dir + 'train_metadata.csv')\nprint(train_meta.shape)\ntrain_meta.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels = pd.read_csv(base_dir + 'train_soundscape_labels.csv')\nprint(train_labels.shape)\ntrain_labels.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(data = train_labels, x = 'site');","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(data = train_meta, x = 'rating');","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels['birds'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- There are audios with more than one bird sound","metadata":{}},{"cell_type":"markdown","source":"__Loading an Audio file__","metadata":{}},{"cell_type":"code","source":"sample_audio = base_dir + 'train_short_audio/rucwar/XC133150.ogg'\n\nsignal, sr = librosa.load(sample_audio)\n\nprint(f\"Sample rate  : {sr}\")\nprint(f\"Signal Length: {len(signal)}\")\nprint(f\"Duration     : {len(signal) / sr} secs\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"signal, _ = librosa.load(sample_audio, sr = 44100, duration = 15)\n\nplt.figure(figsize = (20, 5))\nlibrosa.display.waveplot(signal)\nplt.xlabel('Time')\nplt.ylabel('Amplitude')\nplt.show()\n\nplt.figure(figsize = (20, 5))\nmels = librosa.feature.melspectrogram(y = signal, sr = 44100, n_mels = 256, fmax = 8000)\nlibrosa.display.specshow(librosa.power_to_db(mels, ref = np.max), x_axis = 'time', y_axis = 'mel')\nplt.title('Melspectrogram')\nplt.colorbar()\nplt.show()\n\n#Short-term Fourier Transform\nplt.figure(figsize = (20, 5))\nstft = librosa.stft(y = signal)\nstft_db = librosa.amplitude_to_db(stft)\nlibrosa.display.specshow(stft_db, x_axis = 'time', y_axis = 'hz')\nplt.title('Spectrogram - STFT')\nplt.colorbar()\nplt.show()\n\n#Log Frequency Axis\nplt.figure(figsize = (20, 5))\nlibrosa.display.specshow(stft_db, sr = 44100, x_axis = 'time', y_axis = 'log')\nplt.colorbar()\nplt.title('Log Frequency Axis')\nplt.show()\n\nAudio(sample_audio, rate = 44100)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"__Zero Crossings by Librosa__","metadata":{}},{"cell_type":"code","source":"n0 = 10000\nn1 = 10200\n\nplt.figure(figsize = (20, 5))\nplt.plot(signal[n0: n1])\nplt.title('Zooming in the Signal')\nplt.grid()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"zero_crossings = librosa.zero_crossings(signal[n0: n1], pad = False)\nprint(zero_crossings.shape)\nprint(f\"Number of Zero crossings: {sum(zero_crossings)}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"__Recordings Count by Year__","metadata":{}},{"cell_type":"code","source":"train_meta['year'] = train_meta['date'].apply(lambda x: x.split('-')[0])\ntrain_meta['month'] = train_meta['date'].apply(lambda x: x.split('-')[1])\ntrain_meta['day'] = train_meta['date'].apply(lambda x: x.split('-')[2])\n#train_meta.head(2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- There are year with vlaues 0000, 0201, 0199 and 0202, we fix them below","metadata":{}},{"cell_type":"code","source":"train_meta['year'] = train_meta['year'].apply(lambda x: x if x[:2] in ['19', '20'] else np.nan)\ntrain_meta['year'].fillna(train_meta['year'].value_counts().index[0], inplace = True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = train_meta['year'].value_counts()\npx.bar(x = temp.index, y = temp.values, \n      title = 'Number of Recordings by Year',\n      labels = {'x': 'Year', 'y': 'Count'}\n      )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = train_meta.groupby('month')['primary_label'].count()\npx.bar(x = temp.index, y = temp.values, \n      title = 'Number of Recordings by Month',\n      labels = {'x': 'Months', 'y': 'Count'}\n      )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = pd.pivot_table(data = train_meta, index = 'primary_label', columns = 'month', values = 'secondary_labels', \n                      aggfunc = 'count')","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t = temp.T.iloc[:, :10].fillna(0)\npx.line(t, \n       title = 'Bird Recordings by Month',\n       labels = {'months': 'Months', 'value': 'Num of Recordings'}, \n    )","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t = temp.T.iloc[:, 11:21].fillna(0)\npx.line(t, \n       title = 'Bird Recordings by Month',\n       labels = {'months': 'Months', 'value': 'Num of Recordings'}, \n    )","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t = temp.T.iloc[:, 300:321].fillna(0)\npx.line(t, \n       title = 'Bird Recordings by Month',\n       labels = {'months': 'Months', 'value': 'Num of Recordings'}, \n    )","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t = temp.T.iloc[:, 322:351].fillna(0)\npx.line(t, \n       title = 'Bird Recordings by Month',\n       labels = {'months': 'Months', 'value': 'Num of Recordings'}, \n    )","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Number of birds in train_short_audio: {len(os.listdir(base_dir + 'train_short_audio/'))}\")\nprint(f\"Number of audio files in train_soundscapes: {len(os.listdir(base_dir + 'train_soundscapes/'))}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- Let's check the birds and their associated audio files","metadata":{}},{"cell_type":"code","source":"audio_path = base_dir + 'train_short_audio/'\nbirds_audio = {}\nfor bird in os.listdir(audio_path):\n    birds_audio[bird] = len(os.listdir(audio_path + bird))\nbirds_df = pd.DataFrame(birds_audio.items())\nbirds_df.columns = ['Birds', 'Num_Audio']\nbirds_df = birds_df.sort_values(by = 'Num_Audio', ascending = False)\npx.bar(birds_df, x = 'Birds', y = 'Num_Audio')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Bird Recording Location on World Map\n- Click the the location dots for the bird name","metadata":{}},{"cell_type":"code","source":"import folium\nimport branca\nimport branca.colormap as cm\n\nbird = np.random.choice(birds_df['Birds'], 1)[0]\ntemp = train_meta[['latitude', 'longitude']][train_meta['primary_label'] == bird] \n\nbird_map = folium.Map(prefer_canvas = True, zoom_start = 10)\n\nfor lat, long in zip(temp['latitude'], temp['longitude']): \n    folium.CircleMarker(location = [lat, long], \n                   radius = 1, \n                    color = 'blue',\n                        popup = bird, \n                            weight = 5).add_to(bird_map)\n\nbird = np.random.choice(birds_df['Birds'], 1)[0]\ntemp = train_meta[['latitude', 'longitude']][train_meta['primary_label'] == bird] \nfor lat, long in zip(temp['latitude'], temp['longitude']): \n    folium.CircleMarker(location = [lat, long], \n                   radius = 1, \n                    color = 'red',\n                        popup = bird,\n                            weight = 5).add_to(bird_map)\n\nbird = np.random.choice(birds_df['Birds'], 1)[0]\ntemp = train_meta[['latitude', 'longitude']][train_meta['primary_label'] == bird] \nfor lat, long in zip(temp['latitude'], temp['longitude']): \n    folium.CircleMarker(location = [lat, long], \n                   radius = 1, \n                    color = 'green',\n                        popup = bird,\n                            weight = 5).add_to(bird_map)\n\nbird = np.random.choice(birds_df['Birds'], 1)[0]\ntemp = train_meta[['latitude', 'longitude']][train_meta['primary_label'] == bird] \nfor lat, long in zip(temp['latitude'], temp['longitude']): \n    folium.CircleMarker(location = [lat, long], \n                   radius = 1, \n                    color = 'yellow',\n                        popup = bird,\n                            weight = 5).add_to(bird_map)\n    \n\nbird_map.fit_bounds(bird_map.get_bounds())\n#bird_map.add_child(folium.LatLngPopup())\n\nbird_map","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Migratory Pattern of Top Birds","metadata":{}},{"cell_type":"code","source":"bird = train_labels['birds'].value_counts().index[1]\ntemp = train_meta[['latitude', 'longitude', 'month']][train_meta['primary_label'] == bird] \n\nbird_map = folium.Map(prefer_canvas = True, zoom_start = 10)\n\ncolormap = cm.LinearColormap(colors = ['red','lightblue', 'blue'], index = [1, 12], \n                             vmin = 1, vmax = 12)\ncolormap.caption = bird.upper()\ncolormap.add_to(bird_map)\n                             \nfor mon in temp['month'].unique():\n    longlat = temp[temp['month'] == mon][['latitude', 'longitude']]\n    for loc in zip(longlat['latitude'], longlat['longitude']): \n        folium.CircleMarker(location = loc, \n                       radius = 1, \n                        color = colormap(int(mon)),\n                            popup = mon, \n                                weight = 5).add_to(bird_map)\nbird_map","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bird = train_labels['birds'].value_counts().index[2]\ntemp = train_meta[['latitude', 'longitude', 'month']][train_meta['primary_label'] == bird] \n\nbird_map = folium.Map(prefer_canvas = True, zoom_start = 10)\n\ncolormap = cm.LinearColormap(colors = ['red','lightblue', 'blue'], index = [1, 12], \n                             vmin = 1, vmax = 12)\ncolormap.caption = bird.upper()\ncolormap.add_to(bird_map)\n                             \nfor mon in temp['month'].unique():\n    longlat = temp[temp['month'] == mon][['latitude', 'longitude']]\n    for loc in zip(longlat['latitude'], longlat['longitude']): \n        folium.CircleMarker(location = loc, \n                       radius = 1, \n                        color = colormap(int(mon)),\n                            popup = mon, \n                                weight = 5).add_to(bird_map)\nbird_map","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bird = train_labels['birds'].value_counts().index[3]\ntemp = train_meta[['latitude', 'longitude', 'month']][train_meta['primary_label'] == bird] \n\nbird_map = folium.Map(prefer_canvas = True, zoom_start = 10)\n\ncolormap = cm.LinearColormap(colors = ['red','lightblue', 'blue'], index = [1, 12], \n                             vmin = 1, vmax = 12)\ncolormap.caption = bird.upper()\ncolormap.add_to(bird_map)\n                             \nfor mon in temp['month'].unique():\n    longlat = temp[temp['month'] == mon][['latitude', 'longitude']]\n    for loc in zip(longlat['latitude'], longlat['longitude']): \n        folium.CircleMarker(location = loc, \n                       radius = 1, \n                        color = colormap(int(mon)),\n                            popup = mon, \n                                weight = 5).add_to(bird_map)\nbird_map","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- Let's first consider only quality data - rating > 3.5","metadata":{}},{"cell_type":"code","source":"train = train_meta[train_meta['rating'] > 3.5]\ntrain.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top_birds = train['primary_label'].value_counts()[train['primary_label'].value_counts().values > 75].index\ntrain = train[train['primary_label'].isin(top_birds)]\ntrain.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"__Create a dataframe with primary_label, filename and filepath__","metadata":{}},{"cell_type":"code","source":"base = base_dir + 'train_short_audio/'\n\ndf = train[['primary_label', 'filename']].sample(frac = 1).reset_index(drop = True)\ndf['filepath'] = base + df['primary_label'].astype(str) + '/' + df['filename'].astype(str)\ndf.head(2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Number of classes in sample df: {df['primary_label'].nunique()}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"__Transform the labels into Multi-label__","metadata":{}},{"cell_type":"code","source":"df = pd.concat([df, pd.get_dummies(df['primary_label'])], axis = 1)\nprint(df.shape)\ndf.head(2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_birds = df['primary_label'].unique()\ntarget_birds.__len__()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"__Extract Spectrogram using Librosa__","metadata":{}},{"cell_type":"code","source":"BATCH_SIZE = 64\nsr = 32000\nlength = sr * 2\nprint(f\"Sampling Rate: {sr}\\nLength: {length}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pad_audio(data): #for length less than sr * 2\n    if len(data) >= length: return data\n    else: return np.pad(data, pad_width = (length - len(data), 0), mode = 'constant', constant_values = (0, 0))\n\ndef chop_audio(samples): #for length > than sr * 2\n    offset = np.random.randint(0, len(samples) - length)\n    return samples[offset: offset + length]\n\ndef load_mels_spec(audio):\n    signal, _ = librosa.load(audio, sr = sr, duration = 15)\n    signal, _ = librosa.effects.trim(signal)\n    signal = pad_audio(signal)\n    if len(signal) > length:\n        signal = chop_audio(signal)\n    mels = librosa.feature.melspectrogram(y = signal, sr = sr, n_mels = 256, fmin = 20, fmax = sr / 2.0)\n    mels_db = librosa.power_to_db(mels, ref = np.max)\n    return mels","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AudioDataGen(tf.keras.utils.Sequence):\n    def __init__(self, data, batch_size, shuffle = False):\n        self.data  = data\n        self.labels = self.data[target_birds]\n        self.shuffle  = shuffle\n        self.batch_size = batch_size\n        self.list_idx = self.data.index.values\n        self.on_epoch_end()\n        \n    def __len__(self):\n        return int(np.ceil(float(len(self.data)) / float(self.batch_size)))\n    \n    def __getitem__(self, index):\n        batch_idx = self.indices[index * self.batch_size:(index + 1) * self.batch_size]\n        \n        idx = [self.list_idx[k] for k in batch_idx]\n        \n        #Data   = np.zeros((len(batch_idx), self.dim, self.dim, 3), dtype = 'float32')\n        Data = []\n        Target = []\n        for i, k in enumerate(idx):\n            audio = load_mels_spec(self.data['filepath'][k])\n            \n            # assign \n            Data.append(audio)\n            Target.append(self.labels.loc[k].values)\n            \n        Data = np.expand_dims(np.array(Data), -1)\n        Target = np.array(Target)\n            \n        return Data, Target\n    \n    def on_epoch_end(self):\n        self.indices = np.arange(len(self.list_idx))\n        if self.shuffle:\n            np.random.shuffle(self.indices)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"traingen = AudioDataGen(data = df, batch_size = BATCH_SIZE)\n\nfor d, l in traingen:\n    print(d.shape)\n    print(l.shape)\n    break\n    \ndel traingen\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(tf.__version__)\nimport tensorflow.keras.layers as L\nfrom tensorflow.keras.utils import Sequence, to_categorical\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras import backend as K","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"__Let's build a simple tesorflow model__","metadata":{}},{"cell_type":"code","source":"def get_2d_model(input_shape = (256, 126, 1), learning_rate = 0.001):\n    \n    \n    inp = L.Input(shape = input_shape)\n    \n    x = L.Conv2D(96, (4,10), padding = \"same\")(inp)\n    x = L.BatchNormalization()(x)\n    x = L.Activation(\"relu\")(x)\n    x = L.MaxPool2D()(x)\n    \n    x = L.Conv2D(64, (4,10), padding = \"same\")(x)\n    x = L.BatchNormalization()(x)\n    x = L.Activation(\"relu\")(x)\n    x = L.MaxPool2D()(x)\n    \n    x = L.Conv2D(48, (4,10), padding = \"same\")(x)\n    x = L.BatchNormalization()(x)\n    x = L.Activation(\"relu\")(x)\n    x = L.MaxPool2D()(x)\n    \n    x = L.Conv2D(32, (4,10), padding = \"same\")(x)\n    x = L.BatchNormalization()(x)\n    x = L.Activation(\"relu\")(x)\n    x = L.MaxPool2D()(x)\n    x = L.Flatten()(x)\n    \n    x = L.Dropout(0.5)(x)\n    x = L.Dense(80)(x)\n    x = L.BatchNormalization()(x)\n    x = L.Activation(\"relu\")(x)\n    \n    x = L.Dense(80)(x)\n    x = L.BatchNormalization()(x)\n    x = L.Activation(\"relu\")(x)\n    \n    out = L.Dense(len(target_birds), activation = 'softmax')(x)\n\n    model = Model(inputs = inp, outputs = out)\n    opt = Adam(learning_rate)\n\n    model.compile(optimizer = opt, loss = tf.keras.losses.CategoricalCrossentropy(), \n                  metrics = ['accuracy'])\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reduce = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'val_loss', patience = 2, verbose = 1, factor = 0.5)\nearly = tf.keras.callbacks.EarlyStopping(monitor = 'val_loss', verbose = 1, patience = 5)\ncheck = tf.keras.callbacks.ModelCheckpoint(filepath = 'clef_model.h5', monitor = 'val_loss', verbose = 0, \n                                           save_best_only = True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = get_2d_model()\nmodel.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df, valid_df = train_test_split(df, test_size = 0.2, random_state = 2021)\nprint(train_df.shape, valid_df.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"STEPS_PER_EPOCH = len(train_df) // BATCH_SIZE\n\ntraingen = AudioDataGen(data = train_df, batch_size = BATCH_SIZE)\nvalidgen = AudioDataGen(data = valid_df, batch_size = BATCH_SIZE)\n\ndel train_df, valid_df\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(\n                traingen, \n                epochs = 3,\n                verbose = 1,\n                callbacks = [check, reduce, early],\n                steps_per_epoch = STEPS_PER_EPOCH,\n                validation_data = validgen\n        )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# WIP","metadata":{"_kg_hide-input":false}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}