{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Copied and modified from: https://www.kaggle.com/code/duythanhng/birdclef-2022-keras-simple-tutorial\n# Followed this article for LSTM setup for audio files: https://towardsdatascience.com/recurrent-neural-nets-for-audio-classification-81cb62327990\n\nimport os\nimport json \nimport librosa\nimport librosa.display\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nimport tensorflow as tf\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Dense, GlobalAveragePooling2D\nfrom tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\nfrom tensorflow.keras.optimizers import Adam, SGD, RMSprop\nfrom tensorflow.keras import optimizers\nfrom tensorflow.keras.utils import plot_model, Sequence\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.applications import MobileNetV2\nfrom tensorflow.keras.layers import Input, Concatenate","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-28T05:22:29.456094Z","iopub.execute_input":"2022-05-28T05:22:29.456929Z","iopub.status.idle":"2022-05-28T05:22:37.029513Z","shell.execute_reply.started":"2022-05-28T05:22:29.45682Z","shell.execute_reply":"2022-05-28T05:22:37.02866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_meta = pd.read_csv(\"/kaggle/input/birdclef-2022/train_metadata.csv\")\ntrain_meta = train_meta.dropna()\ntrain_meta","metadata":{"execution":{"iopub.status.busy":"2022-05-28T05:22:37.031504Z","iopub.execute_input":"2022-05-28T05:22:37.031757Z","iopub.status.idle":"2022-05-28T05:22:37.177886Z","shell.execute_reply.started":"2022-05-28T05:22:37.031722Z","shell.execute_reply":"2022-05-28T05:22:37.177194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from numpy import array\nfrom sklearn.preprocessing import OneHotEncoder, LabelEncoder\n\nlabels = list(train_meta['primary_label'].unique())\nlabel_encoder = LabelEncoder()\n\n# From: https://machinelearningmastery.com/how-to-one-hot-encode-sequence-data-in-python/#:~:text=A%20one%20hot%20encoding%20is,is%20marked%20with%20a%201.\ndef encode(labels):\n    values = array(labels)\n    integer_encoded = label_encoder.fit_transform(values)\n    # binary encode\n    onehot_encoder = OneHotEncoder(sparse=False)\n    integer_encoded = integer_encoded.reshape(len(integer_encoded), 1)\n    return onehot_encoder.fit_transform(integer_encoded)\n\nencoded_labels = encode(labels)\nencoded_labels","metadata":{"execution":{"iopub.status.busy":"2022-05-28T05:22:37.179026Z","iopub.execute_input":"2022-05-28T05:22:37.180299Z","iopub.status.idle":"2022-05-28T05:22:37.196936Z","shell.execute_reply.started":"2022-05-28T05:22:37.180255Z","shell.execute_reply":"2022-05-28T05:22:37.196006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Split audio into 5 seconds chunks","metadata":{}},{"cell_type":"code","source":"import soundfile as sf\nimport os\n\ndef cutAudio(file_path, is_save):\n    # First load the file\n    filename = file_path.replace(\"/\", \"_\")\n    file_path = \"/kaggle/input/birdclef-2022/train_audio/\" + file_path\n    audio, sr = librosa.load(file_path)\n\n    # Get number of samples for 5 seconds; replace 5 by any number\n    buffer = 5 * sr\n\n    samples_total = len(audio)\n    samples_wrote = 0\n    counter = 1\n\n    audio_split = []\n    audio_filenames = []\n    while samples_wrote < samples_total:\n        #check if the buffer is not exceeding total samples \n        if buffer > (samples_total - samples_wrote):\n            buffer = samples_total - samples_wrote\n\n        block = audio[samples_wrote : (samples_wrote + buffer)]\n        audio_split.append(block)\n\n        # Write 5 second segment\n        if is_save == True:\n            out_filename = \"/kaggle/working/each5s/split_\" + str(counter) + \"_\" + filename\n            audio_filenames.append(out_filename)\n            sf.write(out_filename, block, sr)\n        counter += 1\n        samples_wrote += buffer\n    return audio_split, sr, audio_filenames","metadata":{"execution":{"iopub.status.busy":"2022-05-28T05:22:37.199529Z","iopub.execute_input":"2022-05-28T05:22:37.199804Z","iopub.status.idle":"2022-05-28T05:22:37.20764Z","shell.execute_reply.started":"2022-05-28T05:22:37.199765Z","shell.execute_reply":"2022-05-28T05:22:37.206772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def splitTrainAudio(_df):\n    data = []\n    for index, row in _df.iterrows():\n        cutAudio(row[\"filename\"], True)\n        audio_lst, sr, filenames = cutAudio(row[\"filename\"], True)\n        for idx, y in enumerate(audio_lst):\n            data.append([row[\"primary_label\"], row[\"filename\"], filenames[idx]])\n\n    data_df = pd.DataFrame(data, columns=['primary_label', 'original_filename', 'filename'])\n    data_df.to_csv(\"/kaggle/working/data_df.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T05:22:37.209177Z","iopub.execute_input":"2022-05-28T05:22:37.209451Z","iopub.status.idle":"2022-05-28T05:22:37.218579Z","shell.execute_reply.started":"2022-05-28T05:22:37.209414Z","shell.execute_reply":"2022-05-28T05:22:37.21794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sample Data\ndata_frames = []\nfor label in labels:\n    tmp_df = train_meta[train_meta[\"primary_label\"] == label].sample(n=1, replace=True).reset_index(drop=True)\n    data_frames.append(tmp_df)\nsample_df = pd.concat(data_frames).reset_index(drop=True)\nsample_df\n\n# Uncomment for debugging.\n#sample_df = sample_df[:30]","metadata":{"execution":{"iopub.status.busy":"2022-05-28T05:22:37.220055Z","iopub.execute_input":"2022-05-28T05:22:37.220353Z","iopub.status.idle":"2022-05-28T05:22:37.698814Z","shell.execute_reply.started":"2022-05-28T05:22:37.220316Z","shell.execute_reply":"2022-05-28T05:22:37.698083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p \"/kaggle/working/each5s\"\nsplitTrainAudio(sample_df)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T05:22:37.699991Z","iopub.execute_input":"2022-05-28T05:22:37.700287Z","iopub.status.idle":"2022-05-28T05:24:02.765207Z","shell.execute_reply.started":"2022-05-28T05:22:37.70025Z","shell.execute_reply":"2022-05-28T05:24:02.764432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_df = pd.read_csv(\"/kaggle/working/data_df.csv\")\ndata_df","metadata":{"execution":{"iopub.status.busy":"2022-05-28T05:24:02.767Z","iopub.execute_input":"2022-05-28T05:24:02.767452Z","iopub.status.idle":"2022-05-28T05:24:02.784698Z","shell.execute_reply.started":"2022-05-28T05:24:02.767414Z","shell.execute_reply":"2022-05-28T05:24:02.783676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#### The full data can be found [here](https://www.kaggle.com/duythanhng/birdclef-2022-audio-per-5-second)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T05:24:02.786249Z","iopub.execute_input":"2022-05-28T05:24:02.786537Z","iopub.status.idle":"2022-05-28T05:24:02.792042Z","shell.execute_reply.started":"2022-05-28T05:24:02.786488Z","shell.execute_reply":"2022-05-28T05:24:02.789771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"from os import listdir\n\n# Padding numpy array from: https://stackoverflow.com/questions/59241216/padding-numpy-arrays-to-a-specific-size\ndef padding(array, xx, yy):\n    \"\"\"\n    :param array: numpy array\n    :param xx: desired height\n    :param yy: desirex width\n    :return: padded array\n    \"\"\"\n\n    h = array.shape[0]\n    w = array.shape[1]\n\n    a = (xx - h) // 2\n    aa = xx - a - h\n\n    b = (yy - w) // 2\n    bb = yy - b - w\n\n    return np.pad(array, pad_width=((a, aa), (b, bb)), mode='constant')\n\ndef get_bird_data():\n    paths = []\n    path_labels = []\n    for i in data_df.index:\n        paths.append(data_df['filename'][i]);\n        path_labels.append(data_df['primary_label'][i])\n    return np.array(paths), np.array(path_labels)\n\nX, y = get_bird_data()\n\nprint(len(y))","metadata":{"execution":{"iopub.status.busy":"2022-05-28T05:24:02.795363Z","iopub.execute_input":"2022-05-28T05:24:02.795651Z","iopub.status.idle":"2022-05-28T05:24:02.809261Z","shell.execute_reply.started":"2022-05-28T05:24:02.795598Z","shell.execute_reply":"2022-05-28T05:24:02.808332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.utils import plot_model, Sequence\nfrom sklearn.preprocessing import StandardScaler\nimport matplotlib.pyplot as plt\nimport librosa.display\nfrom sklearn.preprocessing import normalize\nimport warnings\nimport math\n\nhop_length = 512 #the default spacing between frames\nn_fft = 255 #number of samples\nn_mfcc = 255\nsr = 28000\ncolumns = math.ceil(sr * 5 / hop_length)\n\n# Adapted from: https://medium.datadriveninvestor.com/keras-training-on-large-datasets-3e9d9dbc09d4\n\ndef gen_audio(block, sr):\n    with warnings.catch_warnings():\n        warnings.simplefilter(\"ignore\")\n        mfcc = librosa.feature.mfcc(block, \n         n_fft=n_fft,hop_length=hop_length,n_mfcc=n_mfcc)\n    mfcc = np.nan_to_num(mfcc)\n\n    padded_mfcc = padding(mfcc, 128, columns)\n    padded_mfcc = normalize(padded_mfcc)\n\n    return np.array([padded_mfcc]).reshape(128,columns,1)\n\nclass DataGenerator(Sequence):\n    def __init__(self, paths, labels, batch_size) :\n        self.paths = paths\n        self.labels = labels\n        self.batch_size = batch_size\n\n    def __len__(self):\n        return (np.ceil(len(self.paths) / float(self.batch_size))).astype(np.int)\n\n    def __getitem__(self, idx):\n        batch_paths = self.paths[idx * self.batch_size : ((idx+1) * self.batch_size)]\n\n        batch_x = np.empty((len(batch_paths), 128, columns, 1))\n        np.empty((len(batch_paths), 128, columns, 1))\n        for i in range(len(batch_paths)):\n            block, _ = librosa.load(batch_paths[i],sr=sr)\n            batch_x[i] = gen_audio(block,sr)\n\n        batch_labels = self.labels[idx * self.batch_size : (idx+1) * self.batch_size]\n        batch_labels = list(map(lambda x: labels.index(x), batch_labels))\n\n        return batch_x, np.array(batch_labels)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T05:27:05.435592Z","iopub.execute_input":"2022-05-28T05:27:05.435976Z","iopub.status.idle":"2022-05-28T05:27:05.461318Z","shell.execute_reply.started":"2022-05-28T05:27:05.435936Z","shell.execute_reply":"2022-05-28T05:27:05.460247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TODO: Check shuffle parameter.\n#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=123, shuffle=True)\n#X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.25, random_state=123)\n\n#Print the shapes\n#X_train.shape, X_test.shape, X_val.shape, len(y_train), len(y_test), len(y_val)\n\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.30, random_state=123, shuffle=True)\nX_train.shape, X_val.shape, len(y_train), len(y_val)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T05:26:47.302188Z","iopub.execute_input":"2022-05-28T05:26:47.302885Z","iopub.status.idle":"2022-05-28T05:26:47.310853Z","shell.execute_reply.started":"2022-05-28T05:26:47.302846Z","shell.execute_reply":"2022-05-28T05:26:47.310197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fighting overfitting: https://towardsdatascience.com/dont-overfit-how-to-prevent-overfitting-in-your-deep-learning-models-63274e552323\n\nimport tensorflow\nimport tensorflow.keras as keras\nfrom keras.layers import Conv2D, LSTM, Dense, Dropout, Flatten, BatchNormalization, Activation, ConvLSTM2D, Masking, Bidirectional, MaxPooling2D\nfrom keras import backend as K\nfrom keras.regularizers import L2\n\n# TODO: Look into https://stackoverflow.com/questions/48140989/keras-lstm-input-dimension-setting\nbatch_size = 4\ndroput_rate = 0.7\nreg = 0.0001\nstrides = (2, 2)\n\nmodel = keras.Sequential()\n'''\nmodel.add(ConvLSTM2D(32, kernel_size=(3, 3), \n                     input_shape=(batch_size,128,columns,1),\n                     #strides=(3,3),\n                     dropout=0.7,\n                     kernel_regularizer=L2(0.00001), recurrent_regularizer=L2(0.00001)))\n'''\nmodel.add(Conv2D(16, (7,7), input_shape=(128, columns, 1), \n                 activation='relu', padding='same', strides=strides, kernel_regularizer=L2(reg), activity_regularizer=L2(reg)))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Dropout(droput_rate))\nmodel.add(Conv2D(32, (5,5), activation='relu', padding='same', strides=strides, kernel_regularizer=L2(reg), activity_regularizer=L2(reg)))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Dropout(droput_rate))\nmodel.add(Conv2D(64, (3,3), activation='relu', padding='same', strides=strides, kernel_regularizer=L2(reg), activity_regularizer=L2(reg)))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Dropout(droput_rate))\nmodel.add(Flatten())\nmodel.add(Dense(len(labels), activation='softmax', kernel_regularizer=L2(reg)))\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-05-28T05:26:48.826056Z","iopub.execute_input":"2022-05-28T05:26:48.826606Z","iopub.status.idle":"2022-05-28T05:26:48.896096Z","shell.execute_reply.started":"2022-05-28T05:26:48.826569Z","shell.execute_reply":"2022-05-28T05:26:48.895373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"learning_rate= 0.001\n\n# Cross entropies explained:\n# https://stackoverflow.com/questions/49161174/tensorflow-logits-and-labels-must-have-the-same-first-dimension\n# https://stackoverflow.com/questions/61742556/valueerror-shapes-none-1-and-none-2-are-incompatible\nmodel.compile(optimizer=Adam(learning_rate=learning_rate\n                             #,clipnorm=1.0\n                            ),loss='SparseCategoricalCrossentropy',metrics=['acc'])","metadata":{"execution":{"iopub.status.busy":"2022-05-28T05:26:51.305089Z","iopub.execute_input":"2022-05-28T05:26:51.305998Z","iopub.status.idle":"2022-05-28T05:26:51.317442Z","shell.execute_reply.started":"2022-05-28T05:26:51.305944Z","shell.execute_reply":"2022-05-28T05:26:51.316009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"from keras.callbacks import EarlyStopping\n\nepochs = 100\n\nes_callback = EarlyStopping(monitor='val_loss', patience=10)\n\n# TODO: Remove warning suppression.\nwith warnings.catch_warnings():\n    warnings.simplefilter(\"ignore\")\n    history = model.fit(DataGenerator(X_train, y_train, batch_size),\n                       epochs = epochs,\n                       validation_data = DataGenerator(X_val, y_val, batch_size), \n                        callbacks=[es_callback],\n                        #shuffle=True\n                       )\n\n# Record number of epochs in case of early stopping.\n# https://stackoverflow.com/questions/49852241/return-number-of-epochs-for-earlystopping-callback-in-keras\nepochs = len(history.history['loss'])","metadata":{"execution":{"iopub.status.busy":"2022-05-28T05:27:08.430671Z","iopub.execute_input":"2022-05-28T05:27:08.430929Z","iopub.status.idle":"2022-05-28T05:28:32.603527Z","shell.execute_reply.started":"2022-05-28T05:27:08.430899Z","shell.execute_reply":"2022-05-28T05:28:32.60036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n#Adapted from Deep Learning with Python by Francois Chollet, 2018\nhistory_dict=history.history\nloss_values=history_dict['loss']\nacc_values=history_dict['acc']\nval_loss_values = history_dict['val_loss']\nval_acc_values=history_dict['val_acc']\nepochs_range=range(1,epochs+1)\nfig,(ax1,ax2)=plt.subplots(1,2,figsize=(15,5))\nax1.plot(epochs_range,loss_values,'co',label='Training Loss')\nax1.plot(epochs_range,val_loss_values,'m', label='Validation Loss')\nax1.set_title('Training and validation loss')\nax1.set_xlabel('Epochs')\nax1.set_ylabel('Loss')\nax1.legend()\nax2.plot(epochs_range,acc_values,'co', label='Training accuracy')\nax2.plot(epochs_range,val_acc_values,'m',label='Validation accuracy')\nax2.set_title('Training and validation accuracy')\nax2.set_xlabel('Epochs')\nax2.set_ylabel('Accuracy')\nax2.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-28T05:24:07.021188Z","iopub.status.idle":"2022-05-28T05:24:07.021793Z","shell.execute_reply.started":"2022-05-28T05:24:07.021524Z","shell.execute_reply":"2022-05-28T05:24:07.021553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_path = \"/kaggle/input/birdclef-2022/test_soundscapes/\"\nfiles = [f.split('.')[0] for f in sorted(os.listdir(test_path))]\n\nbirds_path = \"/kaggle/input/birdclef-2022/scored_birds.json\"\nwith open(birds_path) as bf:\n    birds = json.load(bf)\n\ndata = []\nfor f in files:\n    file_path = test_path + f + '.ogg'\n    audio, sr = librosa.load(file_path)\n    # Get number of samples for 5 seconds; replace 5 by any number\n    buffer = 5 * sr\n    samples_total = len(audio)\n    samples_wrote = 0\n    counter = 1\n\n    while samples_wrote < samples_total:\n        #check if the buffer is not exceeding total samples \n        if buffer > (samples_total - samples_wrote):\n            buffer = samples_total - samples_wrote\n\n        block = audio[samples_wrote : (samples_wrote + buffer)]\n        x = gen_audio(block, sr)\n        x = x.reshape(1, 128, columns, 1)\n\n        pred = model.predict(x)\n        label_index = np.argmax(pred,axis=1)[0]\n        \n        for b in birds:\n            segment_end = counter * 5   \n            row_id = f + '_' + b + '_' + str(segment_end)\n            target = False\n            if labels[label_index] == b:\n                target = True\n            data.append([row_id, target])\n        counter += 1\n        samples_wrote += buffer\n        \nsubmission_df = pd.DataFrame(data, columns=['row_id', 'target'])\nsubmission_df","metadata":{"execution":{"iopub.status.busy":"2022-05-28T05:24:07.023009Z","iopub.status.idle":"2022-05-28T05:24:07.023603Z","shell.execute_reply.started":"2022-05-28T05:24:07.023352Z","shell.execute_reply":"2022-05-28T05:24:07.023379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv(\"/kaggle/working/submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T05:24:07.024696Z","iopub.status.idle":"2022-05-28T05:24:07.025245Z","shell.execute_reply.started":"2022-05-28T05:24:07.024997Z","shell.execute_reply":"2022-05-28T05:24:07.025022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}