{"cells":[{"metadata":{},"cell_type":"markdown","source":" # Overview\n Readable code to create Resnet50 model in keras with various data augmentations and noise.  Any suggestions for improvement welcome.  Models seem to have trouble getting past ~.65 val_lwlrap score."},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"pip install audiomentations","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import necessary libraries\nimport os\nimport sys\nimport pickle\nimport numpy as np\nimport pandas as pd\nimport plotly.express as px\nimport librosa, librosa.display\nimport tensorflow\nimport keras\nimport tensorflow as tf\nfrom PIL import Image\nfrom skimage.transform import resize\n#from specAugment import spec_augment_tensorflow # doesn't work with tensorflow 2.0\n#import tensorflow_io as tfio # tensorflow_io has dependency problem when using TPU\nfrom audiomentations import Compose, SpecFrequencyMask, FrequencyMask, TimeMask, AddGaussianNoise\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, BatchNormalization\nfrom tensorflow.keras import models, layers\nfrom tensorflow.keras.applications.resnet50 import ResNet50\n\n# ---------------------------\nfrom tensorflow.keras.layers import Dense,GlobalAveragePooling2D,Convolution2D,BatchNormalization\nfrom tensorflow.keras.layers import Flatten,MaxPooling2D,Dropout\nfrom tensorflow.keras.applications import DenseNet121\nfrom tensorflow.keras.applications.densenet import preprocess_input\nfrom tensorflow.keras.preprocessing import image\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator,img_to_array\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n#-----------------------------\n\nfrom sklearn.model_selection import train_test_split\nimport IPython.display as ipd\nfrom sklearn.preprocessing import OneHotEncoder,LabelBinarizer\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Process data:"},{"metadata":{},"cell_type":"markdown","source":"## Windowing data\n\nThe audio files will be processed in the following manor to give the most robust data set.  First the minimum and maximum frequencies for the data set will be found and reduced/increased by 25% to give some margin.  Next, the max time difference between t_min and t_max is found.  This max time difference is then used to calculate a step distance with margin.  With the step distance calculated, the t_min and t_max for each audio sample is used to find the center of the audio cut.  A random number between -3 and 3 is added to this center to randomly shift the labeled event left or right.  Extra logic is included if events fall to close to the beginning or end of audio file.  The final product is a dataframe with new ‘start’ and ‘end’ time stamps for each audio file.\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"# read in true positives csv\ntrain_t = pd.read_csv('../input/rfcx-species-audio-detection/train_tp.csv')\n\ndf_in = train_t\n\n# get min and max frequencies\nf_min = round(df_in.f_min.min() * .75, 1)\nf_max = round(df_in.f_max.max() * 1.25, 1)\n\n\n# create time diff column:\ndf_in['t_diff'] = df_in['t_max'] - df_in['t_min']\n\n# get max time_diff\nt_max = round(train_t['t_diff'].max(),0)\n\n# margin in seconds\nmargin = 1\n\n# calculate step length\nstep = (t_max/2) + margin\n\n# initialize start and end rows in dataframe\ndf_in['start'] = 0\ndf_in['end'] = 0\n\nfor index, row in df_in.iterrows():\n    \n    # set noise\n    #noise=0\n    noise = np.random.uniform(-3,3)\n    \n    # set window size\n    t_center = row['t_min'] + row['t_diff'] + noise\n    \n    # get start and end of window\n    start = t_center - step\n    end = t_center + step\n    \n    # special process events too close to start or end\n    if start < 0:\n        t_start = 0\n        t_end = 2*step\n        \n    else:\n        t_start = start\n        t_end = end\n        \n    if end > 60:\n        t_end = 60\n        t_start = t_end - (2*step)\n    \n    \n    # add start and end to dataframe\n    df_in['start'][index] = t_start\n    df_in['end'][index] = t_end\n\n    \n# check to make sure all windows are correct\ndf_in['temp'] = df_in['end'] - df_in['start']\ndf_in.temp.describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Split into train/test (add stratified k-fold in new version)\nSplit data into train (80%) and test (20%) and OneHot encode labels.\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"# train test split\nX_train, X_test, y_train, y_test = train_test_split(df_in[['recording_id', 'start', 'end']],\n                                                    df_in[['species_id', 'start', 'end']],\n                                                    test_size=0.20, random_state=8)\n\n# create train_df and test_df\ntrain_df = X_train\ntrain_df['label'] = y_train['species_id']\n\ntest_df = X_test\ntest_df['label'] = y_test['species_id']\n\n# Onehot encode labels\nenc = LabelBinarizer()\n\ntrain_onehot = enc.fit_transform((np.array(train_df.label)).reshape(len(train_df),1))\ntest_onehot  = enc.fit_transform((np.array(test_df.label)).reshape(len(test_df),1))\n\n# add to dataframes\ntrain_df['onehot_label'] = train_onehot.tolist()\ntest_df['onehot_label'] = test_onehot.tolist()\n\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## View class distribution in train/test split"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_bar = px.histogram(train_df, x='label', histfunc='count', opacity=0.8,\n                         template='plotly_white')#, title='Train')\ntrain_bar.update_layout(title={'text':'Training Split Class Distribution','x':0.5})\n\ntrain_bar.show()\n\ntest_bar = px.histogram(test_df, x='label', histfunc='count', opacity=0.8,\n                        template='plotly_white',nbins=24)\ntest_bar.update_layout(title={'text':'Testing Split Class Distribution','x':0.5})\n\ntest_bar.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Create spectrogram function\nSpectrogram function creates spectrogram with option to add gaussian noise, time mask, and frequency mask.  \n"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_spec(audio_path, start, end, shape, power=2.0, sr=None, third_dim=False, time_freq_mask=True):\n    '''\n    docstring here\n    '''\n    # convert shape into array\n    shape = np.array(shape)\n    \n    # load audio and get sample rate\n    loaded_audio, sample_rate = librosa.load(audio_path, sr=sr)\n    \n    # create window\n    loaded_audio = loaded_audio[(start*sample_rate):(end*sample_rate)]\n    \n    # add time and frequency masking\n    if time_freq_mask == True:\n        augment = Compose([\n            AddGaussianNoise(min_amplitude=0.001, max_amplitude=0.015, p=1),\n            TimeMask(min_band_part=0.005, max_band_part=0.10, p=1),\n            FrequencyMask(min_frequency_band=0.005, max_frequency_band=0.10, p=1)\n        ])\n        \n        loaded_audio = augment(loaded_audio, sample_rate=sample_rate)\n        \n    # create Mel spectrogram\n    stft = librosa.feature.melspectrogram(loaded_audio, sr=sample_rate, power=power,\n                                          fmin=f_min, fmax=f_max, n_mels=shape[0])\n    stft_to_db = librosa.core.amplitude_to_db(np.abs(stft))\n    \n    # reshape image size\n\n    # resize array\n    stft_to_db = resize(stft_to_db, shape)\n    \n    # normalize \n    stft_to_db = stft_to_db - np.min(stft_to_db)\n    stft_to_db = stft_to_db / np.max(stft_to_db)\n    \n    if third_dim == True:\n        stft_to_db = np.stack((stft_to_db,stft_to_db,stft_to_db))\n        \n    \n    return(stft_to_db)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Create training and testing/validation dictionary with data\n\n#### Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"# create dictionary with data and metadata\n\n# initialize empty dictionary\ndata = {'index':[],\n        'recording_id':[],\n        'spec_data':[],\n        'label':[]}\n\n# set paths\nroot_path = '../input/rfcx-species-audio-detection/train/'\nfile_ext = '.flac'\n\nn=0\n\n# specify shape for spectrograms, important parameter for rest of code\nshape = (256,512)\npower = 1.5\n\n# loop through each row in dataframe\nfor index, row in train_df.iterrows():\n    \n    # create filepath\n    f_path = root_path+row['recording_id']+file_ext\n    \n    # pass f_path to stft function and add 3rd dimension\n    temp_spec = create_spec(f_path, row['start'], row['end'], shape, power, third_dim=True, time_freq_mask=False)\n    temp_spec = temp_spec.reshape(temp_spec.shape[1], temp_spec.shape[2], temp_spec.shape[0])\n    \n    # add data to data dictionary\n    data['index'].append(index)\n    data['recording_id'].append(row['recording_id'])\n    data['spec_data'].append(temp_spec)\n    #data['label'].append(row['onehot_label'])\n    data['label'].append(np.array(row['onehot_label']))\n    \n    n+=1\n    \n    if n%100 == 0:\n        print(n,'files processed')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Test"},{"metadata":{"trusted":true},"cell_type":"code","source":"# create dictionary with data and metadata\n\n# initialize empty dictionary\nval_data = {'index':[],\n            'recording_id':[],\n            'spec_data':[],\n            'label':[]}\n\n# set counter\nn=0\n\n# loop through each row in dataframe\nfor index, row in test_df.iterrows():\n    \n    # create filepath\n    f_path = root_path+row['recording_id']+file_ext\n    \n    # pass f_path to stft function and add 3rd dimension\n    temp_spec = create_spec(f_path, row['start'], row['end'], shape,\n                            power=power, third_dim=True, time_freq_mask=False)\n    temp_spec = temp_spec.reshape(temp_spec.shape[1], temp_spec.shape[2], temp_spec.shape[0])\n    \n    # add data to data dictionary\n    val_data['index'].append(index)\n    val_data['recording_id'].append(row['recording_id'])\n    val_data['spec_data'].append(temp_spec)\n    #data['label'].append(row['onehot_label'])\n    val_data['label'].append(np.array(row['onehot_label']))\n    \n    n+=1\n    \n    if n%100 == 0:\n        print(n,'files processed')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### See example spectrogram with preprocessing"},{"metadata":{"trusted":true},"cell_type":"code","source":"# set paths\nroot_path = '../input/rfcx-species-audio-detection/train/'\nfile_ext = '.flac'\n\ni=4\n\n# create and display spectrogram\nname = root_path+df_in['recording_id'][i]+file_ext\nstart = df_in['start'][i]\nend = df_in['end'][i]\n\nspec = create_spec(name, start, end, shape=(256,512), power=1.5)\n\n# display spectrogram\nlibrosa.display.specshow(spec, sr=48000, x_axis='time', y_axis='mel')\n\nprint(spec.shape)\nprint(df_in.loc[i])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create Model:"},{"metadata":{},"cell_type":"markdown","source":"## Add keras custom scoring metric LWLRAP\ncredit: https://www.kaggle.com/carlthome/l-lrap-metric-for-tf-keras/comments"},{"metadata":{"trusted":true},"cell_type":"code","source":"# credit: https://www.kaggle.com/carlthome/l-lrap-metric-for-tf-keras/comments\n\n@tf.function\ndef _one_sample_positive_class_precisions(example):\n    y_true, y_pred = example\n\n    retrieved_classes = tf.argsort(y_pred, direction='DESCENDING')\n    class_rankings = tf.argsort(retrieved_classes)\n    retrieved_class_true = tf.gather(y_true, retrieved_classes)\n    retrieved_cumulative_hits = tf.math.cumsum(tf.cast(retrieved_class_true, tf.float32))\n\n    idx = tf.where(y_true)[:, 0]\n    i = tf.boolean_mask(class_rankings, y_true)\n    r = tf.gather(retrieved_cumulative_hits, i)\n    c = 1 + tf.cast(i, tf.float32)\n    precisions = r / c\n\n    dense = tf.scatter_nd(idx[:, None], precisions, [y_pred.shape[0]])\n    return dense\n\n\nclass LWLRAP(tf.keras.metrics.Metric):\n    def __init__(self, num_classes, name='lwlrap'):\n        super().__init__(name=name)\n\n        self._precisions = self.add_weight(\n            name='per_class_cumulative_precision',\n            shape=[num_classes],\n            initializer='zeros',\n        )\n\n        self._counts = self.add_weight(\n            name='per_class_cumulative_count',\n            shape=[num_classes],\n            initializer='zeros',\n        )\n\n    def update_state(self, y_true, y_pred, sample_weight=None):\n        precisions = tf.map_fn(\n            fn=_one_sample_positive_class_precisions,\n            elems=(y_true, y_pred),\n            dtype=(tf.float32),\n        )\n\n        increments = tf.cast(precisions > 0, tf.float32)\n        total_increments = tf.reduce_sum(increments, axis=0)\n        total_precisions = tf.reduce_sum(precisions, axis=0)\n\n        self._precisions.assign_add(total_precisions)\n        self._counts.assign_add(total_increments)        \n\n    def result(self):\n        per_class_lwlrap = self._precisions / tf.maximum(self._counts, 1.0)\n        per_class_weight = self._counts / tf.reduce_sum(self._counts)\n        overall_lwlrap = tf.reduce_sum(per_class_lwlrap * per_class_weight)\n        return overall_lwlrap\n\n    def reset_states(self):\n        self._precisions.assign(self._precisions * 0)\n        self._counts.assign(self._counts * 0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Create Resnet50 model and train"},{"metadata":{},"cell_type":"markdown","source":"#### GPU setup"},{"metadata":{"trusted":true},"cell_type":"code","source":"# gpu set up\ngpus = tf.config.experimental.list_physical_devices('GPU')\nif gpus:\n  try:\n    for gpu in gpus:\n      tf.config.experimental.set_memory_growth(gpu, True)\n  except RuntimeError as e:\n    print(e)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"image_input = keras.layers.Input(shape=(shape[0],shape[1], 3))\n\nrestnet = ResNet50(weights='imagenet', include_top=False, input_tensor=image_input)\n#restnet = InceptionResNetV2(weights='imagenet', include_top=False, input_tensor=image_input)\n\noutput = restnet.layers[-1].output\noutput = keras.layers.Flatten()(output)\nrestnet = Model(restnet.input, restnet.output)\n\nfor layer in restnet.layers:\n    layer.trainable = False\n\n\n# Add final layers for rainforest classifier\n# instantiate model\nmodel = keras.models.Sequential()\n\n# add resnet model\nmodel.add(restnet)\n#model.add(layers.Flatten())\nmodel.add(keras.layers.GlobalAveragePooling2D())\nmodel.add(keras.layers.BatchNormalization())\nmodel.add(keras.layers.Dropout(0.5))\nmodel.add(keras.layers.Dense(1024, activation='relu'))\n#     model.add(keras.layers.BatchNormalization())\n#     model.add(keras.layers.Dropout(0.4))\n#     model.add(keras.layers.Dense(512, activation='relu'))\nmodel.add(keras.layers.BatchNormalization())\nmodel.add(keras.layers.Dropout(0.5))\nmodel.add(keras.layers.Dense(24, activation='softmax'))\n\n# set optimizer\noptimizer = tf.keras.optimizers.Nadam(learning_rate=0.001)\n#optimizer = tfa.optimizers.RectifiedAdam(lr=0.002, total_steps=1152, warmup_proportion=0.3, min_lr=.000001)\n\nmodel.compile(optimizer=optimizer,\n              #loss=tfa.losses.SigmoidFocalCrossEntropy(),\n              loss=tf.keras.losses.CategoricalCrossentropy(),\n              metrics=[LWLRAP(24)])\n              #metrics=['accuracy'])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Set up callback and train model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# set model callbacks to save best model\n#filepath = \"saved-model-{epoch:02d}-{val_lwlrap:.2f}.hdf5\"\nfilepath = \"saved-model-best.hdf5\"\nmy_callbacks = [keras.callbacks.ModelCheckpoint(filepath, monitor='val_lwlrap', verbose=0,\n                                                save_best_only=True, mode='max')    \n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(np.array(data['spec_data']), np.array(data['label']), epochs=100, batch_size=96,\n                    validation_data=(np.array(val_data['spec_data']), np.array(val_data['label'])),\n                    shuffle=True, callbacks=[my_callbacks])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Show model plot"},{"metadata":{"trusted":true},"cell_type":"code","source":"# create dataframe for history\nhist_df = pd.DataFrame(history.history)\n\n# plot model history\nplot_1 = px.line(hist_df, y=['lwlrap', 'val_lwlrap'],)\nplot_1.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Generate Submission File\n1. read in test data\n2. process test data\n3. run predictions\n4. change to submission format"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_test_spec(audio, shape, sr=48000, power=2.0, third_dim=True):\n    '''\n    docstring here\n    '''\n    \n    # initialize list \n    final_list = []\n    # break audio into 10 seconds sub arrays\n    audio = np.split(audio, 10)\n\n    # loop through sub arrays\n    for sub_array in audio:\n\n        stft = librosa.feature.melspectrogram(sub_array, sr=sr, power=power, fmin=f_min, fmax=f_max, n_mels=shape[0])\n        stft_to_db = librosa.core.amplitude_to_db(np.abs(stft))\n\n        # resize array\n        stft_to_db = resize(stft_to_db, shape)\n\n        # normalize \n        stft_to_db = stft_to_db - np.min(stft_to_db)\n        stft_to_db = stft_to_db / np.max(stft_to_db)\n    \n    \n        if third_dim == True:\n            stft_to_db = np.stack((stft_to_db,stft_to_db,stft_to_db))\n        \n        # reshape output\n        stft_to_db = stft_to_db.reshape(stft_to_db.shape[1], stft_to_db.shape[2], stft_to_db.shape[0])\n        \n        # append stft_to_db to list\n        final_list.append(stft_to_db)\n    \n    return(final_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# load model\nmodel = keras.models.load_model('./saved-model-best.hdf5', compile=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# compile model\nmodel.compile(optimizer=optimizer,\n              #loss=tfa.losses.SigmoidFocalCrossEntropy(),\n              loss=tf.keras.losses.CategoricalCrossentropy(),\n              metrics=[LWLRAP(24)])\n              #metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get test headers from sample_submission.csv\ntemp = pd.read_csv('../input/rfcx-species-audio-detection/sample_submission.csv')\ncols = temp.columns.tolist()\nn=0\n\n# initialize list\nprediction_list = []\n\n# loop through test files\ntest_path = '../input/rfcx-species-audio-detection/test'\n\nfor root, dirs, files in os.walk(test_path):\n    \n    # get total number of files for progress report\n    total = len(files)\n    \n    # loop through files \n    for file in files:\n        \n        # load in audio\n        loaded_audio, sample_rate = librosa.load(test_path+'/'+file, sr=None)\n        \n        # pass loaded_audio to create_test_spec function\n        test_list = create_test_spec(loaded_audio, shape=shape, power=power)\n        \n        # get predictions\n        preds = model.predict(np.array(test_list))\n        \n        # get mean prediction probabilities\n        mean_preds = np.mean(preds, axis=0)\n        \n        # append mean probs to list\n        prediction_list.append(mean_preds)\n        \n        if n%100 == 0:\n            print(f'{n} of {total} files processed')\n        n+=1\n        \n        \n# convert final list to dataframe --> this is messy and needs to be cleaned up\nsub_df = pd.DataFrame({cols[0]:files})\ncols2 = cols[1:]\ntemp_df= pd.DataFrame(prediction_list, columns=cols2)\ntemp_df['recording_id']=files\nsub_df = temp_df[cols]\n\n# eliminate file extension\nsub_df['recording_id'] = sub_df['recording_id'].str.replace(r'.flac$','')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create submission .csv\nsub_df.to_csv('./submission.csv',index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}