{"cells":[{"metadata":{},"cell_type":"markdown","source":"# 1. Installing and importing required packages"},{"metadata":{},"cell_type":"markdown","source":"**Librosa** is a python package for music and audio analysis. \nIt provides the building blocks necessary to create music information retrieval systems."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"! pip install librosa","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"! pip install -q efficientnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Default Libraries required\nimport numpy as np\nimport pandas as pd\n\nimport math, os, re, warnings, random\nimport tensorflow as tf\n\nimport librosa\nimport librosa.display\n\nfrom kaggle_datasets import KaggleDatasets\n\n#Visualization Libraries\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n#File Accessing Libraries\nimport os\nimport os.path\nfrom os import path\n\nimport time\n\nfrom IPython.display import Audio\nimport IPython.display as ipd\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.model_selection import StratifiedKFold\n\nfrom tensorflow.keras import Model, layers\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler\nfrom tensorflow.keras.layers import GlobalAveragePooling2D, Input, Dense, Dropout, GaussianNoise\nfrom tensorflow.keras.applications import ResNet50\nimport tensorflow.keras.backend as K\n\nimport efficientnet.keras as efn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed=0):\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\nseed = 42\nseed_everything(seed)\nwarnings.filterwarnings('ignore')  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# For tf.dataset\nAUTO = tf.data.experimental.AUTOTUNE\n\n# Configuration\nEPOCHS = 10\nBATCH_SIZE = 8 * strategy.num_replicas_in_sync","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2. Data Exploration"},{"metadata":{"trusted":true},"cell_type":"code","source":"traindf=pd.read_csv(r\"/kaggle/input/rfcx-species-audio-detection/train_tp.csv\")\ntraindf.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"traindf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"total Species in Dataset : {len(traindf.recording_id)}\")\nprint(f\"total Recordings in Dataset : {len(traindf.recording_id.unique())}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(x=traindf['species_id'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Species_id is balanced, all the categories have sufficient number of entries.<br/>\nLet's look into other features"},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(x=traindf['songtype_id'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path='/kaggle/input/rfcx-species-audio-detection/train/003bec244.flac'\ny, sr = librosa.load(path)\nplt.figure(figsize=(20,5))\nlibrosa.display.waveplot(y, sr=sr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#display Spectrogram\npath='/kaggle/input/rfcx-species-audio-detection/train/003bec244.flac'\nx, sr = librosa.load(path)\nX = librosa.stft(x)\nXdb = librosa.amplitude_to_db(abs(X))\nplt.figure(figsize=(20, 5))\nlibrosa.display.specshow(Xdb, sr=sr, x_axis='time', y_axis='hz') \n#If to pring log of frequencies  \n#librosa.display.specshow(Xdb, sr=sr, x_axis='time', y_axis='log')\nplt.colorbar()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Using TFRecord Data for model building to leverage the features of Tensorflow.<br/>\n* The TFRecord format is a simple format for storing a sequence of binary records.<br/>\n* To Explore more about TFRecord, associated methods and functions, please refer -https://www.tensorflow.org/tutorials/load_data/tfrecord\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"def _bytes_feature(value):\n  \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n  if isinstance(value, type(tf.constant(0))):\n    value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n  \"\"\"Returns a float_list from a float / double.\"\"\"\n  return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef serialize_data(wav, recording_id, target, song_id, tmin,fmin, tmax, fmax):\n  feature = {\n      'wav': _bytes_feature(wav),\n      'recording_id': _bytes_feature(recording_id),\n      'target': _float_feature(target),\n      'song_id': _float_feature(song_id),\n      'tmin': _float_feature(tmin),\n      'fmin' : _float_feature(fmin),\n      'tmax': _float_feature(tmax),\n      'fmax' : _float_feature(fmax),\n  }\n  proto_buff = tf.train.Example(features=tf.train.Features(feature=feature))\n  return proto_buff.SerializeToString() ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"from sklearn.model_selection import StratifiedKFold\n\ntfrec_num = 0\nkfold = StratifiedKFold(n_splits=10, shuffle=False)\nfor fold, (train_idx, test_idx) in enumerate(kfold.split(traindf['recording_id'], traindf['species_id'])):\n    x_train , y_train = traindf['recording_id'][test_idx] , traindf['species_id'][test_idx]\n   \n    with tf.io.TFRecordWriter('tp%.2i-%.2i.tfrec'%(tfrec_num, len(test_idx))) as writer:\n        print('Writing_tfrecords ',fold)\n        for recording_id , true_value in zip(x_train, y_train): \n            wav, _ = librosa.load(f'../input/rfcx-species-audio-detection/train/{recording_id}.flac', sr = None)\n            label_info = traindf.loc[traindf['recording_id'] == str(recording_id)].values[0]\n            wav = tf.audio.encode_wav(tf.reshape(wav,(wav.shape[0], 1)) ,sample_rate = 48000)\n            recording_id = label_info[0].encode()\n            target = label_info[1]\n            song_id = label_info[2]\n            tmin = label_info[3]\n            fmin = label_info[4]\n            tmax = label_info[5]\n            fmax = label_info[6]\n            proto_buffer = serialize_data(wav, recording_id, target, song_id, tmin,fmin, tmax, fmax)\n            writer.write(proto_buffer)\n    tfrec_num += 1"},{"metadata":{},"cell_type":"markdown","source":"The above step will generate TFRecords in Kaggle/Working folder, however these files can't be used in TPU processing as TPU works only on Public dataset which are part of Input folder. so these newly created TFrecords are to be stored in Public datasets and have to be imported."},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\nignore_order = tf.data.Options()\nignore_order.experimental_deterministic = False\n    \n# On Kaggle you can also use KaggleDatasets().get_gcs_path() to obtain the GCS path of a Kaggle dataset\n\nTRAIN_DATA_DIR = 'rfcx-audio-detection'\nTRAIN_GCS_PATH = KaggleDatasets().get_gcs_path(TRAIN_DATA_DIR)\nFILENAMES = tf.io.gfile.glob(TRAIN_GCS_PATH + '/tp*.tfrec')\n\ndataset = tf.data.TFRecordDataset(FILENAMES, num_parallel_reads=AUTO)\ndataset = dataset.with_options(ignore_order)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#test_files\nTEST_DATA_DIR = 'rfcx-species-audio-detection'\nTEST_GCS_PATH =  KaggleDatasets().get_gcs_path(TEST_DATA_DIR)\nTEST_FILES = tf.io.gfile.glob(TEST_GCS_PATH + '/tfrecords/test/*.tfrec')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in FILENAMES]\nprint(f\"Total number of files :{np.sum(n)}\")\nno_of_training_samples=np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Configuration Variables"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Model Related Configurations\nCUT = 10\nTIME = 10\nGLOBAL_BATCH_SIZE = 16 * REPLICAS\nLEARNING_RATE = 0.0015\nWARMUP_LEARNING_RATE = 1e-5\nWARMUP_EPOCHS = int(EPOCHS*0.1)\nPATIENCE = 8\nSTEPS_PER_EPOCH = 128\nN_FOLDS = 5\nNUM_TRAINING_SAMPLES = no_of_training_samples\n\n#Audio related Configurations\nsample_rate = 48000\nstft_window_seconds: float = 0.025\nstft_hop_seconds: float = 0.005\nframe_length: int =  1200    \nmel_bands: int = 512\nmel_min_hz: float = 50.0\nmel_max_hz: float = 24000.0\nlog_offset: float = 0.001\npatch_window_seconds: float = 0.96\npatch_hop_seconds: float = 0.48\n\npatch_frames =  int(round(patch_window_seconds / stft_hop_seconds))\npatch_bands = mel_bands\nheight = mel_bands\nwidth = 2000\nnum_classes: int = 24\ndropout = 0.35\nclassifier_activation: str = 'sigmoid'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature_description_label = {\n    'wav': tf.io.FixedLenFeature([], tf.string),\n    'recording_id': tf.io.FixedLenFeature([], tf.string ),\n    'target' : tf.io.FixedLenFeature([], tf.float32),\n    'song_id': tf.io.FixedLenFeature([], tf.float32),\n     'tmin' : tf.io.FixedLenFeature([], tf.float32),\n     'fmin' : tf.io.FixedLenFeature([], tf.float32),\n     'tmax' : tf.io.FixedLenFeature([], tf.float32),\n     'fmax' : tf.io.FixedLenFeature([], tf.float32),\n}\nfeature_description_unlabel = {\n    'recording_id': tf.io.FixedLenFeature([], tf.string),\n    'audio_wav': tf.io.FixedLenFeature([], tf.string),\n    }\nfeature_dtype = {\n    'wav': tf.float32,\n    'recording_id': tf.string,\n    'target': tf.float32,\n    'song_id': tf.float32,\n    't_min': tf.float32,\n    'f_min': tf.float32,\n    't_max': tf.float32,\n    'f_max':tf.float32,\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Cut the audio frame to required time regions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def ReadLabeledAudio(protoBuff):\n    sample = tf.io.parse_single_example(protoBuff, feature_description_label)\n    wav, _ = tf.audio.decode_wav(sample['wav'], desired_channels=1) # mono\n    target = tf.cast(sample['target'],tf.float32)\n    target = tf.squeeze(tf.one_hot([target,], depth = num_classes), axis = 0)\n    #get tmin,tmax fmin,fmax\n    tmin = tf.cast(sample['tmin'], tf.float32)\n    fmin = tf.cast(sample['fmin'], tf.float32)\n    tmax = tf.cast(sample['tmax'], tf.float32)\n    fmax = tf.cast(sample['fmax'], tf.float32)\n    \n    tmax_s = tmax * tf.cast(sample_rate, tf.float32)\n    tmin_s = tmin * tf.cast(sample_rate, tf.float32)\n    cut_s = tf.cast(CUT * sample_rate, tf.float32)\n    all_s = tf.cast(60 * sample_rate, tf.float32)\n    tsize_s = tmax_s - tmin_s\n    cut_min = tf.cast(tf.maximum(0.0,tf.minimum(tmin_s - (cut_s - tsize_s) / 2,tf.minimum(tmax_s + (cut_s - tsize_s) / 2, all_s) - cut_s)), tf.int32)\n    cut_max = cut_min + CUT * sample_rate\n    wav = tf.squeeze(wav[cut_min : cut_max])\n    \n    return wav, target","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def ReadUnlabeledAudio(protoBuff):\n\n    sample = tf.io.parse_single_example(protoBuff, feature_description_unlabel)\n    wav, _ = tf.audio.decode_wav(sample['audio_wav'], desired_channels=1) # mono\n    recording_id = tf.reshape(tf.cast(sample['recording_id'] , tf.string), [1])\n\n    def _cut_audio(i):\n        _sample = {\n            'audio_wav': tf.reshape(wav[i*sample_rate*TIME:(i+1)*sample_rate*TIME], [sample_rate*TIME]),\n            'recording_id': sample['recording_id']\n        }\n        return _sample\n\n    return tf.map_fn(_cut_audio, tf.range(60//TIME), dtype={\n        'audio_wav': tf.float32,\n        'recording_id': tf.string\n    })","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"A spectrogram shows how the frequency content of a signal changes over time and can be calculated from the time domain signal.\n\nTo know more about method - tf.signal.stft() - https://www.tensorflow.org/api_docs/python/tf/signal/stft\n\nTo know more about linear_to_mel_weight_matrix() - https://www.tensorflow.org/api_docs/python/tf/signal/linear_to_mel_weight_matrix"},{"metadata":{},"cell_type":"markdown","source":"Transform raw audio data to Log-Mel-Spectogram\n1. **Compute the short-time Fourier transform of audio signals** - STFT divides a long signal into shorter segments, often called frames, and computes the spectrum for each frame. The frames typically overlap to minimize data-loss at the edges\n2. **Compute the magnitudes**- STFT from the previous step returns a tensor of complex values. Use tf.abs() to compute the magnitudes.\n3. **Instantiate the mel filterbank**- Transforming standard spectrograms to mel-spectrograms involves warping frequencies to the mel-scale and combining FFT bins to mel-frequency bins.\n4. **Warp the linear-scale magnitude** -spectrograms to mel-scale -Multiply the squared magnitude-spectrograms with the mel-filterbank and you get mel-scaled power-spectrograms.\n5. **Transform magnitudes to log-scale** - We perceive changes in loudness logarithmically. So, in this last step, we want to scale the mel-spectrograms’ magnitudes logarithmically, too.\n\nhttps://towardsdatascience.com/how-to-easily-process-audio-on-your-gpu-with-tensorflow-2d9d91360f06"},{"metadata":{"trusted":true},"cell_type":"code","source":"def power_to_db(S, amin=1e-16, top_db=80.0):\n    \"\"\"Convert a power-spectrogram (magnitude squared) to decibel (dB) units.\n    Computes the scaling ``10 * log10(S / max(S))`` in a numerically\n    stable way.\n    Based on:\n    https://librosa.github.io/librosa/generated/librosa.core.power_to_db.html\n    \"\"\"\n    def _tf_log10(x):\n        numerator = tf.math.log(x)\n        denominator = tf.math.log(tf.constant(10, dtype=numerator.dtype))\n        return numerator / denominator\n    \n    # Scale magnitude relative to maximum value in S. Zeros in the output \n    # correspond to positions where S == ref.\n    ref = tf.reduce_max(S)\n\n    log_spec = 10.0 * _tf_log10(tf.maximum(amin, S))\n    log_spec -= 10.0 * _tf_log10(tf.maximum(amin, ref))\n\n    log_spec = tf.maximum(log_spec, tf.reduce_max(log_spec) - top_db)\n\n    return log_spec","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def transform_log_mel_spect(waveform,target):\n    #1. Compute the short-time Fourier transform of your audio signals  \n    window_length_samples = int(\n      round(sample_rate * stft_window_seconds))\n    hop_length_samples = int(\n      round(sample_rate * stft_hop_seconds))\n    fft_length = 2 ** int(np.ceil(np.log(window_length_samples) / np.log(2.0)))\n    num_spectrogram_bins = fft_length // 2 + 1\n    #2. Compute the magnitudes\n    magnitude_spectrogram = tf.abs(tf.signal.stft(\n      signals=waveform,\n      frame_length=frame_length,\n      frame_step=hop_length_samples,\n      fft_length= fft_length))\n    # magnitude_spectrogram has shape [<# STFT frames>, num_spectrogram_bins]\n\n    #3.Instantiate the mel filterbank\n    mel_filterbank = tf.signal.linear_to_mel_weight_matrix(\n        num_mel_bins=mel_bands,\n        num_spectrogram_bins=num_spectrogram_bins,\n        sample_rate=sample_rate,\n        lower_edge_hertz=mel_min_hz,\n        upper_edge_hertz=mel_max_hz)\n    mel_spectrogram = tf.matmul(\n      magnitude_spectrogram, mel_filterbank)\n \n    log_mel = tf.math.log(mel_spectrogram + log_offset)\n#     log_magnitude_mel_spect has shape [<# STFT frames>, mel_bands]\n    log_mel = tf.transpose(log_mel)\n    log_magnitude_mel_spect = tf.reshape(log_mel , [tf.shape(log_mel)[0] ,tf.shape(log_mel)[1],1])\n    # Frame spectrogram (shape [<# STFT frames>, mel_bands]) into patches\n    # (the input examples). Only complete frames are emitted, so if there is\n    # less than patch_window_seconds of waveform then nothing is emitted\n    # (to avoid this, zero-pad before processing).\n    spectrogram_hop_length_samples = int(\n      round(sample_rate * stft_hop_seconds))\n    spectrogram_sample_rate = sample_rate / spectrogram_hop_length_samples\n    patch_window_length_samples = int(\n      round(spectrogram_sample_rate * patch_window_seconds))\n    patch_hop_length_samples = int(\n      round(spectrogram_sample_rate * patch_hop_seconds))\n    features = tf.signal.frame(\n        signal=log_magnitude_mel_spect,\n        frame_length=patch_window_length_samples,\n        frame_step=patch_hop_length_samples,\n        axis=0)\n    # features has shape [<# patches>, <# STFT frames in an patch>, mel_bands]\n    \n    return log_magnitude_mel_spect, target","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def frequency_masking(mel_spectrogram):\n    \n    frequency_masking_para = 80, \n    frequency_mask_num = 2\n    \n    fbank_size = tf.shape(mel_spectrogram)\n#     print(fbank_size)\n    n, v = fbank_size[0], fbank_size[1]\n\n    for i in range(frequency_mask_num):\n        f = tf.random.uniform([], minval=0, maxval= tf.squeeze(frequency_masking_para), dtype=tf.int32)\n        v = tf.cast(v, dtype=tf.int32)\n        f0 = tf.random.uniform([], minval=0, maxval= tf.squeeze(v-f), dtype=tf.int32)\n\n        # warped_mel_spectrogram[f0:f0 + f, :] = 0\n        mask = tf.concat((tf.ones(shape=(n, v - f0 - f,1)),\n                          tf.zeros(shape=(n, f,1)),\n                          tf.ones(shape=(n, f0,1)),\n                          ),1)\n        mel_spectrogram = mel_spectrogram * mask\n    return tf.cast(mel_spectrogram, dtype=tf.float32)\n\n\ndef time_masking(mel_spectrogram):\n    time_masking_para = 40, \n    time_mask_num = 1\n    \n    fbank_size = tf.shape(mel_spectrogram)\n    n, v = fbank_size[0], fbank_size[1]\n\n   \n    for i in range(time_mask_num):\n        t = tf.random.uniform([], minval=0, maxval=tf.squeeze(time_masking_para), dtype=tf.int32)\n        t0 = tf.random.uniform([], minval=0, maxval= n-t, dtype=tf.int32)\n\n        # mel_spectrogram[:, t0:t0 + t] = 0\n        mask = tf.concat((tf.ones(shape=(n-t0-t, v,1)),\n                          tf.zeros(shape=(t, v,1)),\n                          tf.ones(shape=(t0, v,1)),\n                          ), 0)\n        \n        mel_spectrogram = mel_spectrogram * mask\n    return tf.cast(mel_spectrogram, dtype=tf.float32)\n\n\ndef random_brightness(image):\n    return tf.image.random_brightness(image, 0.2)\n\ndef random_gamma(image):\n    return tf.image.random_contrast(image, lower = 0.1, upper = 0.3)\n\ndef random_flip_right(image):\n    return tf.image.random_flip_left_right(image)\n\ndef random_flip_up_down(image):\n    return tf.image.random_flip_left_right(image)\n\navailable_ops = [\n          frequency_masking ,\n          time_masking, \n          random_brightness, \n          random_flip_up_down,\n          random_flip_right \n         ]\n\ndef apply_augmentation(image, target):\n    num_layers = int(np.random.uniform(low = 0, high = 3))\n    \n    for layer_num in range(num_layers):\n        op_to_select = tf.random.uniform([], maxval=len(available_ops), dtype=tf.int32, seed = seed)\n        for (i, op_name) in enumerate(available_ops):\n            image = tf.cond(\n            tf.equal(i, op_to_select),\n            lambda selected_func=op_name,: selected_func(\n                image),\n            lambda: image)\n    return image, target\n                                                                                       ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Audio Data Augmentation - https://www.tensorflow.org/io/tutorials/audio"},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocess(image, target):\n    \n    image = tf.image.grayscale_to_rgb(image)\n    image = tf.image.resize(image, [height,width])\n    image = tf.image.per_image_standardization(image)\n    return image , target\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_dataset(filenames, labeled = True, ordered = False , training = True):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # Disregarding data order. Order does not matter since we will be shuffling the data anyway\n    \n    ignore_order = tf.data.Options()\n    if not ordered:\n        # disable order, increase speed\n        ignore_order.experimental_deterministic = False \n        \n    # automatically interleaves reads from multiple files\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO )\n    # use data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(ReadLabeledAudio , num_parallel_calls = AUTO )\n    dataset = dataset.map(transform_log_mel_spect , num_parallel_calls = AUTO)   \n    if training:\n        dataset = dataset.map(apply_augmentation, num_parallel_calls = AUTO)\n    dataset = dataset.map(preprocess, num_parallel_calls = AUTO)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_dataset(filenames, training = True):\n    if training:\n        dataset = load_dataset(filenames , training = True)\n        dataset = dataset.shuffle(256).repeat()\n        dataset = dataset.batch(GLOBAL_BATCH_SIZE, drop_remainder = True)\n    else:\n        dataset = load_dataset(filenames , training = False)\n        dataset = dataset.batch(GLOBAL_BATCH_SIZE).cache()\n    \n    dataset = dataset.prefetch(AUTO)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FILENAMES","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# mel spectrogram visualization\ntrain_dataset = get_dataset(FILENAMES, training = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(16,6))\nfor i, (wav, target) in enumerate(train_dataset.unbatch().take(4)):\n    plt.subplot(2,2,i+1)\n    plt.imshow(wav[:, :, 0])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3. Model Building"},{"metadata":{},"cell_type":"markdown","source":"### LWLRAP \nLabel-weighted label-ranking average precision (lwlrap, pronounced \"Lol wrap\"). This measures the average precision of retrieving a ranked list of relevant labels for each test clip.<br/>\nWe use label weighting because it allows per-class values to be calculated, and still have the overall metric be expressed as simple average of the per-class metrics (weighted by each label's prior in the test set).\nfor more info - https://www.kaggle.com/pkmahan/understanding-lwlrap"},{"metadata":{"trusted":true},"cell_type":"code","source":"def _one_sample_positive_class_precisions(example):\n    y_true, y_pred = example\n    y_true = tf.reshape(y_true, tf.shape(y_pred))\n    retrieved_classes = tf.argsort(y_pred, direction='DESCENDING')\n#     shape = tf.shape(retrieved_classes)\n    class_rankings = tf.argsort(retrieved_classes)\n    retrieved_class_true = tf.gather(y_true, retrieved_classes)\n    retrieved_cumulative_hits = tf.math.cumsum(tf.cast(retrieved_class_true, tf.float32))\n\n    idx = tf.where(y_true)[:, 0]\n    i = tf.boolean_mask(class_rankings, y_true)\n    r = tf.gather(retrieved_cumulative_hits, i)\n    c = 1 + tf.cast(i, tf.float32)\n    precisions = r / c\n\n    dense = tf.scatter_nd(idx[:, None], precisions, [y_pred.shape[0]])\n    return dense\n\n# @tf.function\nclass LWLRAP(tf.keras.metrics.Metric):\n    def __init__(self, num_classes, name='lwlrap'):\n        super().__init__(name=name)\n\n        self._precisions = self.add_weight(\n            name='per_class_cumulative_precision',\n            shape=[num_classes],\n            initializer='zeros',\n        )\n\n        self._counts = self.add_weight(\n            name='per_class_cumulative_count',\n            shape=[num_classes],\n            initializer='zeros',\n        )\n\n    def update_state(self, y_true, y_pred, sample_weight=None):\n        precisions = tf.map_fn(\n            fn=_one_sample_positive_class_precisions,\n            elems=(y_true, y_pred),\n            dtype=(tf.float32),\n        )\n\n        increments = tf.cast(precisions > 0, tf.float32)\n        total_increments = tf.reduce_sum(increments, axis=0)\n        total_precisions = tf.reduce_sum(precisions, axis=0)\n\n        self._precisions.assign_add(total_precisions)\n        self._counts.assign_add(total_increments)        \n\n    def result(self):\n        per_class_lwlrap = self._precisions / tf.maximum(self._counts, 1.0)\n        per_class_weight = self._counts / tf.reduce_sum(self._counts)\n        overall_lwlrap = tf.reduce_sum(per_class_lwlrap * per_class_weight)\n        return overall_lwlrap\n\n    def reset_states(self):\n        self._precisions.assign(self._precisions * 0)\n        self._counts.assign(self._counts * 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def cosine_decay_with_warmup(global_step,\n                             learning_rate_base,\n                             total_steps,\n                             warmup_learning_rate=0.0,\n                             warmup_steps= 0,\n                             hold_base_rate_steps=0):\n \n    if total_steps < warmup_steps:\n        raise ValueError('total_steps must be larger or equal to '\n                     'warmup_steps.')\n    learning_rate = 0.5 * learning_rate_base * (1 + tf.cos(\n        np.pi *\n        (tf.cast(global_step, tf.float32) - warmup_steps - hold_base_rate_steps\n        ) / float(total_steps - warmup_steps - hold_base_rate_steps)))\n    if hold_base_rate_steps > 0:\n        learning_rate = tf.where(\n          global_step > warmup_steps + hold_base_rate_steps,\n          learning_rate, learning_rate_base)\n    if warmup_steps > 0:\n        if learning_rate_base < warmup_learning_rate:\n            raise ValueError('learning_rate_base must be larger or equal to '\n                         'warmup_learning_rate.')\n        slope = (learning_rate_base - warmup_learning_rate) / warmup_steps\n        warmup_rate = slope * tf.cast(global_step,\n                                    tf.float32) + warmup_learning_rate\n        learning_rate = tf.where(global_step < warmup_steps, warmup_rate,\n                               learning_rate)\n    return tf.where(global_step > total_steps, 0.0, learning_rate,\n                    name='learning_rate')\n\n\n#dummy example\nrng = [i for i in range(int(EPOCHS * STEPS_PER_EPOCH))]\nWARMUP_STEPS =  int(WARMUP_EPOCHS * STEPS_PER_EPOCH)\ny = [cosine_decay_with_warmup(x , LEARNING_RATE, len(rng), 1e-5, WARMUP_STEPS) for x in rng]\n\nsns.set(style='whitegrid')\nfig, ax = plt.subplots(figsize=(20, 6))\nplt.plot(rng, y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# below code copied from - https://www.kaggle.com/ashusma/training-rfcx-tensorflow-tpu-effnet-b2#This-notebook-shows-the-training-of-RFCX-data-on-Tensorflow-TPU\n# to apply learning rate schedule stepwise we need to subclass keras callback\n# if we would have applied lr schedule epoch wise then it is not needed we can only call class learningrateschedule \n\nclass WarmUpCosineDecayScheduler(tf.keras.callbacks.Callback):\n\n    def __init__(self,\n                 learning_rate_base,\n                 total_steps,\n                 global_step_init=0,\n                 warmup_learning_rate=0.0,\n                 warmup_steps=0,\n                 hold_base_rate_steps=0,\n                 verbose=0):\n\n        super(WarmUpCosineDecayScheduler, self).__init__()\n        self.learning_rate_base = learning_rate_base\n        self.total_steps = total_steps\n        self.global_step = global_step_init\n        self.warmup_learning_rate = warmup_learning_rate\n        self.warmup_steps = warmup_steps\n        self.hold_base_rate_steps = hold_base_rate_steps\n        self.verbose = verbose\n        self.learning_rates = []\n\n    def on_batch_end(self, batch, logs=None):\n        self.global_step = self.global_step + 1\n        lr = K.get_value(self.model.optimizer.lr)\n        self.learning_rates.append(lr)\n\n    def on_batch_begin(self, batch, logs=None):\n        lr = cosine_decay_with_warmup(global_step=self.global_step,\n                                      learning_rate_base=self.learning_rate_base,\n                                      total_steps=self.total_steps,\n                                      warmup_learning_rate=self.warmup_learning_rate,\n                                      warmup_steps=self.warmup_steps,\n                                      hold_base_rate_steps=self.hold_base_rate_steps)\n        K.set_value(self.model.optimizer.lr, lr)\n        if self.verbose > 0:\n            print('\\nBatch %05d: setting learning '\n                  'rate to %s.' % (self.global_step + 1, lr.numpy()))\n            \n\ntotal_steps = int(EPOCHS * STEPS_PER_EPOCH)\n# Compute the number of warmup batches or steps.\nwarmup_steps = int(WARMUP_EPOCHS * STEPS_PER_EPOCH)\nwarmup_learning_rate = WARMUP_LEARNING_RATE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def Build_MODEL():\n    waveform = Input(shape=(None,None,3), dtype=tf.float32)\n    noisy_waveform = GaussianNoise(0.2)(waveform)\n    model = efn.EfficientNetB2(include_top=False, weights='imagenet',) \n    model_output = model(noisy_waveform)\n    model_output = GlobalAveragePooling2D()(model_output)\n    dense = Dropout(dropout)(model_output)\n    predictions = Dense(num_classes, activation = classifier_activation )(dense)\n    model = Model(name='Efficientnet', inputs=waveform,outputs=[predictions])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model():\n    with strategy.scope():\n        model = Build_MODEL()\n        model.summary()\n        model.compile(optimizer = 'adam',\n                                loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.1),\n                                metrics = [LWLRAP(num_classes = num_classes),\n                                ])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"skf = KFold(n_splits=10, shuffle=True, random_state=seed)\noof_pred = []; oof_labels = []; history_list = []\n\nfor fold,(idxT, idxV) in enumerate(skf.split(np.arange(10))):\n    if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n    print(f'\\nFOLD: {fold+1}')\n    print(f'TRAIN: {idxT} VALID: {idxV}')\n\n    # Create train and validation sets\n    TRAIN_FILENAMES = [FILENAMES[x] for x in idxT]\n    VALID_FILENAMES = [FILENAMES[x] for x in idxV]\n    np.random.shuffle(TRAIN_FILENAMES)\n    \n    train_dataset =  get_dataset(TRAIN_FILENAMES, training=True,)\n    validation_data= get_dataset(VALID_FILENAMES, training=False) \n\n    model = get_model()\n\n    model_path = f'model_fold {fold}.h5'\n    early_stopping = EarlyStopping(monitor = 'val_lwlrap', mode = 'max', \n                       patience = PATIENCE, restore_best_weights=True, verbose=1)\n\n    # Create the Learning rate scheduler.\n    cosine_warm_up_lr = WarmUpCosineDecayScheduler(learning_rate_base= LEARNING_RATE,\n                                    total_steps= total_steps,\n                                    warmup_learning_rate= warmup_learning_rate,\n                                    warmup_steps= warmup_steps,\n                                    hold_base_rate_steps=0)\n\n    ## TRAIN\n    history = model.fit(train_dataset, \n                        steps_per_epoch=STEPS_PER_EPOCH, \n                        callbacks=[early_stopping, cosine_warm_up_lr], \n                        epochs=EPOCHS,  \n                        validation_data = validation_data,\n                        verbose = 2).history\n\n    history_list.append(history)\n    # Save last model weights\n    model.save_weights(model_path)\n\n# OOF predictions\n    ds_valid = get_dataset(VALID_FILENAMES, training = False)\n    oof_labels.append([target.numpy() for frame, target in iter(ds_valid.unbatch())])\n    x_oof = ds_valid.map(lambda frames, target: frames)\n    oof_pred.append(np.argmax(model.predict(x_oof), axis=-1))\n\n    ## RESULTS\n    print(f\"#### FOLD {fold+1} OOF Accuracy = {np.max(history['val_lwlrap']):.3f}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 4.Generating Test output & submitting them"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_test_dataset(filenames, training = False):\n    \n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO )  \n    dataset = dataset.map(ReadUnlabeledAudio , num_parallel_calls = AUTO ).unbatch()\n    dataset = dataset.map(lambda spec : transform_log_mel_spect(spec['audio_wav'], spec['recording_id']) , num_parallel_calls = AUTO)\n    dataset = dataset.map(preprocess, num_parallel_calls = AUTO)\n    return dataset.batch(GLOBAL_BATCH_SIZE*4).cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_predict = []\n\ntest_data = get_test_dataset(TEST_FILES, training = False)\ntest_audio = test_data.map(lambda frames, recording_id: frames)\n\nfor fold in range(N_FOLDS):\n    model.load_weights(f'model_fold {fold}.h5')\n    test_predict.append(model.predict(test_audio, verbose = 1 ))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.array(test_predict).shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SUB = pd.read_csv('../input/rfcx-species-audio-detection/sample_submission.csv')\n\npredict = np.array(test_predict).reshape(N_FOLDS, len(SUB), 60 // TIME, num_classes)\npredict = np.mean(np.max(predict ,axis = 2) , axis = 0)\n# predict = np.mean(predict, axis =  0)\n\nrecording_id = test_data.map(lambda frames, recording_id: recording_id).unbatch()\n# # all in one batch\ntest_ids = next(iter(recording_id.batch(len(SUB) * 60 // TIME))).numpy().astype('U').reshape(len(SUB), 60 // TIME)\n\npred_df = pd.DataFrame({ 'recording_id' : test_ids[:, 0],\n             **{f's{i}' : predict[:, i] for i in range(num_classes)} })","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_df.sort_values('recording_id', inplace = True) \npred_df.to_csv('submission.csv', index = False)  ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}