{"cells":[{"metadata":{},"cell_type":"markdown","source":"# This notebook shows the training of RFCX data on Tensorflow TPU\n\nThe dataset used in this notebook is 10 fold Groupkfold tp only tfrecords that i have created [here](http://www.kaggle.com/ashusma/rfcx-audio-detection) and the simple script for the notebook is [this](https://www.kaggle.com/ashusma/rfcx-audio-creating-tfrecords?scriptVersionId=51531240).\n\nTraining description :\n\n* training with 10 sec clip around true positives\n* taking full spectrogram size \n* random augmentation and gaussian noise\n* label smoothing\n* stepwise cosine decay with warm restarts and early stopping\n* for inference 10sec clip is used and then aggregrating and taking max of the audio wav prediction \n\n\nSince this notebook uses tpu accelerator having 128 gb (16 gb each replica) so for efficient use i have done following optimization :\n* increased the spectrogram size\n* caching validation and test set as both are small in number for faster computation\n* wrapped all user defined function with map that allow parallel computation\n* reduced the python overhead \n* tensorflow 2.3 and above has argument execution per step in model.compile function that significantly improves performance by running multiple steps within tpu worker. but since kaggle has not updated tf version we cannot take advantage of that but one can try it on google colab\n* above step can also be done by using custom training loop"},{"metadata":{"trusted":true},"cell_type":"code","source":"! pip install -q efficientnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import math, os, re, warnings, random\nimport tensorflow as tf\nimport numpy as np\nimport pandas as pd\nimport librosa\nfrom kaggle_datasets import KaggleDatasets\nimport matplotlib.pyplot as plt\nfrom IPython.display import Audio\nfrom tensorflow.keras import Model, layers\nfrom sklearn.model_selection import KFold\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler\nfrom tensorflow.keras.layers import GlobalAveragePooling2D, Input, Dense, Dropout, GaussianNoise\nfrom tensorflow.keras.applications import ResNet50\nimport efficientnet.keras as efn\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# TPU Detection And Initialization"},{"metadata":{"trusted":true},"cell_type":"code","source":"# TPU or GPU detection\n# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print(f'Running on TPU {tpu.master()}')\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed=0):\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\nseed = 42\nseed_everything(seed)\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\n# train_files\n\nTRAIN_DATA_DIR = 'rfcx-audio-detection'\nTRAIN_GCS_PATH = KaggleDatasets().get_gcs_path(TRAIN_DATA_DIR)\nFILENAMES = tf.io.gfile.glob(TRAIN_GCS_PATH + '/tp*.tfrec')\n\n\n#test_files\nTEST_DATA_DIR = 'rfcx-species-audio-detection'\nTEST_GCS_PATH =  KaggleDatasets().get_gcs_path(TEST_DATA_DIR)\nTEST_FILES = tf.io.gfile.glob(TEST_GCS_PATH + '/tfrecords/test/*.tfrec')\n\nno_of_training_samples = count_data_items(FILENAMES)\n\nprint('num_training_samples are', no_of_training_samples)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CUT = 10\nTIME = 10\nEPOCHS = 25\nGLOBAL_BATCH_SIZE = 4 * REPLICAS\nLEARNING_RATE = 0.0015\nWARMUP_LEARNING_RATE = 1e-5\nWARMUP_EPOCHS = int(EPOCHS*0.1)\nPATIENCE = 8\nSTEPS_PER_EPOCH = 64\nN_FOLDS = 5\nNUM_TRAINING_SAMPLES = no_of_training_samples\n\n\nclass params:\n    sample_rate = 48000\n    stft_window_seconds: float = 0.025\n    stft_hop_seconds: float = 0.005\n    frame_length: int =  1200    \n    mel_bands: int = 512\n    mel_min_hz: float = 50.0\n    mel_max_hz: float = 24000.0\n    log_offset: float = 0.001\n    patch_window_seconds: float = 0.96\n    patch_hop_seconds: float = 0.48\n\n  \n    patch_frames =  int(round(patch_window_seconds / stft_hop_seconds))\n\n  \n    patch_bands = mel_bands\n    height = mel_bands\n    width = 2000\n    num_classes: int = 24\n    dropout = 0.35\n    classifier_activation: str = 'sigmoid'\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature_description = {\n    'wav': tf.io.FixedLenFeature([], tf.string),\n    'recording_id': tf.io.FixedLenFeature([], tf.string ),\n    'target' : tf.io.FixedLenFeature([], tf.float32),\n    'song_id': tf.io.FixedLenFeature([], tf.float32),\n     'tmin' : tf.io.FixedLenFeature([], tf.float32),\n     'fmin' : tf.io.FixedLenFeature([], tf.float32),\n     'tmax' : tf.io.FixedLenFeature([], tf.float32),\n     'fmax' : tf.io.FixedLenFeature([], tf.float32),\n}\nfeature_dtype = {\n    'wav': tf.float32,\n    'recording_id': tf.string,\n    'target': tf.float32,\n    'song_id': tf.float32,\n    't_min': tf.float32,\n    'f_min': tf.float32,\n    't_max': tf.float32,\n    'f_max':tf.float32,\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def waveform_to_log_mel_spectrogram(waveform,target_or_rec_id):\n    \"\"\"Compute log mel spectrogram patches of a 1-D waveform.\"\"\"\n    # waveform has shape [<# samples>]\n\n    # Convert waveform into spectrogram using a Short-Time Fourier Transform.\n    # Note that tf.signal.stft() uses a periodic Hann window by default.\n\n    window_length_samples = int(\n      round(params.sample_rate * params.stft_window_seconds))\n    hop_length_samples = int(\n      round(params.sample_rate * params.stft_hop_seconds))\n    fft_length = 2 ** int(np.ceil(np.log(window_length_samples) / np.log(2.0)))\n#     print(fft_length, window_length_samples, hop_length_samples)\n    num_spectrogram_bins = fft_length // 2 + 1\n    magnitude_spectrogram = tf.abs(tf.signal.stft(\n      signals=waveform,\n      frame_length=params.frame_length,\n      frame_step=hop_length_samples,\n      fft_length= fft_length))\n    # magnitude_spectrogram has shape [<# STFT frames>, num_spectrogram_bins]\n\n    # Convert spectrogram into log mel spectrogram.\n    linear_to_mel_weight_matrix = tf.signal.linear_to_mel_weight_matrix(\n        num_mel_bins=params.mel_bands,\n        num_spectrogram_bins=num_spectrogram_bins,\n        sample_rate=params.sample_rate,\n        lower_edge_hertz=params.mel_min_hz,\n        upper_edge_hertz=params.mel_max_hz)\n    mel_spectrogram = tf.matmul(\n      magnitude_spectrogram, linear_to_mel_weight_matrix)\n    log_mel = tf.math.log(mel_spectrogram + params.log_offset)\n#     log_mel_spectrogram has shape [<# STFT frames>, params.mel_bands]\n    log_mel = tf.transpose(log_mel)\n    log_mel_spectrogram = tf.reshape(log_mel , [tf.shape(log_mel)[0] ,tf.shape(log_mel)[1],1])\n    # Frame spectrogram (shape [<# STFT frames>, params.mel_bands]) into patches\n    # (the input examples). Only complete frames are emitted, so if there is\n    # less than params.patch_window_seconds of waveform then nothing is emitted\n    # (to avoid this, zero-pad before processing).\n    spectrogram_hop_length_samples = int(\n      round(params.sample_rate * params.stft_hop_seconds))\n    spectrogram_sample_rate = params.sample_rate / spectrogram_hop_length_samples\n    patch_window_length_samples = int(\n      round(spectrogram_sample_rate * params.patch_window_seconds))\n    patch_hop_length_samples = int(\n      round(spectrogram_sample_rate * params.patch_hop_seconds))\n    features = tf.signal.frame(\n        signal=log_mel_spectrogram,\n        frame_length=patch_window_length_samples,\n        frame_step=patch_hop_length_samples,\n        axis=0)\n    # features has shape [<# patches>, <# STFT frames in an patch>, params.mel_bands]\n    \n    return log_mel_spectrogram, target_or_rec_id","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"def frequency_masking(mel_spectrogram):\n    \n    frequency_masking_para = 80, \n    frequency_mask_num = 2\n    \n    fbank_size = tf.shape(mel_spectrogram)\n#     print(fbank_size)\n    n, v = fbank_size[0], fbank_size[1]\n\n    for i in range(frequency_mask_num):\n        f = tf.random.uniform([], minval=0, maxval= tf.squeeze(frequency_masking_para), dtype=tf.int32)\n        v = tf.cast(v, dtype=tf.int32)\n        f0 = tf.random.uniform([], minval=0, maxval= tf.squeeze(v-f), dtype=tf.int32)\n\n        # warped_mel_spectrogram[f0:f0 + f, :] = 0\n        mask = tf.concat((tf.ones(shape=(n, v - f0 - f,1)),\n                          tf.zeros(shape=(n, f,1)),\n                          tf.ones(shape=(n, f0,1)),\n                          ),1)\n        mel_spectrogram = mel_spectrogram * mask\n    return tf.cast(mel_spectrogram, dtype=tf.float32)\n\n\ndef time_masking(mel_spectrogram):\n    time_masking_para = 40, \n    time_mask_num = 1\n    \n    fbank_size = tf.shape(mel_spectrogram)\n    n, v = fbank_size[0], fbank_size[1]\n\n   \n    for i in range(time_mask_num):\n        t = tf.random.uniform([], minval=0, maxval=tf.squeeze(time_masking_para), dtype=tf.int32)\n        t0 = tf.random.uniform([], minval=0, maxval= n-t, dtype=tf.int32)\n\n        # mel_spectrogram[:, t0:t0 + t] = 0\n        mask = tf.concat((tf.ones(shape=(n-t0-t, v,1)),\n                          tf.zeros(shape=(t, v,1)),\n                          tf.ones(shape=(t0, v,1)),\n                          ), 0)\n        \n        mel_spectrogram = mel_spectrogram * mask\n    return tf.cast(mel_spectrogram, dtype=tf.float32)\n\n\ndef random_brightness(image):\n    return tf.image.random_brightness(image, 0.2)\n\ndef random_gamma(image):\n    return tf.image.random_contrast(image, lower = 0.1, upper = 0.3)\n\ndef random_flip_right(image):\n    return tf.image.random_flip_left_right(image)\n\ndef random_flip_up_down(image):\n    return tf.image.random_flip_left_right(image)\n\navailable_ops = [\n          frequency_masking ,\n          time_masking, \n          random_brightness, \n          random_flip_up_down,\n          random_flip_right \n         ]\n\ndef apply_augmentation(image, target):\n    num_layers = int(np.random.uniform(low = 0, high = 3))\n    \n    for layer_num in range(num_layers):\n        op_to_select = tf.random.uniform([], maxval=len(available_ops), dtype=tf.int32, seed = seed)\n        for (i, op_name) in enumerate(available_ops):\n            image = tf.cond(\n            tf.equal(i, op_to_select),\n            lambda selected_func=op_name,: selected_func(\n                image),\n            lambda: image)\n    return image, target","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training Data Pipeline"},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocess(image, target_or_rec_id):\n    \n    image = tf.image.grayscale_to_rgb(image)\n    image = tf.image.resize(image, [params.height,params.width])\n    image = tf.image.per_image_standardization(image)\n    return image , target_or_rec_id\n\n\ndef read_labeled_tfrecord(example_proto):\n    sample = tf.io.parse_single_example(example_proto, feature_description)\n    wav, _ = tf.audio.decode_wav(sample['wav'], desired_channels=1) # mono\n    target = tf.cast(sample['target'],tf.float32)\n    target = tf.squeeze(tf.one_hot([target,], depth = params.num_classes), axis = 0)\n    \n    tmin = tf.cast(sample['tmin'], tf.float32)\n    fmin = tf.cast(sample['fmin'], tf.float32)\n    tmax = tf.cast(sample['tmax'], tf.float32)\n    fmax = tf.cast(sample['fmax'], tf.float32)\n    \n    tmax_s = tmax * tf.cast(params.sample_rate, tf.float32)\n    tmin_s = tmin * tf.cast(params.sample_rate, tf.float32)\n    cut_s = tf.cast(CUT * params.sample_rate, tf.float32)\n    all_s = tf.cast(60 * params.sample_rate, tf.float32)\n    tsize_s = tmax_s - tmin_s\n    cut_min = tf.cast(\n    tf.maximum(0.0, \n        tf.minimum(tmin_s - (cut_s - tsize_s) / 2,\n                   tf.minimum(tmax_s + (cut_s - tsize_s) / 2, all_s) - cut_s)\n    ), tf.int32\n      )\n    cut_max = cut_min + CUT * params.sample_rate\n    wav = tf.squeeze(wav[cut_min : cut_max] )\n    \n    return wav, target\n\ndef read_unlabeled_tfrecord(example):\n    feature_description = {\n    'recording_id': tf.io.FixedLenFeature([], tf.string),\n    'audio_wav': tf.io.FixedLenFeature([], tf.string),\n    }\n    sample = tf.io.parse_single_example(example, feature_description)\n    wav, _ = tf.audio.decode_wav(sample['audio_wav'], desired_channels=1) # mono\n    recording_id = tf.reshape(tf.cast(sample['recording_id'] , tf.string), [1])\n#     wav = tf.squeeze(wav)\n\n    def _cut_audio(i):\n        _sample = {\n            'audio_wav': tf.reshape(wav[i*params.sample_rate*TIME:(i+1)*params.sample_rate*TIME], [params.sample_rate*TIME]),\n            'recording_id': sample['recording_id']\n        }\n        return _sample\n\n    return tf.map_fn(_cut_audio, tf.range(60//TIME), dtype={\n        'audio_wav': tf.float32,\n        'recording_id': tf.string\n    })","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_dataset(filenames, labeled = True, ordered = False , training = True):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # Diregarding data order. Order does not matter since we will be shuffling the data anyway\n    \n    ignore_order = tf.data.Options()\n    if not ordered:\n        # disable order, increase speed\n        ignore_order.experimental_deterministic = False \n        \n    # automatically interleaves reads from multiple files\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO )\n    # use data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord , num_parallel_calls = AUTO )\n    dataset = dataset.map(waveform_to_log_mel_spectrogram , num_parallel_calls = AUTO)   \n    if training:\n        dataset = dataset.map(apply_augmentation, num_parallel_calls = AUTO)\n    dataset = dataset.map(preprocess, num_parallel_calls = AUTO)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_dataset(filenames, training = True):\n    if training:\n        dataset = load_dataset(filenames , training = True)\n        dataset = dataset.shuffle(256).repeat()\n        dataset = dataset.batch(GLOBAL_BATCH_SIZE, drop_remainder = True)\n    else:\n        dataset = load_dataset(filenames , training = False)\n        dataset = dataset.batch(GLOBAL_BATCH_SIZE).cache()\n    \n    dataset = dataset.prefetch(AUTO)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# mel spectrogram visualization\n\ntrain_dataset = get_dataset(FILENAMES, training = True)\n\nplt.figure(figsize=(16,6))\nfor i, (wav, target) in enumerate(train_dataset.unbatch().take(4)):\n    plt.subplot(2,2,i+1)\n    plt.imshow(wav[:, :, 0])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Competition Metric"},{"metadata":{"trusted":true},"cell_type":"code","source":"# from https://www.kaggle.com/carlthome/l-lrap-metric-for-tf-keras\n\ndef _one_sample_positive_class_precisions(example):\n    y_true, y_pred = example\n    y_true = tf.reshape(y_true, tf.shape(y_pred))\n    retrieved_classes = tf.argsort(y_pred, direction='DESCENDING')\n#     shape = tf.shape(retrieved_classes)\n    class_rankings = tf.argsort(retrieved_classes)\n    retrieved_class_true = tf.gather(y_true, retrieved_classes)\n    retrieved_cumulative_hits = tf.math.cumsum(tf.cast(retrieved_class_true, tf.float32))\n\n    idx = tf.where(y_true)[:, 0]\n    i = tf.boolean_mask(class_rankings, y_true)\n    r = tf.gather(retrieved_cumulative_hits, i)\n    c = 1 + tf.cast(i, tf.float32)\n    precisions = r / c\n\n    dense = tf.scatter_nd(idx[:, None], precisions, [y_pred.shape[0]])\n    return dense\n\n# @tf.function\nclass LWLRAP(tf.keras.metrics.Metric):\n    def __init__(self, num_classes, name='lwlrap'):\n        super().__init__(name=name)\n\n        self._precisions = self.add_weight(\n            name='per_class_cumulative_precision',\n            shape=[num_classes],\n            initializer='zeros',\n        )\n\n        self._counts = self.add_weight(\n            name='per_class_cumulative_count',\n            shape=[num_classes],\n            initializer='zeros',\n        )\n\n    def update_state(self, y_true, y_pred, sample_weight=None):\n        precisions = tf.map_fn(\n            fn=_one_sample_positive_class_precisions,\n            elems=(y_true, y_pred),\n            dtype=(tf.float32),\n        )\n\n        increments = tf.cast(precisions > 0, tf.float32)\n        total_increments = tf.reduce_sum(increments, axis=0)\n        total_precisions = tf.reduce_sum(precisions, axis=0)\n\n        self._precisions.assign_add(total_precisions)\n        self._counts.assign_add(total_increments)        \n\n    def result(self):\n        per_class_lwlrap = self._precisions / tf.maximum(self._counts, 1.0)\n        per_class_weight = self._counts / tf.reduce_sum(self._counts)\n        overall_lwlrap = tf.reduce_sum(per_class_lwlrap * per_class_weight)\n        return overall_lwlrap\n\n    def reset_states(self):\n        self._precisions.assign(self._precisions * 0)\n        self._counts.assign(self._counts * 0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Stepwise Cosine Decay Callback"},{"metadata":{"trusted":true},"cell_type":"code","source":"def cosine_decay_with_warmup(global_step,\n                             learning_rate_base,\n                             total_steps,\n                             warmup_learning_rate=0.0,\n                             warmup_steps= 0,\n                             hold_base_rate_steps=0):\n \n    if total_steps < warmup_steps:\n        raise ValueError('total_steps must be larger or equal to '\n                     'warmup_steps.')\n    learning_rate = 0.5 * learning_rate_base * (1 + tf.cos(\n        np.pi *\n        (tf.cast(global_step, tf.float32) - warmup_steps - hold_base_rate_steps\n        ) / float(total_steps - warmup_steps - hold_base_rate_steps)))\n    if hold_base_rate_steps > 0:\n        learning_rate = tf.where(\n          global_step > warmup_steps + hold_base_rate_steps,\n          learning_rate, learning_rate_base)\n    if warmup_steps > 0:\n        if learning_rate_base < warmup_learning_rate:\n            raise ValueError('learning_rate_base must be larger or equal to '\n                         'warmup_learning_rate.')\n        slope = (learning_rate_base - warmup_learning_rate) / warmup_steps\n        warmup_rate = slope * tf.cast(global_step,\n                                    tf.float32) + warmup_learning_rate\n        learning_rate = tf.where(global_step < warmup_steps, warmup_rate,\n                               learning_rate)\n    return tf.where(global_step > total_steps, 0.0, learning_rate,\n                    name='learning_rate')\n\n\n#dummy example\nrng = [i for i in range(int(EPOCHS * STEPS_PER_EPOCH))]\nWARMUP_STEPS =  int(WARMUP_EPOCHS * STEPS_PER_EPOCH)\ny = [cosine_decay_with_warmup(x , LEARNING_RATE, len(rng), 1e-5, WARMUP_STEPS) for x in rng]\n\nsns.set(style='whitegrid')\nfig, ax = plt.subplots(figsize=(20, 6))\nplt.plot(rng, y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# to apply learning rate schedule stepwise we need to subclass keras callback\n# if we would have applied lr schedule epoch wise then it is not needed we can only call class learningrateschedule \n\nclass WarmUpCosineDecayScheduler(tf.keras.callbacks.Callback):\n\n    def __init__(self,\n                 learning_rate_base,\n                 total_steps,\n                 global_step_init=0,\n                 warmup_learning_rate=0.0,\n                 warmup_steps=0,\n                 hold_base_rate_steps=0,\n                 verbose=0):\n\n        super(WarmUpCosineDecayScheduler, self).__init__()\n        self.learning_rate_base = learning_rate_base\n        self.total_steps = total_steps\n        self.global_step = global_step_init\n        self.warmup_learning_rate = warmup_learning_rate\n        self.warmup_steps = warmup_steps\n        self.hold_base_rate_steps = hold_base_rate_steps\n        self.verbose = verbose\n        self.learning_rates = []\n\n    def on_batch_end(self, batch, logs=None):\n        self.global_step = self.global_step + 1\n        lr = K.get_value(self.model.optimizer.lr)\n        self.learning_rates.append(lr)\n\n    def on_batch_begin(self, batch, logs=None):\n        lr = cosine_decay_with_warmup(global_step=self.global_step,\n                                      learning_rate_base=self.learning_rate_base,\n                                      total_steps=self.total_steps,\n                                      warmup_learning_rate=self.warmup_learning_rate,\n                                      warmup_steps=self.warmup_steps,\n                                      hold_base_rate_steps=self.hold_base_rate_steps)\n        K.set_value(self.model.optimizer.lr, lr)\n        if self.verbose > 0:\n            print('\\nBatch %05d: setting learning '\n                  'rate to %s.' % (self.global_step + 1, lr.numpy()))\n            \n\ntotal_steps = int(EPOCHS * STEPS_PER_EPOCH)\n# Compute the number of warmup batches or steps.\nwarmup_steps = int(WARMUP_EPOCHS * STEPS_PER_EPOCH)\nwarmup_learning_rate = WARMUP_LEARNING_RATE","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Definition"},{"metadata":{"trusted":true},"cell_type":"code","source":"def RFCX_MODEL():\n    waveform = Input(shape=(None,None,3), dtype=tf.float32)\n    noisy_waveform = GaussianNoise(0.2)(waveform)\n    model = efn.EfficientNetB2(include_top=False, weights='imagenet',) \n    model_output = model(noisy_waveform)\n    model_output = GlobalAveragePooling2D()(model_output)\n    dense = Dropout(params.dropout)(model_output)\n    predictions = Dense(params.num_classes, activation = params.classifier_activation )(dense)\n    model = Model(\n      name='Efficientnet', inputs=waveform,\n      outputs=[predictions])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model():\n    with strategy.scope():\n        model = RFCX_MODEL()\n        model.summary()\n        model.compile(optimizer = 'adam',\n                                loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.1),\n                                metrics = [LWLRAP(num_classes = params.num_classes),\n                                ])\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training And Validation Loop"},{"metadata":{"trusted":true},"cell_type":"code","source":"skf = KFold(n_splits=N_FOLDS, shuffle=True, random_state=seed)\noof_pred = []; oof_labels = []; history_list = []\n\nfor fold,(idxT, idxV) in enumerate(skf.split(np.arange(10))):\n    if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n    print(f'\\nFOLD: {fold+1}')\n    print(f'TRAIN: {idxT} VALID: {idxV}')\n\n    # Create train and validation sets\n    TRAIN_FILENAMES = [FILENAMES[x] for x in idxT]\n    VALID_FILENAMES = [FILENAMES[x] for x in idxV]\n    np.random.shuffle(TRAIN_FILENAMES)\n    \n    train_dataset =  get_dataset(TRAIN_FILENAMES, training=True,)\n    validation_data= get_dataset(VALID_FILENAMES, training=False) \n\n    model = get_model()\n\n    model_path = f'RFCX_model_fold {fold}.h5'\n    early_stopping = EarlyStopping(monitor = 'val_lwlrap', mode = 'max', \n                       patience = PATIENCE, restore_best_weights=True, verbose=1)\n\n    # Create the Learning rate scheduler.\n    cosine_warm_up_lr = WarmUpCosineDecayScheduler(learning_rate_base= LEARNING_RATE,\n                                    total_steps= total_steps,\n                                    warmup_learning_rate= warmup_learning_rate,\n                                    warmup_steps= warmup_steps,\n                                    hold_base_rate_steps=0)\n\n    ## TRAIN\n    history = model.fit(train_dataset, \n                        steps_per_epoch=STEPS_PER_EPOCH, \n                        callbacks=[early_stopping, cosine_warm_up_lr], \n                        epochs=EPOCHS,  \n                        validation_data = validation_data,\n                        verbose = 2).history\n\n    history_list.append(history)\n    # Save last model weights\n    model.save_weights(model_path)\n\n# OOF predictions\n    ds_valid = get_dataset(VALID_FILENAMES, training = False)\n    oof_labels.append([target.numpy() for frame, target in iter(ds_valid.unbatch())])\n    x_oof = ds_valid.map(lambda frames, target: frames)\n    oof_pred.append(np.argmax(model.predict(x_oof), axis=-1))\n\n    ## RESULTS\n    print(f\"#### FOLD {fold+1} OOF Accuracy = {np.max(history['val_lwlrap']):.3f}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Plot curve"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_history(history):\n    plt.figure(figsize=(8,3))\n    plt.subplot(1,2,1)\n    plt.plot(history[\"loss\"])\n    plt.plot(history[\"val_loss\"])\n    plt.legend(['Train', 'Test'], loc='upper left')\n    plt.title(\"loss\")\n\n    plt.subplot(1,2,2)\n    plt.plot(history[\"lwlrap\"])\n    plt.plot(history[\"val_lwlrap\"])\n    plt.legend(['Train', 'Test'], loc='upper left')\n    plt.title(\"lwlrap\")\n    \nfor hist in history_list:\n    plot_history(hist)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_test_dataset(filenames, training = False):\n    \n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO )  \n    dataset = dataset.map(read_unlabeled_tfrecord , num_parallel_calls = AUTO ).unbatch()\n    dataset = dataset.map(lambda spec : waveform_to_log_mel_spectrogram(spec['audio_wav'], spec['recording_id']) , num_parallel_calls = AUTO)\n    dataset = dataset.map(preprocess, num_parallel_calls = AUTO)\n    return dataset.batch(GLOBAL_BATCH_SIZE*4).cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_predict = []\n\ntest_data = get_test_dataset(TEST_FILES, training = False)\ntest_audio = test_data.map(lambda frames, recording_id: frames)\n\nfor fold in range(N_FOLDS):\n    model.load_weights(f'./RFCX_model_fold {fold}.h5')\n    test_predict.append(model.predict(test_audio, verbose = 1 ))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"np.array(test_predict).shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SUB = pd.read_csv('../input/rfcx-species-audio-detection/sample_submission.csv')\n\npredict = np.array(test_predict).reshape(N_FOLDS, len(SUB), 60 // TIME, params.num_classes)\npredict = np.mean(np.max(predict ,axis = 2) , axis = 0)\n# predict = np.mean(predict, axis =  0)\n\nrecording_id = test_data.map(lambda frames, recording_id: recording_id).unbatch()\n# # all in one batch\ntest_ids = next(iter(recording_id.batch(len(SUB) * 60 // TIME))).numpy().astype('U').reshape(len(SUB), 60 // TIME)\n\npred_df = pd.DataFrame({ 'recording_id' : test_ids[:, 0],\n             **{f's{i}' : predict[:, i] for i in range(params.num_classes)} })","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_df.sort_values('recording_id', inplace = True) \npred_df.to_csv('submission.csv', index = False)    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}