{"cells":[{"metadata":{},"cell_type":"markdown","source":"# This notebook shows the custom training of RFCX data on Tensorflow TPU.\n \nIn my earlier [notebook](http://www.kaggle.com/ashusma/training-rfcx-tensorflow-tpu-effnet-b2) i have trained the model using keras.fit method, but if we want to take control of every little detail then we need to write custom loop so here i have trained model using optimized custom training loop.\n\n\nThe dataset used in this notebook is 10 fold Groupkfold tp only tfrecords that i have created [here](http://www.kaggle.com/ashusma/rfcx-audio-detection) and the simple script for the notebook is [this](https://www.kaggle.com/ashusma/rfcx-audio-creating-tfrecords?scriptVersionId=51531240).\n\nTraining description :\n\n* training with 10 sec clip around true positives\n* taking full spectrogram size \n* label smoothing, random_augmentation and gaussian noise\n* stepwise cosine decay with warm restarts and early stopping\n* for inference 10sec clip is used and then aggregrating and taking max of the audio wav prediction \n\n\n* version1 : efficientnet b4 , image_size = (512, 2000)\n* version3 : resnet50, image_size= (512, 1280)\n* version4 : some hyperparameters tweaking , and image_size = (256, 1024), no_augand label smoothing"},{"metadata":{"trusted":true},"cell_type":"code","source":"! pip install -q efficientnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import math, os, re, warnings, random , time\nfrom collections import namedtuple\nimport tensorflow as tf\nimport numpy as np\nimport pandas as pd\nimport librosa\nfrom kaggle_datasets import KaggleDatasets\nimport matplotlib.pyplot as plt\nfrom IPython.display import Audio\nfrom tensorflow.keras import Model, layers , optimizers\nfrom sklearn.model_selection import KFold\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler\nfrom tensorflow.keras.layers import GlobalAveragePooling2D, Input, Dense, Dropout, GaussianNoise, concatenate\nfrom tensorflow.keras.applications import ResNet50\nimport efficientnet.keras as efn\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# TPU Detection And Initialization"},{"metadata":{"trusted":true},"cell_type":"code","source":"# TPU or GPU detection\n# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print(f'Running on TPU {tpu.master()}')\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed=0):\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\nseed = 42\nseed_everything(seed)\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\n# train_files\n\nTRAIN_DATA_DIR = 'rfcx-audio-detection'\nTRAIN_GCS_PATH = KaggleDatasets().get_gcs_path(TRAIN_DATA_DIR)\nFILENAMES = tf.io.gfile.glob(TRAIN_GCS_PATH + '/tp*.tfrec')\n\n\n#test_files\nTEST_DATA_DIR = 'rfcx-species-audio-detection'\nTEST_GCS_PATH =  KaggleDatasets().get_gcs_path(TEST_DATA_DIR)\nTEST_FILES = tf.io.gfile.glob(TEST_GCS_PATH + '/tfrecords/test/*.tfrec')\n\nno_of_training_samples = count_data_items(FILENAMES)\n\nprint('num_training_samples are', no_of_training_samples)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CUT = 10\nTIME = 10\nEPOCHS = 25\nGLOBAL_BATCH_SIZE = 4 * REPLICAS\nLEARNING_RATE = 0.0015\nWARMUP_LEARNING_RATE = 1e-5\nWARMUP_EPOCHS = int(EPOCHS*0.1)\nPATIENCE = 10\nSTEPS_PER_EPOCH = 64\nN_FOLDS = 5\nNUM_TRAINING_SAMPLES = no_of_training_samples\n\nclass params:\n    sample_rate = 48000\n    stft_window_seconds: float = 0.025\n    stft_hop_seconds: float = 0.005\n    frame_length: int =  1200\n    mel_bands: int = 256\n    mel_min_hz: float = 50.0\n    mel_max_hz: float = 16000.0\n    log_offset: float = 0.001\n\n  \n    patch_bands = mel_bands\n    conv_padding: str = 'same'\n    batchnorm_center: bool = True\n    batchnorm_scale: bool = False\n    batchnorm_epsilon: float = 1e-4\n    num_classes: int = 24\n    dropout = 0.40\n    classifier_activation: str = 'sigmoid'\n    height = mel_bands\n    width = 1024","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature_description = {\n    'wav': tf.io.FixedLenFeature([], tf.string),\n    'recording_id': tf.io.FixedLenFeature([], tf.string ),\n    'target' : tf.io.FixedLenFeature([], tf.float32),\n    'song_id': tf.io.FixedLenFeature([], tf.float32),\n     'tmin' : tf.io.FixedLenFeature([], tf.float32),\n     'fmin' : tf.io.FixedLenFeature([], tf.float32),\n     'tmax' : tf.io.FixedLenFeature([], tf.float32),\n     'fmax' : tf.io.FixedLenFeature([], tf.float32),\n}\nfeature_dtype = {\n    'wav': tf.float32,\n    'recording_id': tf.string,\n    'target': tf.float32,\n    'song_id': tf.float32,\n    't_min': tf.float32,\n    'f_min': tf.float32,\n    't_max': tf.float32,\n    'f_max':tf.float32,\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def waveform_to_log_mel_spectrogram(waveform,target_or_rec_id):\n    \"\"\"Compute log mel spectrogram patches of a 1-D waveform.\"\"\"\n    # waveform has shape [<# samples>]\n\n    # Convert waveform into spectrogram using a Short-Time Fourier Transform.\n    # Note that tf.signal.stft() uses a periodic Hann window by default.\n\n    window_length_samples = int(\n      round(params.sample_rate * params.stft_window_seconds))\n    hop_length_samples = int(\n      round(params.sample_rate * params.stft_hop_seconds))\n    fft_length = 2 ** int(np.ceil(np.log(window_length_samples) / np.log(2.0)))\n#     print(fft_length, window_length_samples, hop_length_samples)\n    num_spectrogram_bins = fft_length // 2 + 1\n    magnitude_spectrogram = tf.abs(tf.signal.stft(\n      signals=waveform,\n      frame_length=params.frame_length,\n      frame_step=hop_length_samples,\n      fft_length= fft_length))\n    # magnitude_spectrogram has shape [<# STFT frames>, num_spectrogram_bins]\n\n    # Convert spectrogram into log mel spectrogram.\n    linear_to_mel_weight_matrix = tf.signal.linear_to_mel_weight_matrix(\n        num_mel_bins=params.mel_bands,\n        num_spectrogram_bins=num_spectrogram_bins,\n        sample_rate=params.sample_rate,\n        lower_edge_hertz=params.mel_min_hz,\n        upper_edge_hertz=params.mel_max_hz)\n    mel_spectrogram = tf.matmul(\n      magnitude_spectrogram, linear_to_mel_weight_matrix)\n    log_mel = tf.math.log(mel_spectrogram + params.log_offset)\n#     log_mel_spectrogram has shape [<# STFT frames>, params.mel_bands]\n    log_mel = tf.transpose(log_mel)\n    log_mel_spectrogram = tf.reshape(log_mel , [tf.shape(log_mel)[0] ,tf.shape(log_mel)[1],1])\n    \n    return log_mel_spectrogram, target_or_rec_id","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"def frequency_masking(mel_spectrogram):\n    \n    frequency_masking_para = 80, \n    frequency_mask_num = 2\n    \n    fbank_size = tf.shape(mel_spectrogram)\n#     print(fbank_size)\n    n, v = fbank_size[0], fbank_size[1]\n\n    for i in range(frequency_mask_num):\n        f = tf.random.uniform([], minval=0, maxval= tf.squeeze(frequency_masking_para), dtype=tf.int32)\n        v = tf.cast(v, dtype=tf.int32)\n        f0 = tf.random.uniform([], minval=0, maxval= tf.squeeze(v-f), dtype=tf.int32)\n\n        # warped_mel_spectrogram[f0:f0 + f, :] = 0\n        mask = tf.concat((tf.ones(shape=(n, v - f0 - f,1)),\n                          tf.zeros(shape=(n, f,1)),\n                          tf.ones(shape=(n, f0,1)),\n                          ),1)\n        mel_spectrogram = mel_spectrogram * mask\n    return tf.cast(mel_spectrogram, dtype=tf.float32)\n\n\ndef time_masking(mel_spectrogram):\n    time_masking_para = 40, \n    time_mask_num = 1\n    \n    fbank_size = tf.shape(mel_spectrogram)\n    n, v = fbank_size[0], fbank_size[1]\n\n   \n    for i in range(time_mask_num):\n        t = tf.random.uniform([], minval=0, maxval=tf.squeeze(time_masking_para), dtype=tf.int32)\n        t0 = tf.random.uniform([], minval=0, maxval= n-t, dtype=tf.int32)\n\n        # mel_spectrogram[:, t0:t0 + t] = 0\n        mask = tf.concat((tf.ones(shape=(n-t0-t, v,1)),\n                          tf.zeros(shape=(t, v,1)),\n                          tf.ones(shape=(t0, v,1)),\n                          ), 0)\n        \n        mel_spectrogram = mel_spectrogram * mask\n    return tf.cast(mel_spectrogram, dtype=tf.float32)\n\n\ndef random_brightness(image):\n    return tf.image.random_brightness(image, 0.2)\n\ndef random_gamma(image):\n    return tf.image.random_contrast(image, lower = 0.1, upper = 0.3)\n\ndef random_flip_right(image):\n    return tf.image.random_flip_left_right(image)\n\ndef random_flip_up_down(image):\n    return tf.image.random_flip_left_right(image)\n\navailable_ops = [\n          frequency_masking ,\n          time_masking, \n          random_brightness, \n          random_flip_up_down,\n          random_flip_right \n         ]\n\ndef apply_augmentation(image, target):\n    num_layers = int(np.random.uniform(low = 0, high = 3))\n    \n    for layer_num in range(num_layers):\n        op_to_select = tf.random.uniform([], maxval=len(available_ops), dtype=tf.int32)\n        for (i, op_name) in enumerate(available_ops):\n            image = tf.cond(\n            tf.equal(i, op_to_select),\n            lambda selected_func=op_name,: selected_func(\n                image),\n            lambda: image)\n    return image, target","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training Data Pipeline"},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocess(image, target_or_rec_id):\n    \n    image = tf.image.grayscale_to_rgb(image)\n    image = tf.image.resize(image, [params.height,params.width])\n    image = tf.image.per_image_standardization(image)\n    return image , target_or_rec_id\n\n\ndef read_labeled_tfrecord(example_proto):\n    sample = tf.io.parse_single_example(example_proto, feature_description)\n    wav, _ = tf.audio.decode_wav(sample['wav'], desired_channels=1) # mono\n    target = tf.cast(sample['target'],tf.float32)\n    target = tf.squeeze(tf.one_hot([target,], depth = params.num_classes), axis = 0)\n    \n    tmin = tf.cast(sample['tmin'], tf.float32)\n    fmin = tf.cast(sample['fmin'], tf.float32)\n    tmax = tf.cast(sample['tmax'], tf.float32)\n    fmax = tf.cast(sample['fmax'], tf.float32)\n    \n    tmax_s = tmax * tf.cast(params.sample_rate, tf.float32)\n    tmin_s = tmin * tf.cast(params.sample_rate, tf.float32)\n    cut_s = tf.cast(CUT * params.sample_rate, tf.float32)\n    all_s = tf.cast(60 * params.sample_rate, tf.float32)\n    tsize_s = tmax_s - tmin_s\n    cut_min = tf.cast(\n    tf.maximum(0.0, \n        tf.minimum(tmin_s - (cut_s - tsize_s) / 2,\n                   tf.minimum(tmax_s + (cut_s - tsize_s) / 2, all_s) - cut_s)\n    ), tf.int32\n      )\n    cut_max = cut_min + CUT * params.sample_rate\n    wav = tf.squeeze(wav[cut_min : cut_max] )\n    \n    return wav, target\n\ndef read_unlabeled_tfrecord(example):\n    feature_description = {\n    'recording_id': tf.io.FixedLenFeature([], tf.string),\n    'audio_wav': tf.io.FixedLenFeature([], tf.string),\n    }\n    sample = tf.io.parse_single_example(example, feature_description)\n    wav, _ = tf.audio.decode_wav(sample['audio_wav'], desired_channels=1) # mono\n    recording_id = tf.reshape(tf.cast(sample['recording_id'] , tf.string), [1])\n#     wav = tf.squeeze(wav)\n\n    def _cut_audio(i):\n        _sample = {\n            'audio_wav': tf.reshape(wav[i*params.sample_rate*TIME:(i+1)*params.sample_rate*TIME], [params.sample_rate*TIME]),\n            'recording_id': sample['recording_id']\n        }\n        return _sample\n\n    return tf.map_fn(_cut_audio, tf.range(60//TIME), dtype={\n        'audio_wav': tf.float32,\n        'recording_id': tf.string\n    })","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_dataset(filenames, labeled = True, ordered = False , training = True):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # Diregarding data order. Order does not matter since we will be shuffling the data anyway\n    \n    ignore_order = tf.data.Options()\n    if not ordered:\n        # disable order, increase speed\n        ignore_order.experimental_deterministic = False \n        \n    # automatically interleaves reads from multiple files\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO )\n    # use data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord , num_parallel_calls = AUTO )\n    dataset = dataset.map(waveform_to_log_mel_spectrogram , num_parallel_calls = AUTO)   \n#     if training:\n#         dataset = dataset.map(apply_augmentation, num_parallel_calls = AUTO)\n    dataset = dataset.map(preprocess, num_parallel_calls = AUTO)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_dataset(filenames, training = True):\n    if training:\n        dataset = load_dataset(filenames , training = True)\n        dataset = dataset.shuffle(256).repeat()\n        dataset = dataset.batch(GLOBAL_BATCH_SIZE, drop_remainder = True)\n    else:\n        dataset = load_dataset(filenames , training = False)\n        dataset = dataset.repeat().batch(GLOBAL_BATCH_SIZE)\n    \n    dataset = dataset.prefetch(AUTO)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# mel spectrogram visualization\n\ntrain_dataset = get_dataset(FILENAMES, training = True)\n\nplt.figure(figsize=(16,6))\nfor i, (wav, target) in enumerate(train_dataset.unbatch().take(4)):\n    plt.subplot(2,2,i+1)\n    plt.imshow(wav[:, :, 0])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Competition Metric"},{"metadata":{"trusted":true},"cell_type":"code","source":"# from https://www.kaggle.com/carlthome/l-lrap-metric-for-tf-keras\n@tf.function\ndef _one_sample_positive_class_precisions(example):\n    y_true, y_pred = example\n    y_true = tf.reshape(y_true, tf.shape(y_pred))\n    retrieved_classes = tf.argsort(y_pred, direction='DESCENDING')\n#     shape = tf.shape(retrieved_classes)\n    class_rankings = tf.argsort(retrieved_classes)\n    retrieved_class_true = tf.gather(y_true, retrieved_classes)\n    retrieved_cumulative_hits = tf.math.cumsum(tf.cast(retrieved_class_true, tf.float32))\n\n    idx = tf.where(y_true)[:, 0]\n    i = tf.boolean_mask(class_rankings, y_true)\n    r = tf.gather(retrieved_cumulative_hits, i)\n    c = 1 + tf.cast(i, tf.float32)\n    precisions = r / c\n\n    dense = tf.scatter_nd(idx[:, None], precisions, [y_pred.shape[0]])\n    return dense\n\n\nclass LWLRAP(tf.keras.metrics.Metric):\n    def __init__(self, num_classes, name='lwlrap'):\n        super().__init__(name=name)\n\n        self._precisions = self.add_weight(\n            name='per_class_cumulative_precision',\n            shape=[num_classes],\n            initializer='zeros',\n        )\n\n        self._counts = self.add_weight(\n            name='per_class_cumulative_count',\n            shape=[num_classes],\n            initializer='zeros',\n        )\n\n    def update_state(self, y_true, y_pred, sample_weight=None):\n        precisions = tf.map_fn(\n            fn=_one_sample_positive_class_precisions,\n            elems=(y_true, y_pred),\n            dtype=(tf.float32),\n        )\n\n        increments = tf.cast(precisions > 0, tf.float32)\n        total_increments = tf.reduce_sum(increments, axis=0)\n        total_precisions = tf.reduce_sum(precisions, axis=0)\n\n        self._precisions.assign_add(total_precisions)\n        self._counts.assign_add(total_increments)        \n\n    def result(self):\n        per_class_lwlrap = self._precisions / tf.maximum(self._counts, 1.0)\n        per_class_weight = self._counts / tf.reduce_sum(self._counts)\n        overall_lwlrap = tf.reduce_sum(per_class_lwlrap * per_class_weight)\n        return overall_lwlrap\n\n    def reset_states(self):\n        self._precisions.assign(self._precisions * 0)\n        self._counts.assign(self._counts * 0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Stepwise Cosine Decay Callback"},{"metadata":{"trusted":true},"cell_type":"code","source":"learning_rate_base = LEARNING_RATE\ntotal_steps = STEPS_PER_EPOCH * EPOCHS\nwarmup_learning_rate = WARMUP_LEARNING_RATE\nwarmup_steps= WARMUP_EPOCHS * STEPS_PER_EPOCH\n\n\n@tf.function\ndef cosine_decay_with_warmup(global_step,\n                             hold_base_rate_steps=0):\n\n    if total_steps < warmup_steps:\n        raise ValueError('total_steps must be larger or equal to '\n                     'warmup_steps.')\n    learning_rate = 0.5 * learning_rate_base * (1 + tf.cos(\n        np.pi *\n        (tf.cast(global_step, tf.float32) - warmup_steps - hold_base_rate_steps\n        ) / float(total_steps - warmup_steps - hold_base_rate_steps)))\n    if hold_base_rate_steps > 0:\n        learning_rate = tf.where(\n          global_step > warmup_steps + hold_base_rate_steps,\n          learning_rate, learning_rate_base)\n    if warmup_steps > 0:\n        if learning_rate_base < warmup_learning_rate:\n            raise ValueError('learning_rate_base must be larger or equal to '\n                         'warmup_learning_rate.')\n        slope = (learning_rate_base - warmup_learning_rate) / warmup_steps\n        warmup_rate = slope * tf.cast(global_step,\n                                    tf.float32) + warmup_learning_rate\n        learning_rate = tf.where(global_step < warmup_steps, warmup_rate,\n                               learning_rate)\n    return tf.where(global_step > total_steps, 0.0, learning_rate,\n                    name='learning_rate')\n\n\n#dummy example\nrng = [i for i in range(int(EPOCHS * STEPS_PER_EPOCH))]\nWARMUP_STEPS =  int(WARMUP_EPOCHS * STEPS_PER_EPOCH)\ny = [cosine_decay_with_warmup(x) for x in rng]\n\nsns.set(style='whitegrid')\nfig, ax = plt.subplots(figsize=(20, 6))\nplt.plot(rng, y)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Definition"},{"metadata":{"trusted":true},"cell_type":"code","source":"# custom model\n\nclass RFCX_MODEL(tf.keras.Model):\n    def __init__(self):\n        super(RFCX_MODEL , self).__init__()\n        self.gaussian_noise = GaussianNoise(0.05)\n        self.resnet_model = ResNet50(include_top=False, weights='imagenet')\n        self.model_output = GlobalAveragePooling2D()\n        self.dropout = Dropout(params.dropout)\n        self.predictions = Dense(params.num_classes, activation = params.classifier_activation )\n        \n    def call(self, inputs):\n        noisy_input = self.gaussian_noise(inputs)\n        resnet_output = self.resnet_model(noisy_input)\n        x = self.model_output(resnet_output)\n        x = self.dropout(x)\n        x = self.predictions(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training And Validation Loop"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_one_fold(train_dataset, valid_dataset):\n    print('Start fine-tuning!', flush=True)\n    # now we will distribute the dataset according to the strategy here it is TPUStrategy\n    train_dist_dataset = strategy.experimental_distribute_dataset(train_dataset)\n    valid_dist_dataset = strategy.experimental_distribute_dataset(valid_dataset)\n    start_time = epoch_start_time = time.time()\n\n    print(\"Steps per epoch:\", STEPS_PER_EPOCH)\n    History = namedtuple('History', 'history')\n    history = History(history={'train_loss': [], 'val_loss': [], 'train_lwlrap' : [], 'val_lwlrap' : []})\n    \n    train_iterator = iter(train_dist_dataset)\n    val_iterator = iter(valid_dist_dataset)\n    \n    \n    steps = 0\n    best_val_lwlrap = 0\n    for epoch in range(EPOCHS):\n        print('\\nEPOCH {:d}/{:d}'.format(epoch+1, EPOCHS))\n        # each iteration on train dist dataset returns per replica object dictionary containing data for each worker or replica\n        train_multiple_steps(train_iterator , tf.convert_to_tensor(STEPS_PER_EPOCH))\n        steps += STEPS_PER_EPOCH \n        \n\n        if (steps // STEPS_PER_EPOCH ) > epoch:\n            val_steps = 0\n            val_multiple_steps(val_iterator, tf.convert_to_tensor(VAL_STEPS))\n            val_steps += VAL_STEPS \n#             print('=' , end = ' ' , flush = True)\n\n            history.history['train_loss'].append(train_loss.result().numpy() / (STEPS_PER_EPOCH) )\n            history.history['val_loss'].append((val_loss.result().numpy() / val_steps))\n            history.history['train_lwlrap'].append(train_lwlrap.result().numpy())\n            history.history['val_lwlrap'].append(val_lwlrap.result().numpy())\n            \n            # show metrics\n            epoch_time = time.time() - epoch_start_time\n            \n            print('time: {:0.1f}s'.format(epoch_time),\n                  'train_loss: {:0.4f}'.format(history.history['train_loss'][-1]),\n                  'val_loss: {:0.4f}'.format(history.history['val_loss'][-1]),\n                  'train_lwlrap: {:0.4f}'.format(history.history['train_lwlrap'][-1]),\n                  'val_lwlrap: {:0.4f}'.format(history.history['val_lwlrap'][-1]),\n                  'lr : {:0.6f}'.format(cosine_decay_with_warmup(steps))\n                  )\n\n            # Early stopping monitor\n            if history.history['val_lwlrap'][-1] >= best_val_lwlrap:\n                best_val_lwlrap = history.history['val_lwlrap'][-1]\n                model.save_weights(model_path)\n                print(f'Saved model weights at \"{model_path}\"')\n                patience_cnt = 1\n            else:\n                patience_cnt += 1\n            if patience_cnt > PATIENCE:\n                print(f'Epoch {epoch:05d}: early stopping')\n                break              \n\n\n            # set up next epoch\n\n            epoch_start_time = time.time()\n            train_loss.reset_states()\n            val_loss.reset_states()\n            train_lwlrap.reset_states()\n            val_lwlrap.reset_states()  \n            \n    \n    history_list.append(history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kfold = KFold(n_splits = N_FOLDS,shuffle = True ,random_state = seed)\nhistory_list = [] \nfor fold, (train_idx, test_idx) in enumerate(kfold.split(np.arange(10))):\n    if tpu : tf.tpu.experimental.initialize_tpu_system()\n    K.clear_session()\n    train_files = [FILENAMES[i] for i in train_idx]\n    test_files = [FILENAMES[i] for i in test_idx]\n    VAL_STEPS = count_data_items(test_files) // GLOBAL_BATCH_SIZE \n    train_dataset = get_dataset(train_files, training = True)\n    valid_dataset = get_dataset(test_files, training = False)\n    print('fold', fold+1)\n    \n    @tf.function\n    def train_multiple_steps(train_iterator, steps):\n        def train_step(wav, target):\n\n            with tf.GradientTape() as tape:\n                predictions = model(wav, training = True)\n                total_loss = loss_fn(target, predictions)\n            gradients = tape.gradient(total_loss, model.trainable_variables)\n            optimizer.apply_gradients(list(zip(gradients, model.trainable_variables)))\n            train_loss.update_state(total_loss)\n            train_lwlrap.update_state(target, predictions)\n\n        for _ in tf.range(steps):\n             #strategy.run will distribute train_step and execute operation as specified by function on each replica \n            strategy.run(train_step, args = (next(train_iterator)))\n\n    @tf.function\n    def val_multiple_steps(val_iterator, val_steps):\n        def val_step(wav, target):\n            predictions = model(wav, training = False)\n            total_loss = loss_fn(target, predictions)\n            val_loss.update_state(total_loss)\n            val_lwlrap.update_state(target, predictions)\n        for _ in tf.range(val_steps):\n            strategy.run(val_step, args = (next(val_iterator)))\n\n    # defining model and variables under strategy to allow tpu to track them\n    with strategy.scope():\n        model = RFCX_MODEL() \n        model.build((None,None,None, 3))\n        model.summary()\n        loss = tf.keras.losses.BinaryCrossentropy(reduction= tf.keras.losses.Reduction.NONE)\n        loss_fn = lambda target, predict : tf.nn.compute_average_loss(loss(target, predict) , global_batch_size = GLOBAL_BATCH_SIZE)\n        class LRSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n            def __call__(self, step):\n                return cosine_decay_with_warmup(step)\n        optimizer = tf.keras.optimizers.Adam(learning_rate=LRSchedule())\n        train_loss = tf.keras.metrics.Sum()\n        val_loss = tf.keras.metrics.Sum()\n        train_lwlrap = LWLRAP(params.num_classes)\n        val_lwlrap = LWLRAP(params.num_classes)\n        \n       \n    model_path = f'RFCX_model_fold {fold}.h5'\n    # training for one fold\n    train_one_fold(train_dataset, valid_dataset)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Plot curve"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_history(history):\n    plt.figure(figsize=(8,3))\n    plt.subplot(1,2,1)\n    plt.plot(history.history[\"train_loss\"])\n    plt.plot(history.history[\"val_loss\"])\n    plt.legend(['Train', 'Test'], loc='upper left')\n    plt.title(\"loss\")\n\n    plt.subplot(1,2,2)\n    plt.plot(history.history[\"train_lwlrap\"])\n    plt.plot(history.history[\"val_lwlrap\"])\n    plt.legend(['Train', 'Test'], loc='upper left')\n    plt.title(\"lwlrap\")\n    \nfor hist in history_list:\n#     print(hist)\n    plot_history(hist)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_test_dataset(filenames, training = False):\n    \n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO )  \n    dataset = dataset.map(read_unlabeled_tfrecord , num_parallel_calls = AUTO ).unbatch()\n    dataset = dataset.map(lambda spec : waveform_to_log_mel_spectrogram(spec['audio_wav'], spec['recording_id']) , num_parallel_calls = AUTO)\n    dataset = dataset.map(preprocess, num_parallel_calls = AUTO)\n    return dataset.batch(GLOBAL_BATCH_SIZE*4).cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_predict = []\n\ntest_data = get_test_dataset(TEST_FILES, training = False)\ntest_audio = test_data.map(lambda frames, recording_id: frames)\n\nfor fold in range(N_FOLDS):\n    model.load_weights(f'./RFCX_model_fold {fold}.h5')\n    test_predict.append(model.predict(test_audio, verbose = 1 ))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"np.array(test_predict).shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SUB = pd.read_csv('../input/rfcx-species-audio-detection/sample_submission.csv')\n\npredict = np.array(test_predict).reshape(N_FOLDS, len(SUB), 60 // TIME, params.num_classes)\npredict = np.mean(np.max(predict ,axis = 2) , axis = 0)\n# predict = np.mean(predict, axis =  0)\n\nrecording_id = test_data.map(lambda frames, recording_id: recording_id).unbatch()\n# # all in one batch\ntest_ids = next(iter(recording_id.batch(len(SUB) * 60 // TIME))).numpy().astype('U').reshape(len(SUB), 60 // TIME)\n\npred_df = pd.DataFrame({ 'recording_id' : test_ids[:, 0],\n             **{f's{i}' : predict[:, i] for i in range(params.num_classes)} })","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_df.sort_values('recording_id', inplace = True) \npred_df.to_csv('submission.csv', index = False)    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_df","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}