{"cells":[{"metadata":{},"cell_type":"markdown","source":"Post-competition edit:\nAdding postprocessing from Chris Deotte https://www.kaggle.com/c/rfcx-species-audio-detection/discussion/220389"},{"metadata":{},"cell_type":"markdown","source":"# This notebook shows the training of RFCX data on Tensorflow TPU\n\nThe dataset used in this notebook is 10 fold Groupkfold tp only tfrecords that i have created [here](http://www.kaggle.com/ashusma/rfcx-audio-detection) and the simple script for the notebook is [this](https://www.kaggle.com/ashusma/rfcx-audio-creating-tfrecords?scriptVersionId=51531240).\n\nTraining description :\n\n* training with 10 sec clip around true positives\n* taking full spectrogram size \n* random augmentation and gaussian noise\n* label smoothing\n* stepwise cosine decay with warm restarts and early stopping\n* for inference 10sec clip is used and then aggregrating and taking max of the audio wav prediction \n\n\nSince this notebook uses tpu accelerator having 128 gb (16 gb each replica) so for efficient use i have done following optimization :\n* increased the spectrogram size\n* caching validation and test set as both are small in number for faster computation\n* wrapped all user defined function with map that allow parallel computation\n* reduced the python overhead \n* tensorflow 2.3 and above has argument execution per step in model.compile function that significantly improves performance by running multiple steps within tpu worker. but since kaggle has not updated tf version we cannot take advantage of that but one can try it on google colab\n* above step can also be done by using custom training loop"},{"metadata":{"trusted":true},"cell_type":"code","source":"! pip install -q efficientnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN = True\nif TRAIN:\n    MODEL_DIR = \".\"\nelse:\n    MODEL_DIR = \"/kaggle/input/rfcx-train-fp-weights\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import math, os, re, warnings, random\nimport tensorflow as tf\nimport numpy as np\nimport pandas as pd\nimport librosa\nfrom kaggle_datasets import KaggleDatasets\nimport matplotlib.pyplot as plt\nfrom IPython.display import Audio\nfrom tensorflow.keras import Model, layers\nfrom sklearn.model_selection import KFold\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler\nfrom tensorflow.keras.layers import GlobalAveragePooling2D, GlobalMaxPooling2D, Input, Dense, Dropout, GaussianNoise, Conv2D, SpatialDropout2D, Lambda\nfrom tensorflow.keras.applications import ResNet50\nimport tensorflow_addons as tfa\nimport efficientnet.keras as efn\nimport seaborn as sns\nfrom functools import partial","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# TPU Detection And Initialization"},{"metadata":{"trusted":true},"cell_type":"code","source":"# TPU or GPU detection\n# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print(f'Running on TPU {tpu.master()}')\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed=0):\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\nseed = 42\nseed_everything(seed)\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\n# train_files\n\nTRAIN_DATA_DIR = 'rfcx-audio-detection'\nTRAIN_GCS_PATH = KaggleDatasets().get_gcs_path(TRAIN_DATA_DIR)\n#FILENAMES = tf.io.gfile.glob(TRAIN_GCS_PATH + '/*.tfrec')\n\n\n#test_files\nTEST_DATA_DIR = 'rfcx-species-audio-detection'\nTEST_GCS_PATH =  KaggleDatasets().get_gcs_path(TEST_DATA_DIR)\nTEST_FILES = tf.io.gfile.glob(TEST_GCS_PATH + '/tfrecords/test/*.tfrec')\n\nFILENAMES = tf.io.gfile.glob(TEST_GCS_PATH + '/tfrecords/train/*.tfrec')\nno_of_training_samples = count_data_items(FILENAMES)\n#no_of_fp_samples = count_data_items(FP_FILENAMES)\n\n\nprint('num_training_samples are', no_of_training_samples)\n#print('no_of_fp_samples are', no_of_fp_samples)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#dataset = tf.data.TFRecordDataset(FP_FILENAMES[0], num_parallel_reads = AUTO )\n#for raw_record in dataset.take(1):\n#    example = tf.train.Example()\n#    example.ParseFromString(raw_record.numpy())\n#    print(example.features.feature[\"recording_id\"].numpy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import defaultdict\ndd = defaultdict(list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#raw_dataset = tf.data.TFRecordDataset(FILENAMES[0])\n#for raw_record in raw_dataset:\n#    example = tf.train.Example()\n#    example.ParseFromString(raw_record.numpy())\n#    dd[example.features.feature[\"recording_id\"].bytes_list.value[0]].append(example.features.feature[\"target\"].float_list.value[0])\n#    #print(example.features.feature[\"recording_id\"].bytes_list.value[0],example.features.feature[\"target\"].float_list.value[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#print(dd)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#print(example.features.feature[\"recording_id\"],example.features.feature[\"target\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"GLOBAL_BATCH_SIZE = 4 * REPLICAS\nSTEPS_PER_EPOCH = no_of_training_samples // GLOBAL_BATCH_SIZE\nprint(STEPS_PER_EPOCH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CUT = 5\nSTRIDE = 3\nEPOCHS = 25\n\nAUG_BATCH = GLOBAL_BATCH_SIZE\nVAL_BATCH_SIZE = GLOBAL_BATCH_SIZE\nLEARNING_RATE = 0.0015\nWARMUP_LEARNING_RATE = 1e-5\nWARMUP_EPOCHS = int(EPOCHS*0.1)\nPATIENCE = EPOCHS\n#STEPS_PER_EPOCH = 64\nN_FOLDS = 4\nNUM_TRAINING_SAMPLES = no_of_training_samples\n\n\n\n\nclass params:\n    sample_rate = 48000\n    stft_window_seconds: float = 0.025\n    stft_hop_seconds: float = 0.005\n    frame_length: int =  1200    \n    mel_bands: int = 512\n    mel_min_hz: float = 50.0\n    mel_max_hz: float = 24000.0\n    log_offset: float = 0.001\n    patch_window_seconds: float = 0.96\n    patch_hop_seconds: float = 0.48\n\n  \n    patch_frames =  int(round(patch_window_seconds / stft_hop_seconds))\n\n  \n    patch_bands = mel_bands\n    height = mel_bands\n    width = CUT * 200\n    num_classes: int = 24\n    dropout = 0.35\n    classifier_activation: str = 'sigmoid'\n        \nIMAGE_SIZE = [params.height, params.width]\nCLASSES = params.num_classes\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature_description = {\n    'wav': tf.io.FixedLenFeature([], tf.string),\n    'recording_id': tf.io.FixedLenFeature([], tf.string ),\n    'target' : tf.io.FixedLenFeature([], tf.float32),\n    'song_id': tf.io.FixedLenFeature([], tf.float32),\n     'tmin' : tf.io.FixedLenFeature([], tf.float32),\n     'fmin' : tf.io.FixedLenFeature([], tf.float32),\n     'tmax' : tf.io.FixedLenFeature([], tf.float32),\n     'fmax' : tf.io.FixedLenFeature([], tf.float32),\n     'is_tp' : tf.io.FixedLenFeature([], tf.int32)\n}\nfeature_dtype = {\n    'wav': tf.float32,\n    'recording_id': tf.string,\n    'target': tf.float32,\n    'song_id': tf.float32,\n    't_min': tf.float32,\n    'f_min': tf.float32,\n    't_max': tf.float32,\n    'f_max':tf.float32,\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def waveform_to_log_mel_spectrogram(waveform,target_or_rec_id):\n    \"\"\"Compute log mel spectrogram patches of a 1-D waveform.\"\"\"\n    # waveform has shape [<# samples>]\n\n    # Convert waveform into spectrogram using a Short-Time Fourier Transform.\n    # Note that tf.signal.stft() uses a periodic Hann window by default.\n\n    window_length_samples = int(\n      round(params.sample_rate * params.stft_window_seconds))\n    hop_length_samples = int(\n      round(params.sample_rate * params.stft_hop_seconds))\n    fft_length = 2 ** int(np.ceil(np.log(window_length_samples) / np.log(2.0)))\n#     print(fft_length, window_length_samples, hop_length_samples)\n    num_spectrogram_bins = fft_length // 2 + 1\n    magnitude_spectrogram = tf.abs(tf.signal.stft(\n      signals=waveform,\n      frame_length=params.frame_length,\n      frame_step=hop_length_samples,\n      fft_length= fft_length))\n    # magnitude_spectrogram has shape [<# STFT frames>, num_spectrogram_bins]\n\n    # Convert spectrogram into log mel spectrogram.\n    linear_to_mel_weight_matrix = tf.signal.linear_to_mel_weight_matrix(\n        num_mel_bins=params.mel_bands,\n        num_spectrogram_bins=num_spectrogram_bins,\n        sample_rate=params.sample_rate,\n        lower_edge_hertz=params.mel_min_hz,\n        upper_edge_hertz=params.mel_max_hz)\n    mel_spectrogram = tf.matmul(\n      magnitude_spectrogram, linear_to_mel_weight_matrix)\n    log_mel = tf.math.log(mel_spectrogram + params.log_offset)\n#     log_mel_spectrogram has shape [<# STFT frames>, params.mel_bands]\n    log_mel = tf.transpose(log_mel)\n    log_mel_spectrogram = tf.reshape(log_mel , [tf.shape(log_mel)[0] ,tf.shape(log_mel)[1],1])\n    # Frame spectrogram (shape [<# STFT frames>, params.mel_bands]) into patches\n    # (the input examples). Only complete frames are emitted, so if there is\n    # less than params.patch_window_seconds of waveform then nothing is emitted\n    # (to avoid this, zero-pad before processing).\n    spectrogram_hop_length_samples = int(\n      round(params.sample_rate * params.stft_hop_seconds))\n    spectrogram_sample_rate = params.sample_rate / spectrogram_hop_length_samples\n    patch_window_length_samples = int(\n      round(spectrogram_sample_rate * params.patch_window_seconds))\n    patch_hop_length_samples = int(\n      round(spectrogram_sample_rate * params.patch_hop_seconds))\n    features = tf.signal.frame(\n        signal=log_mel_spectrogram,\n        frame_length=patch_window_length_samples,\n        frame_step=patch_hop_length_samples,\n        axis=0)\n    # features has shape [<# patches>, <# STFT frames in an patch>, params.mel_bands]\n    \n    return log_mel_spectrogram, target_or_rec_id","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"def frequency_masking(mel_spectrogram):\n    \n    frequency_masking_para = 80, \n    frequency_mask_num = 2\n    \n    fbank_size = tf.shape(mel_spectrogram)\n#     print(fbank_size)\n    n, v = fbank_size[0], fbank_size[1]\n\n    for i in range(frequency_mask_num):\n        f = tf.random.uniform([], minval=0, maxval= tf.squeeze(frequency_masking_para), dtype=tf.int32)\n        v = tf.cast(v, dtype=tf.int32)\n        f0 = tf.random.uniform([], minval=0, maxval= tf.squeeze(v-f), dtype=tf.int32)\n\n        # warped_mel_spectrogram[f0:f0 + f, :] = 0\n        mask = tf.concat((tf.ones(shape=(n, v - f0 - f,1)),\n                          tf.zeros(shape=(n, f,1)),\n                          tf.ones(shape=(n, f0,1)),\n                          ),1)\n        mel_spectrogram = mel_spectrogram * mask\n    return tf.cast(mel_spectrogram, dtype=tf.float32)\n\n\ndef time_masking(mel_spectrogram):\n    time_masking_para = 40, \n    time_mask_num = 1\n    \n    fbank_size = tf.shape(mel_spectrogram)\n    n, v = fbank_size[0], fbank_size[1]\n\n   \n    for i in range(time_mask_num):\n        t = tf.random.uniform([], minval=0, maxval=tf.squeeze(time_masking_para), dtype=tf.int32)\n        t0 = tf.random.uniform([], minval=0, maxval= n-t, dtype=tf.int32)\n\n        # mel_spectrogram[:, t0:t0 + t] = 0\n        mask = tf.concat((tf.ones(shape=(n-t0-t, v,1)),\n                          tf.zeros(shape=(t, v,1)),\n                          tf.ones(shape=(t0, v,1)),\n                          ), 0)\n        \n        mel_spectrogram = mel_spectrogram * mask\n    return tf.cast(mel_spectrogram, dtype=tf.float32)\n\n\ndef random_brightness(image):\n    return tf.image.random_brightness(image, 0.2)\n\ndef random_gamma(image):\n    return tf.image.random_contrast(image, lower = 0.1, upper = 0.3)\n\ndef random_flip_right(image):\n    return tf.image.random_flip_left_right(image)\n\ndef random_flip_up_down(image):\n    return tf.image.random_flip_left_right(image)\n\ndef shift_scale_shear(image):\n    a0 = tf.random.uniform([], minval=0.9, maxval=1.1)\n    a1 = tf.random.uniform([], minval=-0.05, maxval=0.05)\n    a2 = tf.random.uniform([], minval=-params.width * 0.1, maxval = 0.1 * params.width)\n    \n    b0 = tf.random.uniform([], minval=-0.05, maxval=0.05)\n    b1 = tf.random.uniform([], minval=0.9, maxval=1.1)\n    b2 = tf.random.uniform([], minval=-params.height * 0.1, maxval = 0.1 * params.height)\n    \n    return tfa.image.transform(image, [a0, a1, a2, b0, b1, b2, 0, 0], interpolation=\"bilinear\", fill_mode=\"reflect\")\n\ndef random_rotate(image):\n    angle = tf.random.uniform([], minval=-10, maxval=10)\n    return tfa.image.rotate(image, angle, interpolation=\"nearest\", fill_mode=\"reflect\")\n    \n\navailable_ops = [\n          frequency_masking ,\n          time_masking, \n          random_brightness, \n          random_flip_up_down,\n          random_flip_right, \n          shift_scale_shear,\n          random_rotate\n         ]\n\ndef apply_augmentation(image, target):\n    num_layers = int(np.random.uniform(low = 0, high = 3))\n    \n    for layer_num in range(num_layers):\n        op_to_select = tf.random.uniform([], maxval=len(available_ops), dtype=tf.int32, seed = seed)\n        for (i, op_name) in enumerate(available_ops):\n            image = tf.cond(\n            tf.equal(i, op_to_select),\n            lambda selected_func=op_name,: selected_func(\n                image),\n            lambda: image)\n    return image, target","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training Data Pipeline"},{"metadata":{"trusted":true},"cell_type":"code","source":"def string_split_semicolon(column):\n    split_labels_sc = tf.strings.split(column, sep=';')\n    return split_labels_sc\n\ndef string_split_comma(column):\n    split_labels_c = tf.strings.split(column, sep=',')\n    return split_labels_c\n\ndef get_label_info(label_info):\n    first_split = string_split_semicolon(label_info)\n    remove_quotes = tf.strings.regex_replace(first_split, '\"', \"\")\n    label_info = string_split_comma(remove_quotes)\n    return label_info","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_original_tfrecord(example):\n    TFREC_FORMAT = {\n        'audio_wav': tf.io.FixedLenFeature([], tf.string), \n        'recording_id': tf.io.FixedLenFeature([], tf.string), \n        'label_info': tf.io.FixedLenFeature([], tf.string, default_value='-1,-1,0,0,0,0,1'), \n    }\n        \n    example = tf.io.parse_single_example(example, TFREC_FORMAT)\n    audio = example['audio_wav']\n    \n    # Break down 'label_info' into the data columns\n    label_info = get_label_info(example['label_info'])\n    species_id = tf.strings.to_number(tf.gather_nd(label_info, [0, 0]), tf.int32)\n#     songtype_id = tf.strings.to_number(tf.gather_nd(label_info, [0, 1]), tf.int32)\n    tmin = tf.strings.to_number(tf.gather_nd(label_info, [0, 2]))\n    fmin = tf.strings.to_number(tf.gather_nd(label_info, [0, 3]))\n    tmax = tf.strings.to_number(tf.gather_nd(label_info, [0, 4]))\n    fmax = tf.strings.to_number(tf.gather_nd(label_info, [0, 5]))\n    is_tp = tf.strings.to_number(tf.gather_nd(label_info, [0, 6]), tf.int32)\n\n    features = {'wav': audio, \n                'recording_id': example['recording_id'], \n                'target': species_id, \n                'tmin': tmin,\n                'tmax': tmax,\n                'fmin': fmin,\n                'fmax': fmax, \n                'is_tp': is_tp\n               }\n    return features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocess(image, target_or_rec_id):\n    \n    image = tf.image.grayscale_to_rgb(image)\n    image = tf.image.resize(image, [params.height,params.width])\n    image = tf.image.per_image_standardization(image)\n    return image , target_or_rec_id\n\n\ndef _cut_wav(sample, random_shift):\n    wav, _ = tf.audio.decode_wav(sample['wav'], desired_channels=1) # mono\n    tmin = tf.cast(sample['tmin'], tf.float32)\n    fmin = tf.cast(sample['fmin'], tf.float32)\n    tmax = tf.cast(sample['tmax'], tf.float32)\n    fmax = tf.cast(sample['fmax'], tf.float32)\n    \n    tmax_s = tmax * tf.cast(params.sample_rate, tf.float32)\n    tmin_s = tmin * tf.cast(params.sample_rate, tf.float32)\n    \n    \n    cut_s = tf.cast(CUT * params.sample_rate, tf.float32)\n    tsize_s = tmax_s - tmin_s\n    \n    center_s = (tmax_s + tmin_s) / 2\n    \n    if random_shift:\n        shift_limit = tf.math.abs(cut_s - tsize_s)\n        shift_s = tf.cast( tf.random.uniform([],-shift_limit/2,shift_limit/2),tf.float32)\n        center_s += shift_s\n    \n    all_s = tf.cast(60 * params.sample_rate, tf.float32)\n    \n    cut_min = tf.maximum(0., center_s - cut_s / 2)\n    cut_min = tf.minimum(cut_min, all_s - cut_s)\n    cut_min = tf.cast(cut_min, tf.int32)\n    \n    cut_max = cut_min + CUT * params.sample_rate\n    wav = tf.squeeze(wav[cut_min : cut_max] )\n    \n    return wav\n    \n\ndef read_labeled_tfrecord(sample, random_shift = False):\n    #sample = tf.io.parse_single_example(example_proto, feature_description)\n    \n    target = tf.cast(sample['target'],tf.float32)\n    target = tf.squeeze(tf.one_hot([target,], depth = params.num_classes), axis = 0)\n    \n    wav = _cut_wav(sample, random_shift)\n    \n    return wav, target\n\ndef read_labeled_tfrecord_fp(sample, random_shift = False):\n    #sample = tf.io.parse_single_example(example_proto, feature_description)\n    #wav, _ = tf.audio.decode_wav(sample['wav'], desired_channels=1) # mono\n    target = tf.cast(sample['target'],tf.float32)\n    target = tf.squeeze(tf.one_hot([target,], depth = params.num_classes), axis = 0)\n    \n    target =  target - 1\n    \n    wav = _cut_wav(sample, random_shift)\n    \n    return wav, target\n    \n    \n\ndef read_unlabeled_tfrecord(example):\n    feature_description = {\n    'recording_id': tf.io.FixedLenFeature([], tf.string),\n    'audio_wav': tf.io.FixedLenFeature([], tf.string),\n    }\n    sample = tf.io.parse_single_example(example, feature_description)\n    wav, _ = tf.audio.decode_wav(sample['audio_wav'], desired_channels=1) # mono\n    recording_id = tf.reshape(tf.cast(sample['recording_id'] , tf.string), [1])\n#     wav = tf.squeeze(wav)\n\n    def _cut_audio(i):\n        _sample = {\n            'audio_wav': tf.reshape(wav[i*params.sample_rate*STRIDE:i*params.sample_rate*STRIDE + params.sample_rate*CUT], [params.sample_rate*CUT]),\n            'recording_id': sample['recording_id']\n        }\n        return _sample\n\n    return tf.map_fn(_cut_audio, tf.range((60 - CUT)//STRIDE + 1), dtype={\n        'audio_wav': tf.float32,\n        'recording_id': tf.string\n    })","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def filter_tp_tfrecord(sample):\n    \n    return tf.cast(sample['is_tp'], tf.bool)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def cutmix(image, label, PROBABILITY = 1.0):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with cutmix applied\n    DIM = IMAGE_SIZE[0]\n    DIM1 = IMAGE_SIZE[1]\n    \n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        # DO CUTMIX WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.int32)\n        # CHOOSE RANDOM IMAGE TO CUTMIX WITH\n        k = tf.cast( tf.random.uniform([],0,AUG_BATCH),tf.int32)\n        # CHOOSE RANDOM LOCATION\n        x = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        y = tf.cast( tf.random.uniform([],0,DIM1),tf.int32)\n        b = tf.random.uniform([],0,1) # this is beta dist with alpha=1.0\n        WIDTH = tf.cast( DIM * tf.math.sqrt(1-b),tf.int32) * P\n        HEIGHT = tf.cast( DIM1 * tf.math.sqrt(1-b),tf.int32) * P\n        ya = tf.math.maximum(0,y-HEIGHT//2)\n        yb = tf.math.minimum(DIM1,y+HEIGHT//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM,x+WIDTH//2)\n        # MAKE CUTMIX IMAGE\n        one = image[j,ya:yb,0:xa,:]\n        two = image[k,ya:yb,xa:xb,:]\n        three = image[j,ya:yb,xb:DIM1,:]\n        middle = tf.concat([one,two,three],axis=1)\n        img = tf.concat([image[j,0:ya,:,:],middle,image[j,yb:DIM,:,:]],axis=0)\n        imgs.append(img)\n        # MAKE CUTMIX LABEL\n        a = tf.cast(WIDTH*WIDTH/DIM/DIM1,tf.float32)\n        if len(label.shape)==1:\n            lab1 = tf.one_hot(label[j],CLASSES)\n            lab2 = tf.one_hot(label[k],CLASSES)\n        else:\n            lab1 = label[j,]\n            lab2 = label[k,]\n        labs.append((1-a)*lab1 + a*lab2)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM, DIM1,3))\n    label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image2,label2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def mixup(image, label, PROBABILITY = 1.0):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with mixup applied\n    DIM = IMAGE_SIZE[0]\n    DIM1 = IMAGE_SIZE[1]\n    \n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        # DO MIXUP WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.float32)\n        # CHOOSE RANDOM\n        k = tf.cast( tf.random.uniform([],0,AUG_BATCH),tf.int32)\n        a = tf.random.uniform([],0,1)*P # this is beta dist with alpha=1.0\n        # MAKE MIXUP IMAGE\n        img1 = image[j,]\n        img2 = image[k,]\n        imgs.append((1-a)*img1 + a*img2)\n        # MAKE CUTMIX LABEL\n        if len(label.shape)==1:\n            lab1 = tf.one_hot(label[j],CLASSES)\n            lab2 = tf.one_hot(label[k],CLASSES)\n        else:\n            lab1 = label[j,]\n            lab2 = label[k,]\n        labs.append((1-a)*lab1 + a*lab2)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM, DIM1,3))\n    label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image2,label2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def transform(image,label):\n    # THIS FUNCTION APPLIES BOTH CUTMIX AND MIXUP\n    DIM = IMAGE_SIZE[0]\n    DIM1 = IMAGE_SIZE[1]\n    SWITCH = 0.5\n    CUTMIX_PROB = 0.666\n    MIXUP_PROB = 0.666\n    # FOR SWITCH PERCENT OF TIME WE DO CUTMIX AND (1-SWITCH) WE DO MIXUP\n    image2, label2 = cutmix(image, label, CUTMIX_PROB)\n    image3, label3 = mixup(image, label, MIXUP_PROB)\n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        P = tf.cast( tf.random.uniform([],0,1)<=SWITCH, tf.float32)\n        imgs.append(P*image2[j,]+(1-P)*image3[j,])\n        labs.append(P*label2[j,]+(1-P)*label3[j,])\n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image4 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM1,3))\n    label4 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image4,label4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_dataset(filenames, labeled = True, ordered = False , training = True):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # Diregarding data order. Order does not matter since we will be shuffling the data anyway\n    \n    ignore_order = tf.data.Options()\n    if not ordered:\n        # disable order, increase speed\n        ignore_order.experimental_deterministic = False \n        \n    # automatically interleaves reads from multiple files\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO )\n    dataset = dataset.map(read_original_tfrecord, num_parallel_calls = AUTO )\n    \n    # use data as soon as it streams in, rather than in its original order\n    if training:\n        dataset_tp = dataset.filter(filter_tp_tfrecord)\n        dataset_fp = dataset.filter(lambda x: not filter_tp_tfrecord(x))\n\n        #dataset_fp = dataset_fp.map(partial(read_labeled_tp_record_filter_bg, FP=True) , num_parallel_calls = AUTO )\n        #dataset_tp = dataset_tp.map(read_labeled_tp_record_filter_bg, num_parallel_calls = AUTO )\n        \n        dataset_fp = dataset_fp.map(partial(read_labeled_tfrecord_fp, random_shift=False) , num_parallel_calls = AUTO )\n        dataset_tp = dataset_tp.map(partial(read_labeled_tfrecord, random_shift=False) , num_parallel_calls = AUTO )\n        \n        dataset_tp = dataset_tp.map(waveform_to_log_mel_spectrogram , num_parallel_calls = AUTO)   \n        dataset_tp = dataset_tp.map(apply_augmentation, num_parallel_calls = AUTO)\n        dataset_tp = dataset_tp.map(preprocess, num_parallel_calls = AUTO)\n        \n        dataset_fp = dataset_fp.map(waveform_to_log_mel_spectrogram , num_parallel_calls = AUTO)   \n        dataset_fp = dataset_fp.map(apply_augmentation, num_parallel_calls = AUTO)\n        dataset_fp = dataset_fp.map(preprocess, num_parallel_calls = AUTO)\n        \n    \n        \n        dataset_tp = dataset_tp.batch(GLOBAL_BATCH_SIZE, drop_remainder = True)\n        dataset_tp = dataset_tp.map(transform, num_parallel_calls=AUTO)\n        dataset_tp = dataset_tp.unbatch()\n\n        dataset_fp = dataset_fp.shuffle(256)\n        dataset = dataset_tp.concatenate(dataset_fp)\n  \n        dataset = dataset.shuffle(2048).repeat()\n        dataset = dataset.batch(GLOBAL_BATCH_SIZE)\n        #dataset = dataset.prefetch(AUTO)\n    else:\n        dataset_tp = dataset.filter(filter_tp_tfrecord)\n        dataset = dataset_tp.map(read_labeled_tfrecord , num_parallel_calls = AUTO )\n    \n        dataset = dataset.map(waveform_to_log_mel_spectrogram , num_parallel_calls = AUTO)\n        dataset = dataset.batch(VAL_BATCH_SIZE).cache()\n    \n        dataset = dataset.map(preprocess, num_parallel_calls = AUTO)\n    dataset = dataset.prefetch(AUTO)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TP_FILENAMES = FILENAMES[:8]\nFP_FILENAMES = FILENAMES[8:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_dataset_(filenames, fp_filenames = None, training = True):\n    if training:\n        dataset = load_dataset(filenames , training = True)\n        dataset = dataset.shuffle(256)\n        dataset = dataset.batch(GLOBAL_BATCH_SIZE, drop_remainder = True)\n        dataset = dataset.map(transform, num_parallel_calls=AUTO)\n        dataset = dataset.unbatch()\n        if fp_filenames is not None:\n            dataset_fp = load_dataset(fp_filenames, training = True, FP=True)\n            dataset_fp = dataset_fp.shuffle(256)\n            dataset = dataset.concatenate(dataset_fp)\n        dataset = dataset.shuffle(2048).repeat()\n        dataset = dataset.batch(GLOBAL_BATCH_SIZE)\n        dataset = dataset.prefetch(AUTO)\n    else:\n        dataset = load_dataset(filenames , training = False)\n        dataset = dataset.batch(VAL_BATCH_SIZE).cache()\n    \n    dataset = dataset.prefetch(AUTO)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# mel spectrogram visualization\nif True:\n    train_dataset = load_dataset(FILENAMES[-1], training = True)\n    #nbatches = 0\n    #for b in train_dataset:\n    #    nbatches += 1\n    #print(nbatches)\n    \n    #train_dataset = load_dataset(FILENAMES[-1], training = True)\n    #nbatches = 0\n    #for b in train_dataset:\n    #    nbatches += 1\n    #print(nbatches)\n\n    plt.figure(figsize=(16,6))\n    for i, (wav, target) in enumerate(train_dataset.unbatch().take(4)):\n        plt.subplot(2,2,i+1)\n        plt.imshow(wav[:, :, 0])\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Competition Metric"},{"metadata":{"trusted":true},"cell_type":"code","source":"# from https://www.kaggle.com/carlthome/l-lrap-metric-for-tf-keras\n\ndef _one_sample_positive_class_precisions(example):\n    y_true, y_pred = example\n    y_true = tf.reshape(y_true, tf.shape(y_pred))\n    retrieved_classes = tf.argsort(y_pred, direction='DESCENDING')\n#     shape = tf.shape(retrieved_classes)\n    class_rankings = tf.argsort(retrieved_classes)\n    retrieved_class_true = tf.gather(y_true, retrieved_classes)\n    retrieved_cumulative_hits = tf.math.cumsum(tf.cast(retrieved_class_true, tf.float32))\n\n    idx = tf.where(y_true)[:, 0]\n    i = tf.boolean_mask(class_rankings, y_true)\n    r = tf.gather(retrieved_cumulative_hits, i)\n    c = 1 + tf.cast(i, tf.float32)\n    precisions = r / c\n\n    dense = tf.scatter_nd(idx[:, None], precisions, [y_pred.shape[0]])\n    return dense\n\n# @tf.function\nclass LWLRAP(tf.keras.metrics.Metric):\n    def __init__(self, num_classes, name='lwlrap'):\n        super().__init__(name=name)\n\n        self._precisions = self.add_weight(\n            name='per_class_cumulative_precision',\n            shape=[num_classes],\n            initializer='zeros',\n        )\n\n        self._counts = self.add_weight(\n            name='per_class_cumulative_count',\n            shape=[num_classes],\n            initializer='zeros',\n        )\n\n    def update_state(self, y_true, y_pred, sample_weight=None):\n        #mask = tf.reduce_sum(tf.cast(y_true < 0.5, tf.float32), axis=1) < 1\n        precisions = tf.map_fn(\n            fn=_one_sample_positive_class_precisions,\n            elems=(y_true, y_pred),\n            dtype=(tf.float32),\n        )\n\n        increments = tf.cast(precisions > 0, tf.float32)\n        total_increments = tf.reduce_sum(increments, axis=0)\n        total_precisions = tf.reduce_sum(precisions, axis=0)\n\n        self._precisions.assign_add(total_precisions)\n        self._counts.assign_add(total_increments)        \n\n    def result(self):\n        per_class_lwlrap = self._precisions / tf.maximum(self._counts, 1.0)\n        per_class_weight = self._counts / tf.reduce_sum(self._counts)\n        overall_lwlrap = tf.reduce_sum(per_class_lwlrap * per_class_weight)\n        return overall_lwlrap\n\n    def reset_states(self):\n        self._precisions.assign(self._precisions * 0)\n        self._counts.assign(self._counts * 0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Stepwise Cosine Decay Callback"},{"metadata":{"trusted":true},"cell_type":"code","source":"def cosine_decay_with_warmup(global_step,\n                             learning_rate_base,\n                             total_steps,\n                             warmup_learning_rate=0.0,\n                             warmup_steps= 0,\n                             hold_base_rate_steps=0):\n \n    if total_steps < warmup_steps:\n        raise ValueError('total_steps must be larger or equal to '\n                     'warmup_steps.')\n    learning_rate = 0.5 * learning_rate_base * (1 + tf.cos(\n        np.pi *\n        (tf.cast(global_step, tf.float32) - warmup_steps - hold_base_rate_steps\n        ) / float(total_steps - warmup_steps - hold_base_rate_steps)))\n    if hold_base_rate_steps > 0:\n        learning_rate = tf.where(\n          global_step > warmup_steps + hold_base_rate_steps,\n          learning_rate, learning_rate_base)\n    if warmup_steps > 0:\n        if learning_rate_base < warmup_learning_rate:\n            raise ValueError('learning_rate_base must be larger or equal to '\n                         'warmup_learning_rate.')\n        slope = (learning_rate_base - warmup_learning_rate) / warmup_steps\n        warmup_rate = slope * tf.cast(global_step,\n                                    tf.float32) + warmup_learning_rate\n        learning_rate = tf.where(global_step < warmup_steps, warmup_rate,\n                               learning_rate)\n    return tf.where(global_step > total_steps, 0.0, learning_rate,\n                    name='learning_rate')\n\n\n#dummy example\nrng = [i for i in range(int(EPOCHS * STEPS_PER_EPOCH))]\nWARMUP_STEPS =  int(WARMUP_EPOCHS * STEPS_PER_EPOCH)\ny = [cosine_decay_with_warmup(x , LEARNING_RATE, len(rng), 1e-5, WARMUP_STEPS) for x in rng]\n\nsns.set(style='whitegrid')\nfig, ax = plt.subplots(figsize=(20, 6))\nplt.plot(rng, y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# to apply learning rate schedule stepwise we need to subclass keras callback\n# if we would have applied lr schedule epoch wise then it is not needed we can only call class learningrateschedule \n\nclass WarmUpCosineDecayScheduler(tf.keras.callbacks.Callback):\n\n    def __init__(self,\n                 learning_rate_base,\n                 total_steps,\n                 global_step_init=0,\n                 warmup_learning_rate=0.0,\n                 warmup_steps=0,\n                 hold_base_rate_steps=0,\n                 verbose=0):\n\n        super(WarmUpCosineDecayScheduler, self).__init__()\n        self.learning_rate_base = learning_rate_base\n        self.total_steps = total_steps\n        self.global_step = global_step_init\n        self.warmup_learning_rate = warmup_learning_rate\n        self.warmup_steps = warmup_steps\n        self.hold_base_rate_steps = hold_base_rate_steps\n        self.verbose = verbose\n        self.learning_rates = []\n\n    def on_batch_end(self, batch, logs=None):\n        self.global_step = self.global_step + 1\n        lr = K.get_value(self.model.optimizer.lr)\n        self.learning_rates.append(lr)\n\n    def on_batch_begin(self, batch, logs=None):\n        lr = cosine_decay_with_warmup(global_step=self.global_step,\n                                      learning_rate_base=self.learning_rate_base,\n                                      total_steps=self.total_steps,\n                                      warmup_learning_rate=self.warmup_learning_rate,\n                                      warmup_steps=self.warmup_steps,\n                                      hold_base_rate_steps=self.hold_base_rate_steps)\n        K.set_value(self.model.optimizer.lr, lr)\n        if self.verbose > 0:\n            print('\\nBatch %05d: setting learning '\n                  'rate to %s.' % (self.global_step + 1, lr.numpy()))\n            \n\ntotal_steps = int(EPOCHS * STEPS_PER_EPOCH)\n# Compute the number of warmup batches or steps.\nwarmup_steps = int(WARMUP_EPOCHS * STEPS_PER_EPOCH)\nwarmup_learning_rate = WARMUP_LEARNING_RATE","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Definition"},{"metadata":{"trusted":true},"cell_type":"code","source":"def stat_pool(x):\n    #avg_pool = tf.math.reduce_mean(x, axis=[1, 2])\n    #stddev_pool = tf.math.reduce_std(x, axis=[1, 2])\n    mean, var = tf.nn.moments(x, axes=[1, 2])\n    return K.concatenate([mean, var], axis=1)\n\ndef RFCX_MODEL():\n    waveform = Input(shape=(None,None,3), dtype=tf.float32)\n    noisy_waveform = GaussianNoise(0.2)(waveform)\n    model = efn.EfficientNetB4(include_top=False, weights='imagenet',) \n    model_output = model(noisy_waveform)\n    #sd = SpatialDropout2D(params.dropout)(model_output)\n    #conv_out = Conv2D(params.num_classes, (1,1), activation = params.classifier_activation)(sd)\n    model_output = GlobalAveragePooling2D()(model_output)\n    #model_output = Lambda(stat_pool)(model_output)\n    dense = Dropout(params.dropout)(model_output)\n    predictions = Dense(params.num_classes, activation = params.classifier_activation )(dense)\n    model = Model(\n      name='Efficientnet', inputs=waveform,\n      outputs=[predictions])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def masked_bce(y_true, y_pred):\n    mask = tf.math.greater(y_true, -0.5)\n    #if tf.math.count_nonzero(mask) == tf.size(mask, out_type=tf.dtypes.int64):\n    #    loss = tf.keras.losses.binary_crossentropy(y_true, y_pred)\n    #else:\n    loss = tf.keras.losses.binary_crossentropy(tf.boolean_mask(y_true, mask), tf.boolean_mask(y_pred, mask), label_smoothing=0.2)\n    return tf.math.reduce_mean(loss)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def masked_bce_focal(y_true, y_pred):\n    mask = tf.math.greater(y_true, -0.5)\n    #if tf.math.count_nonzero(mask) == tf.size(mask, out_type=tf.dtypes.int64):\n    #    loss = tf.keras.losses.binary_crossentropy(y_true, y_pred)\n    #else:\n    loss = tfa.losses.sigmoid_focal_crossentropy(tf.boolean_mask(y_true, mask), tf.boolean_mask(y_pred, mask))\n    return tf.math.reduce_mean(loss)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model():\n    with strategy.scope():\n        model = RFCX_MODEL()\n        model.summary()\n        model.compile(#optimizer = tfa.optimizers.SWA(tfa.optimizers.RectifiedAdam()),\n                                optimizer=\"adam\",\n                                #loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.1),\n                                loss = masked_bce_focal,\n                                #loss = rank_loss,\n                                #loss = combined_loss,\n                                metrics = [LWLRAP(num_classes = params.num_classes),\n                                ])\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training And Validation Loop"},{"metadata":{"trusted":true},"cell_type":"code","source":"if TRAIN:\n    skf = KFold(n_splits=N_FOLDS, shuffle=True, random_state=seed)\n    oof_pred = []; oof_labels = []; history_list = []\n\n    for fold,(idxT, idxV) in enumerate(skf.split(np.arange(len(TP_FILENAMES)))):\n    #for fold in [1]:\n        if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n        print(f'\\nFOLD: {fold+1}')\n        print(f'TRAIN: {idxT} VALID: {idxV}')\n\n        # Create train and validation sets\n        TRAIN_FILENAMES = [TP_FILENAMES[x] for x in idxT]\n        TRAIN_FILENAMES = TRAIN_FILENAMES + FP_FILENAMES\n        VALID_FILENAMES = [TP_FILENAMES[x] for x in idxV]\n        np.random.shuffle(TRAIN_FILENAMES)\n\n        train_dataset =  load_dataset(TRAIN_FILENAMES, training=True,)\n        validation_data= load_dataset(VALID_FILENAMES, training=False) \n\n        model = get_model()\n\n        model_path = f'RFCX_model_fold {fold}.h5'\n        early_stopping = EarlyStopping(monitor = 'val_lwlrap', mode = 'max', \n                           patience = PATIENCE, restore_best_weights=True, verbose=1)\n\n        # Create the Learning rate scheduler.\n        cosine_warm_up_lr = WarmUpCosineDecayScheduler(learning_rate_base= LEARNING_RATE,\n                                        total_steps= total_steps,\n                                        warmup_learning_rate= warmup_learning_rate,\n                                        warmup_steps= warmup_steps,\n                                        hold_base_rate_steps=0)\n\n        ## TRAIN\n        history = model.fit(train_dataset,\n                            #class_weight=cw, \n                            steps_per_epoch=STEPS_PER_EPOCH, \n                            callbacks=[early_stopping, cosine_warm_up_lr], \n                            epochs=EPOCHS,  \n                            validation_data = validation_data,\n                            verbose = 2).history\n\n        history_list.append(history)\n        # Save last model weights\n        model.save_weights(model_path)\n\n    # OOF predictions\n        #ds_valid = get_dataset(VALID_FILENAMES, training = False)\n        #oof_labels.append([target.numpy() for frame, target in iter(ds_valid.unbatch())])\n        #x_oof = ds_valid.map(lambda frames, target: frames)\n        #oof_pred.append(np.argmax(model.predict(x_oof), axis=-1))\n\n        ## RESULTS\n        print(f\"#### FOLD {fold+1} OOF Accuracy = {np.max(history['val_lwlrap']):.3f}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Plot curve"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_history(history):\n    plt.figure(figsize=(8,3))\n    plt.subplot(1,2,1)\n    plt.plot(history[\"loss\"])\n    plt.plot(history[\"val_loss\"])\n    plt.legend(['Train', 'Test'], loc='upper left')\n    plt.title(\"loss\")\n\n    plt.subplot(1,2,2)\n    plt.plot(history[\"lwlrap\"])\n    plt.plot(history[\"val_lwlrap\"])\n    plt.legend(['Train', 'Test'], loc='upper left')\n    plt.title(\"lwlrap\")\n\nif TRAIN:\n    for hist in history_list:\n        plot_history(hist)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_test_dataset(filenames, training = False):\n    \n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO )  \n    dataset = dataset.map(read_unlabeled_tfrecord , num_parallel_calls = AUTO ).unbatch()\n    dataset = dataset.map(lambda spec : waveform_to_log_mel_spectrogram(spec['audio_wav'], spec['recording_id']) , num_parallel_calls = AUTO)\n    dataset = dataset.map(preprocess, num_parallel_calls = AUTO)\n    return dataset.batch(GLOBAL_BATCH_SIZE*4).cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_predict = []\n\ntest_data = get_test_dataset(TEST_FILES, training = False)\ntest_audio = test_data.map(lambda frames, recording_id: frames)\n\nmodel = get_model()\nfor fold in range(N_FOLDS):\n    model.load_weights(f'{MODEL_DIR}/RFCX_model_fold {fold}.h5')\n    test_predict.append(model.predict(test_audio, verbose = 1 ))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"np.array(test_predict).shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SUB = pd.read_csv('../input/rfcx-species-audio-detection/sample_submission.csv')\nNREC = (60 - CUT)//STRIDE + 1\n\npredict = np.array(test_predict).reshape(N_FOLDS, len(SUB), NREC, params.num_classes)\n#predict_argsort = params.num_classes - predict.reshape((-1, params.num_classes)).argsort(axis=-1)\n#predict_argsort = predict_argsort / params.num_classes\n#predict = predict_argsort.reshape(N_FOLDS, len(SUB), 60 // TIME, params.num_classes)\npredict = np.mean(np.max(predict ,axis = 2) , axis = 0)\n# predict = np.mean(predict, axis =  0)\n\nrecording_id = test_data.map(lambda frames, recording_id: recording_id).unbatch()\n# # all in one batch\ntest_ids = next(iter(recording_id.batch(len(SUB) * NREC))).numpy().astype('U').reshape(len(SUB), NREC)\n\npred_df = pd.DataFrame({ 'recording_id' : test_ids[:, 0],\n             **{f's{i}' : predict[:, i] for i in range(params.num_classes)} })","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_df.sort_values('recording_id', inplace = True) \npred_df.to_csv('submission.csv', index = False)    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# USE MODE 1, 2, or 3\nMODE = 1\n\n# LOAD SUBMISSION\nFUDGE = 2.0\nfor k in range(24):\n    pred_df.iloc[:,1+k] -= pred_df.iloc[:,1+k].min()\n    pred_df.iloc[:,1+k] /= pred_df.iloc[:,1+k].max()\n\n# CONVERT PROBS TO ODDS, APPLY MULTIPLIER, CONVERT BACK TO PROBS\ndef scale(probs, factor):\n    probs = probs.copy()\n    idx = np.where(probs!=1)[0]\n    odds = factor * probs[idx] / (1-probs[idx])\n    probs[idx] =  odds/(1+odds)\n    return probs\n\n# DIFFERENT DISTRIBUTIONS\nd1 = pred_df.iloc[:,1:].mean().values\nd2 = np.array([113, 204, 44, 923, 53, 41, 3, 213, 44, 23, 26, 149, 255,  \n    14, 123, 222, 46, 6, 474, 4, 17, 18, 23, 72])/1000.\n\nfor k in range(24):\n    if MODE==1: d = FUDGE\n    if MODE==2: d = d1[k]/(1-d1[k])\n    if MODE==3: s = d2[k] / d1[k]\n    else: s = (d2[k]/(1-d2[k]))/d\n    pred_df.iloc[:,k+1] = scale(pred_df.iloc[:,k+1].values,s)\n\npred_df.to_csv('submission_with_pp.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_df","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}