{"cells":[{"metadata":{},"cell_type":"markdown","source":"* enable 8 TPUs\n* ResNet50 backbone\n    * from v3: more accurate initial bias in the final layer\n* mixup disabled\n* use only TP samples\n* cut 6sec around annotation.\n* Stratified 5-fold and ensemble (just summing up probabilities)\n    * from v3: clip-wise splitting instead of annotation-wise\n* from v2: Grad-CAM visualization\n* augmentation\n    * gaussian noise ($\\sigma=0.3$ while the image variance is 1)\n    * random_brightness\n    * specaugment"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install iterative-stratification\n!pip install tf-nightly\n!pip install --upgrade pip","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow_addons as tfa\nimport numpy as np\nfrom pathlib import Path\nimport io\nimport matplotlib.pyplot as plt\nimport soundfile as sf\nimport librosa\nfrom kaggle_datasets import KaggleDatasets\nfrom tqdm import tqdm\nimport pandas as pd\n# from sklearn.model_selection import StratifiedKFold\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold\nimport seaborn as sns\nfrom IPython.display import Audio\nimport cv2\n\ntf.__version__","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"cfg = {\n    'parse_params': {\n        'cut_time': 10,\n    },\n    'data_params': {\n        'sample_time': 6, # assert 60 % sample_time == 0\n        'spec_fmax': 24000.0,\n        'spec_fmin': 40.0,\n        'spec_mel': 224,\n        'mel_power': 2,\n        'img_shape': (224, 512)\n    },\n    'model_params': {\n        'batchsize_per_tpu': 16,\n        'iteration_per_epoch': 64,\n        'epoch': 15,\n        'arch': tf.keras.applications.ResNet50,\n        'arch_preprocess': tf.keras.applications.resnet50.preprocess_input,\n        'freeze_to': 0,  # Freeze to backbone.layers[:freeze_to]. If None, all layers in the backbone will be freezed.\n        'loss': {\n            'fn': tfa.losses.SigmoidFocalCrossEntropy,\n            'params': {},\n        },\n        'optim': {\n            'fn': tfa.optimizers.RectifiedAdam,\n            'params': {'lr': 1e-3, 'total_steps': 15*64, 'warmup_proportion': 0.3, 'min_lr': 1e-6},\n        },\n        'mixup': False\n    }\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# detect and init the TPU\ntpu = tf.distribute.cluster_resolver.TPUClusterResolver()\ntf.config.experimental_connect_to_cluster(tpu)\ntf.tpu.experimental.initialize_tpu_system(tpu)\nprint(\"All devices: \", tf.config.list_logical_devices('TPU'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"strategy = tf.distribute.experimental.TPUStrategy(tpu)\nAUTOTUNE = tf.data.experimental.AUTOTUNE\nGCS_DS_PATH = KaggleDatasets().get_gcs_path()\n\nTRAIN_TFREC = GCS_DS_PATH + \"/tfrecords/train\"\nTEST_TFREC = GCS_DS_PATH + \"/tfrecords/test\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CUT = cfg['parse_params']['cut_time']\nSR = 48000     # all wave's sample rate may be 48k\n\nTIME = cfg['data_params']['sample_time']\n\nFMAX = cfg['data_params']['spec_fmax']\nFMIN = cfg['data_params']['spec_fmin']\nN_MEL = cfg['data_params']['spec_mel']\n\nHEIGHT, WIDTH = cfg['data_params']['img_shape']\n\nCLASS_N = 24","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Explore the tfrecords, Create dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"raw_dataset = tf.data.TFRecordDataset([TRAIN_TFREC + '/00-148.tfrec'])\nraw_dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## parse tfrecords"},{"metadata":{"trusted":true},"cell_type":"code","source":"feature_description = {\n    'recording_id': tf.io.FixedLenFeature([], tf.string, default_value=''),\n    'audio_wav': tf.io.FixedLenFeature([], tf.string, default_value=''),\n    'label_info': tf.io.FixedLenFeature([], tf.string, default_value=''),\n}\nparse_dtype = {\n    'audio_wav': tf.float32,\n    'recording_id': tf.string,\n    'species_id': tf.int32,\n    'songtype_id': tf.int32,\n    't_min': tf.float32,\n    'f_min': tf.float32,\n    't_max': tf.float32,\n    'f_max':tf.float32,\n    'is_tp': tf.int32\n}\n\n@tf.function\ndef _parse_function(example_proto):\n    sample = tf.io.parse_single_example(example_proto, feature_description)\n    wav, _ = tf.audio.decode_wav(sample['audio_wav'], desired_channels=1) # mono\n    label_info = tf.strings.split(sample['label_info'], sep='\"')[1]\n    labels = tf.strings.split(label_info, sep=';')\n    \n    @tf.function\n    def _cut_audio(label):\n        items = tf.strings.split(label, sep=',')\n        spid = tf.squeeze(tf.strings.to_number(items[0], tf.int32))\n        soid = tf.squeeze(tf.strings.to_number(items[1], tf.int32))\n        tmin = tf.squeeze(tf.strings.to_number(items[2]))\n        fmin = tf.squeeze(tf.strings.to_number(items[3]))\n        tmax = tf.squeeze(tf.strings.to_number(items[4]))\n        fmax = tf.squeeze(tf.strings.to_number(items[5]))\n        tp = tf.squeeze(tf.strings.to_number(items[6], tf.int32))\n\n        tmax_s = tmax * tf.cast(SR, tf.float32)\n        tmin_s = tmin * tf.cast(SR, tf.float32)\n        cut_s = tf.cast(CUT * SR, tf.float32)\n        all_s = tf.cast(60 * SR, tf.float32)\n        tsize_s = tmax_s - tmin_s\n        cut_min = tf.cast(\n            tf.maximum(0.0, \n                tf.minimum(tmin_s - (cut_s - tsize_s) / 2,\n                           tf.minimum(tmax_s + (cut_s - tsize_s) / 2, all_s) - cut_s)\n            ), tf.int32\n        )\n        cut_max = cut_min + CUT * SR\n        \n        _sample = {\n            'audio_wav': tf.reshape(wav[cut_min:cut_max], [CUT*SR]),\n            'recording_id': sample['recording_id'],\n            'species_id': spid,\n            'songtype_id': soid,\n            't_min': tmin - tf.cast(cut_min, tf.float32)/tf.cast(SR, tf.float32),\n            'f_min': fmin,\n            't_max': tmax - tf.cast(cut_min, tf.float32)/tf.cast(SR, tf.float32),\n            'f_max': fmax,\n            'is_tp': tp\n        }\n        return _sample\n    \n    samples = tf.map_fn(_cut_audio, labels, dtype=parse_dtype)\n    return samples\n\nparsed_dataset = raw_dataset.map(_parse_function).unbatch()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef _cut_wav(x):\n    # random cut in training\n    cut_min = tf.random.uniform([], maxval=tf.minimum((CUT-TIME)*SR, tf.cast(x['t_max']*SR, tf.int32)), dtype=tf.int32)\n    cut_max = cut_min + TIME * SR\n    cutwave = tf.reshape(x['audio_wav'][cut_min:cut_max], [TIME*SR])\n    print(\"The cutwave shape is\")\n    print(cutwave.shape)\n    print(\"!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\")\n    y = {}\n    y.update(x)\n    y['audio_wav'] = cutwave\n    y['t_min'] = tf.maximum(0.0, x['t_min'] - tf.cast(cut_min, tf.float32) / SR)\n    y['t_max'] = tf.maximum(0.0, x['t_max'] - tf.cast(cut_min, tf.float32) / SR)\n    return y\n    \n@tf.function\ndef _cut_wav_val(x):\n    # center crop in validation\n    cut_min = tf.minimum((CUT-TIME)*SR // 2, tf.cast((x['t_min'] + x['t_max']) / 2 * SR, tf.int32))\n    cut_max = cut_min + TIME * SR\n    cutwave = tf.reshape(x['audio_wav'][cut_min:cut_max], [TIME*SR])\n    y = {}\n    y.update(x)\n    y['audio_wav'] = cutwave\n    y['t_min'] = tf.maximum(0.0, x['t_min'] - tf.cast(cut_min, tf.float32) / SR)\n    y['t_max'] = tf.maximum(0.0, x['t_max'] - tf.cast(cut_min, tf.float32) / SR)\n    return y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef _filtTP(x):\n    return x['is_tp'] == 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_wav(sample, ax):\n    wav = sample[\"audio_wav\"].numpy()\n    rate = SR\n    ax.plot(np.arange(len(wav)) / rate, wav)\n    ax.set_title(\n        sample[\"recording_id\"].numpy().decode()\n        + (\"/%d\" % sample[\"species_id\"])\n        + (\"TP\" if sample[\"is_tp\"] else \"FP\"))\n\n    return Audio((wav * 2**15).astype(np.int16), rate=rate)\n\nfig, ax = plt.subplots(figsize=(15, 3))\nshow_wav(next(iter(parsed_dataset)), ax)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for elem in (iter(parsed_dataset)):\n#     slice_dice = elem[\"audio_wav\"]\n#     print(type(slice_dice))\n#     #mod = tf.slice(slice_dice,[0],[432000])\n#     first_slice = tf.slice(slice_dice,[0],[5000])\n#     second_slice = tf.slice(slice_dice,[20000],[480000])\n#     #mod = tf.stack([first_slice,second_slice],1)\n#     #elem = tf.squeeze(mod)\n# fig, ax = plt.subplots(figsize=(15, 3))\n# show_wav(next(iter(parsed_dataset)), ax)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## create mel-spectrogram"},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef _wav_to_spec(x):\n    mel_power = cfg['data_params']['mel_power']\n    \n    stfts = tf.signal.stft(x[\"audio_wav\"], frame_length=2048, frame_step=512, fft_length=2048)\n    spectrograms = tf.abs(stfts) ** mel_power\n\n    # Warp the linear scale spectrograms into the mel-scale.\n    num_spectrogram_bins = stfts.shape[-1]\n    lower_edge_hertz, upper_edge_hertz, num_mel_bins = FMIN, FMAX, N_MEL\n    \n    linear_to_mel_weight_matrix = tf.signal.linear_to_mel_weight_matrix(\n      num_mel_bins, num_spectrogram_bins, SR, lower_edge_hertz,\n      upper_edge_hertz)\n    mel_spectrograms = tf.tensordot(\n      spectrograms, linear_to_mel_weight_matrix, 1)\n    mel_spectrograms.set_shape(spectrograms.shape[:-1].concatenate(\n      linear_to_mel_weight_matrix.shape[-1:]))\n\n    # Compute a stabilized log to get log-magnitude mel-scale spectrograms.\n    log_mel_spectrograms = tf.math.log(mel_spectrograms + 1e-6)\n\n    y = {\n        'audio_spec': tf.transpose(log_mel_spectrograms), # (num_mel_bins, frames)\n    }\n    y.update(x)\n    return y\n\nspec_dataset = parsed_dataset.filter(_filtTP).map(_cut_wav).map(_wav_to_spec)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(12,5))\nfor i, s in enumerate(spec_dataset.take(3)):\n    plt.subplot(1,3,i+1)\n    plt.imshow(s['audio_spec'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import librosa.display\nimport matplotlib.patches as patches\n\ndef show_spectrogram(sample, ax, showlabel=False):\n    S_dB = sample[\"audio_spec\"].numpy()\n    img = librosa.display.specshow(S_dB, x_axis='time',\n                             y_axis='mel', sr=SR,\n                             fmax=FMAX, fmin=FMIN, ax=ax, cmap='magma')\n    ax.set(title=f'Mel-frequency spectrogram of {sample[\"recording_id\"].numpy().decode()}')\n    sid, fmin, fmax, tmin, tmax, istp = (\n            sample[\"species_id\"], sample[\"f_min\"], sample[\"f_max\"], sample[\"t_min\"], sample[\"t_max\"], sample[\"is_tp\"])\n    ec = '#00ff00' if istp == 1 else '#0000ff'\n    ax.add_patch(\n        patches.Rectangle(xy=(tmin, fmin), width=tmax-tmin, height=fmax-fmin, ec=ec, fill=False)\n    )\n\n    if showlabel:\n        ax.text(tmin, fmax, \n        f\"{sid.numpy().item()} {'tp' if istp == 1 else 'fp'}\",\n        horizontalalignment='left', verticalalignment='bottom', color=ec, fontsize=16)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(15,3))\nshow_spectrogram(next(iter(spec_dataset)), ax, showlabel=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# in validation, annotations will come to the center\nfig, ax = plt.subplots(figsize=(15,3))\nshow_spectrogram(next(iter(parsed_dataset.filter(_filtTP).map(_cut_wav_val).map(_wav_to_spec))), ax, showlabel=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for sample in spec_dataset.take(5):\n    fig, ax = plt.subplots(figsize=(15,3))\n    show_spectrogram(sample, ax, showlabel=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## create labels"},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef _create_annot(x):\n    targ = tf.one_hot(x[\"species_id\"], CLASS_N, on_value=x[\"is_tp\"], off_value=0)\n    \n    return {\n        'input': x[\"audio_spec\"],\n        'target': tf.cast(targ, tf.float32)\n    }\n\nannot_dataset = spec_dataset.map(_create_annot)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## proprocessing and data augmentation\n\nIn training, I use\n\n* gaussian noise\n* random brightness\n* specaugment"},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef _preprocess_img(x, training=False):\n    image = tf.expand_dims(x, axis=-1)\n    image = tf.image.resize(image, [HEIGHT, WIDTH])\n    image = tf.image.per_image_standardization(image)\n    \n    @tf.function\n    def _specaugment(image):\n        ERASE_TIME = 40\n        ERASE_MEL = 20\n        ERASE_TIME_N = 1\n        ERASE_MEL_N = 1\n        image = tf.expand_dims(image, axis=0)\n        xoff = tf.random.uniform([ERASE_TIME_N], minval=ERASE_TIME//2, maxval=WIDTH-ERASE_TIME//2, dtype=tf.int32)\n        xsize = tf.random.uniform([ERASE_TIME_N], minval=ERASE_TIME//2, maxval=ERASE_TIME, dtype=tf.int32)\n        yoff = tf.random.uniform([ERASE_MEL_N], minval=ERASE_MEL//2, maxval=HEIGHT-ERASE_MEL//2, dtype=tf.int32)\n        ysize = tf.random.uniform([ERASE_MEL_N], minval=ERASE_MEL//2, maxval=ERASE_MEL, dtype=tf.int32)\n        for i in range(ERASE_TIME_N):\n            image = tfa.image.cutout(image, [HEIGHT, xsize[i]], offset=[HEIGHT//2, xoff[i]])\n        for i in range(ERASE_MEL_N):\n            image = tfa.image.cutout(image, [ysize[i], WIDTH], offset=[yoff[i], WIDTH//2])\n        return tf.squeeze(image, axis=0)\n    \n    if training:\n        # gaussian\n        gau = tf.keras.layers.GaussianNoise(0.3)\n        image = tf.cond(tf.random.uniform([]) < 0.5, lambda: gau(image, training=True), lambda: image)\n        # brightness\n        image = tf.image.random_brightness(image, 0.2)\n        # specaugment\n        image = tf.cond(tf.random.uniform([]) < 0.5, lambda: _specaugment(image), lambda: image)\n        \n    image = (image - tf.reduce_min(image)) / (tf.reduce_max(image) - tf.reduce_min(image)) * 255.0 # rescale to [0, 255]\n    image = tf.image.grayscale_to_rgb(image)\n    image = cfg['model_params']['arch_preprocess'](image)\n\n    return image\n\n@tf.function\ndef _preprocess(x):\n    image = _preprocess_img(x['input'], True)\n    return (image, x[\"target\"])\n\n@tf.function\ndef _preprocess_val(x):\n    image = _preprocess_img(x['input'], False)\n    return (image, x[\"target\"])\n\n@tf.function\ndef _preprocess_test(x):\n    image = _preprocess_img(x['audio_spec'], False)\n    return (image, x[\"recording_id\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(16, 4))\nfor i, (inp, targ) in enumerate(annot_dataset.map(_preprocess).take(6)):\n    plt.subplot(2,3,i+1)\n    plt.imshow(inp.numpy()[:,:,0])\n    t = targ.numpy()\n    if t.sum() == 0:\n        plt.title(f'FP')\n    else:\n        plt.title(f'{t.nonzero()[0]}')\n    plt.colorbar()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"def create_model():\n    with strategy.scope():\n        backbone = cfg['model_params']['arch'](include_top=False, weights='imagenet')\n        \n        if cfg['model_params']['freeze_to'] is None:\n            for layer in backbone.layers:\n                layer.trainable = False\n        else:\n            for layer in backbone.layers[:cfg['model_params']['freeze_to']]:\n                layer.trainable = False\n                \n        # negative bias for stability (Section 4.1 in \"Focal Loss\" (https://arxiv.org/abs/1708.02002))\n        prior = 1/CLASS_N\n        pi = -np.log((1-prior) / prior) # sigmoid(pi) == prior\n\n        head = tf.keras.Sequential([\n            tf.keras.layers.GlobalAveragePooling2D(),\n            tf.keras.layers.BatchNormalization(),\n            tf.keras.layers.Dropout(0.4),\n            tf.keras.layers.Dense(1024, activation='relu', kernel_initializer=tf.keras.initializers.he_normal()),\n            tf.keras.layers.BatchNormalization(),\n            tf.keras.layers.Dropout(0.4),\n            tf.keras.layers.Dense(CLASS_N, bias_initializer=tf.keras.initializers.Constant(pi))\n        ])\n        model = tf.keras.Sequential([backbone, head])\n    return model\n\nmodel = create_model()\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef _mixup(inp, targ):\n    indice = tf.range(len(inp))\n    indice = tf.random.shuffle(indice)\n    sinp = tf.gather(inp, indice, axis=0)\n    starg = tf.gather(targ, indice, axis=0)\n    \n    alpha = 0.2\n    t = tf.compat.v1.distributions.Beta(alpha, alpha).sample([len(inp)])\n    tx = tf.reshape(t, [-1, 1, 1, 1])\n    ty = tf.reshape(t, [-1, 1])\n    x = inp * tx + sinp * (1-tx)\n    y = targ * ty + starg * (1-ty)\n#     y = tf.minimum(targ + starg, 1.0) # for multi-label???\n    return x, y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tfrecs = sorted(tf.io.gfile.glob(TRAIN_TFREC + '/*.tfrec'))\nparsed_trainval = (tf.data.TFRecordDataset(tfrecs, num_parallel_reads=AUTOTUNE)\n                    .map(_parse_function, num_parallel_calls=AUTOTUNE).unbatch()\n                    .filter(_filtTP))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Stratified 5-Fold"},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/rfcx-species-audio-detection/train_tp.csv\")\n\ntable = df.groupby('recording_id')['species_id'].apply(\n    lambda x: tf.scatter_nd(tf.expand_dims(x, 1), np.ones_like(x), shape=[CLASS_N]).numpy()\n).reset_index()\n\nskf = MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=0)\nidx_splits = list(skf.split(table.recording_id, np.stack(table.species_id.to_numpy())))\nsplits = list(map(lambda xs: (table.recording_id[xs[0]].to_numpy(), table.recording_id[xs[1]].to_numpy()), idx_splits))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_table = df.set_index('recording_id')\nplt.hist([_table.loc[splits[0][0], 'species_id'], _table.loc[splits[0][1], 'species_id']],\n         bins=CLASS_N,stacked=True)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_recid_filter(recids):\n    @tf.function\n    def _filt(x):\n        return tf.reduce_any(recids == x['recording_id'])\n    return _filt","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Other setup"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_train_dataset(batchsize, train_recids):\n    global parsed_trainval\n    print(\"train split: %d\" % len(train_recids))\n    parsed_train = (parsed_trainval\n                    .filter(create_recid_filter(train_recids)))\n\n    dataset = (parsed_train.cache()\n        .shuffle(len(train_recids))\n        .repeat()\n        .map(_cut_wav, num_parallel_calls=AUTOTUNE)\n        .map(_wav_to_spec, num_parallel_calls=AUTOTUNE)\n        .map(_create_annot, num_parallel_calls=AUTOTUNE)\n        .map(_preprocess, num_parallel_calls=AUTOTUNE)\n        .batch(batchsize))\n\n    print(dataset.element_spec)\n    print(\"dataset shape train %%%%%%%%%%%%%%%%%%%%%%%%%%%%\")\n    if cfg['model_params']['mixup']:\n        dataset = (dataset.map(_mixup, num_parallel_calls=AUTOTUNE)\n                    .prefetch(AUTOTUNE))\n    else:\n        dataset = dataset.prefetch(AUTOTUNE)\n    return dataset\n\ndef create_val_dataset(batchsize, val_recids):\n    global parsed_trainval\n    print(\"val split: %d\" % len(val_recids))\n    parsed_val = (parsed_trainval\n                  .filter(create_recid_filter(val_recids)))\n\n    vdataset = (parsed_val\n        .map(_cut_wav_val, num_parallel_calls=AUTOTUNE)\n        .map(_wav_to_spec, num_parallel_calls=AUTOTUNE)\n        .map(_create_annot, num_parallel_calls=AUTOTUNE)\n        .map(_preprocess_val, num_parallel_calls=AUTOTUNE)\n        .batch(8*strategy.num_replicas_in_sync)\n        .cache())\n    return vdataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Metrics"},{"metadata":{"trusted":true},"cell_type":"code","source":"# from https://www.kaggle.com/carlthome/l-lrap-metric-for-tf-keras\n@tf.function\ndef _one_sample_positive_class_precisions(example):\n    y_true, y_pred = example\n\n    retrieved_classes = tf.argsort(y_pred, direction='DESCENDING')\n    class_rankings = tf.argsort(retrieved_classes)\n    retrieved_class_true = tf.gather(y_true, retrieved_classes)\n    retrieved_cumulative_hits = tf.math.cumsum(tf.cast(retrieved_class_true, tf.float32))\n\n    idx = tf.where(y_true)[:, 0]\n    i = tf.boolean_mask(class_rankings, y_true)\n    r = tf.gather(retrieved_cumulative_hits, i)\n    c = 1 + tf.cast(i, tf.float32)\n    precisions = r / c\n\n    dense = tf.scatter_nd(idx[:, None], precisions, [y_pred.shape[0]])\n    return dense\n\nclass LWLRAP(tf.keras.metrics.Metric):\n    def __init__(self, num_classes, name='lwlrap'):\n        super().__init__(name=name)\n\n        self._precisions = self.add_weight(\n            name='per_class_cumulative_precision',\n            shape=[num_classes],\n            initializer='zeros',\n        )\n\n        self._counts = self.add_weight(\n            name='per_class_cumulative_count',\n            shape=[num_classes],\n            initializer='zeros',\n        )\n\n    def update_state(self, y_true, y_pred, sample_weight=None):\n        precisions = tf.map_fn(\n            fn=_one_sample_positive_class_precisions,\n            elems=(y_true, y_pred),\n            dtype=(tf.float32),\n        )\n\n        increments = tf.cast(precisions > 0, tf.float32)\n        total_increments = tf.reduce_sum(increments, axis=0)\n        total_precisions = tf.reduce_sum(precisions, axis=0)\n\n        self._precisions.assign_add(total_precisions)\n        self._counts.assign_add(total_increments)        \n\n    def result(self):\n        per_class_lwlrap = self._precisions / tf.maximum(self._counts, 1.0)\n        per_class_weight = self._counts / tf.reduce_sum(self._counts)\n        overall_lwlrap = tf.reduce_sum(per_class_lwlrap * per_class_weight)\n        return overall_lwlrap\n\n    def reset_states(self):\n        self._precisions.assign(self._precisions * 0)\n        self._counts.assign(self._counts * 0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Testset and Inference function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def _parse_function_test(example_proto):\n    sample = tf.io.parse_single_example(example_proto, feature_description)\n    wav, _ = tf.audio.decode_wav(sample['audio_wav'], desired_channels=1) # mono\n    \n    @tf.function\n    def _cut_audio(i):\n        _sample = {\n            'audio_wav': tf.reshape(wav[i*SR*TIME:(i+1)*SR*TIME], [SR*TIME]),\n            'recording_id': sample['recording_id']\n        }\n        return _sample\n\n    return tf.map_fn(_cut_audio, tf.range(60//TIME), dtype={\n        'audio_wav': tf.float32,\n        'recording_id': tf.string\n    })\n\ndef inference(model):\n    tdataset = (tf.data.TFRecordDataset(tf.io.gfile.glob(TEST_TFREC + '/*.tfrec'), num_parallel_reads=AUTOTUNE)\n        .map(_parse_function_test, num_parallel_calls=AUTOTUNE).unbatch()\n        .map(_wav_to_spec, num_parallel_calls=AUTOTUNE)\n        .map(_preprocess_test, num_parallel_calls=AUTOTUNE)\n        .batch(128*(60//TIME)).prefetch(AUTOTUNE))\n    \n    rec_ids = []\n    probs = []\n    for inp, rec_id in tqdm(tdataset):\n        with strategy.scope():\n            pred = model.predict_on_batch(tf.reshape(inp, [-1, HEIGHT, WIDTH, 3]))\n            prob = tf.sigmoid(pred)\n            prob = tf.reduce_max(tf.reshape(prob, [-1, 60//TIME, CLASS_N]), axis=1)\n\n        rec_id_stack = tf.reshape(rec_id, [-1, 60//TIME])\n        for rec in rec_id.numpy():\n            assert len(np.unique(rec)) == 1\n        rec_ids.append(rec_id_stack.numpy()[:,0])\n        probs.append(prob.numpy())\n        \n    crec_ids = np.concatenate(rec_ids)\n    cprobs = np.concatenate(probs)\n    \n    sub = pd.DataFrame({\n        'recording_id': list(map(lambda x: x.decode(), crec_ids.tolist())),\n        **{f's{i}': cprobs[:,i] for i in range(CLASS_N)}\n    })\n    sub = sub.sort_values('recording_id')\n    return sub","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Now start training!"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_history(history, name):\n    plt.figure(figsize=(8,3))\n    plt.subplot(1,2,1)\n    plt.plot(history.history[\"loss\"])\n    plt.plot(history.history[\"val_loss\"])\n    plt.legend(['Train', 'Test'], loc='upper left')\n    plt.title(\"loss\")\n    # plt.yscale('log')\n\n    plt.subplot(1,2,2)\n    plt.plot(history.history[\"lwlrap\"])\n    plt.plot(history.history[\"val_lwlrap\"])\n    plt.legend(['Train', 'Test'], loc='upper left')\n    plt.title(\"metric\")\n\n    plt.savefig(name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_and_inference(splits, split_id):\n    batchsize = cfg['model_params']['batchsize_per_tpu'] * strategy.num_replicas_in_sync\n    print(\"batchsize\", batchsize)\n    loss_fn = cfg['model_params']['loss']['fn'](from_logits=True, **cfg['model_params']['loss']['params'])\n\n    idx_train_tf = tf.constant(splits[split_id][0])\n    idx_val_tf = tf.constant(splits[split_id][1])\n\n    dataset = create_train_dataset(batchsize, idx_train_tf)\n    vdataset = create_val_dataset(batchsize, idx_val_tf)\n    \n    optimizer = cfg['model_params']['optim']['fn'](**cfg['model_params']['optim']['params'])\n    model = create_model()\n    with strategy.scope():\n        model.compile(optimizer=optimizer, loss=loss_fn, metrics=[LWLRAP(CLASS_N)])\n        \n    history = model.fit(dataset,\n                        steps_per_epoch=cfg['model_params']['iteration_per_epoch'],\n                        epochs=cfg['model_params']['epoch'],\n                        validation_data=vdataset,\n                        callbacks=[\n                            tf.keras.callbacks.ModelCheckpoint(\n                                filepath='model_best_%d.h5' % split_id,\n                                save_weights_only=True,\n                                monitor='val_lwlrap',\n                                mode='max',\n                                save_best_only=True),\n                        ])\n    plot_history(history, 'history_%d.png' % split_id)\n    \n    ### inference ###\n    model.load_weights('model_best_%d.h5' % split_id)\n    return inference(model), history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"# train and inference\n# sub, _ = train_and_inference(splits, 0)\n\n# N-fold ensemble\nsub = sum(\n    map(\n        lambda i: train_and_inference(splits, i)[0].set_index('recording_id'),\n        range(len(splits))\n    )\n).reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Show Grad-CAM\nLet's take a look at Grad-CAM and check if my model is trained properly"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = create_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# modified https://github.com/sicara/tf-explain/blob/8dff129ff7b1012dba2761a61e3c3e68e9ecbec2/tf_explain/core/grad_cam.py\n\"\"\"MIT License\n\nCopyright (c) 2019 SICARA\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\"\"\"\n\ndef grad_cam(model, inputs, class_index):\n    x = tf.keras.Input((None, None, 3))\n    conv_y = model.get_layer(index=0)(x, training=False)\n    y = model.get_layer(index=1)(conv_y, training=False)\n    grad_model = tf.keras.Model(x, [conv_y, y])\n    \n    with tf.GradientTape() as tape:\n        conv_outputs, predictions = grad_model(inputs, training=False)\n        loss = predictions[:, class_index]\n\n    grads = tape.gradient(loss, conv_outputs)\n    \n    cams = []\n    for grad, output in zip(grads, conv_outputs):\n        weights = tf.reduce_mean(grad, axis=(0, 1))\n        cam = tf.reduce_sum(tf.multiply(weights, output), axis=-1).numpy()\n        cams.append(cam)\n    return cams","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_recids = splits[0][1]\nparsed_val = (parsed_trainval\n              .filter(create_recid_filter(val_recids)))\n\nvdataset = (parsed_val\n    .map(_cut_wav_val)\n    .map(_wav_to_spec))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## before training"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(5, 2, figsize=(16,20))\nfor i, sample in enumerate(vdataset.take(5)):\n    inpts = _preprocess_img(sample['audio_spec'], False)\n    show_spectrogram(sample, ax[i,0], True)\n    sid = sample['species_id']\n    cams = grad_cam(model, tf.expand_dims(inpts, 0), sid)\n    ax[i,1].set_title(\"grad-cam\")\n    ax[i,1].imshow(inpts[::-1,:,0], aspect='auto', interpolation='nearest', cmap='magma')\n    ax[i,1].imshow(cv2.resize(cams[0][::-1], (WIDTH, HEIGHT)), cmap='magma', aspect='auto', interpolation='nearest', alpha=0.5)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## after training"},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_weights(\"./model_best_0.h5\")\nfig, ax = plt.subplots(5, 2, figsize=(16,20))\nfor i, sample in enumerate(vdataset.take(5)):\n    inpts = _preprocess_img(sample['audio_spec'], False)\n    show_spectrogram(sample, ax[i,0], True)\n    sid = sample['species_id']\n    cams = grad_cam(model, tf.expand_dims(inpts, 0), sid)\n    ax[i,1].set_title(\"grad-cam\")\n    ax[i,1].imshow(inpts[::-1,:,0], aspect='auto', interpolation='nearest', cmap='magma')\n    ax[i,1].imshow(cv2.resize(cams[0][::-1], (WIDTH, HEIGHT)), cmap='magma', aspect='auto', interpolation='nearest', alpha=0.5)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir gradcams","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for split_i in range(5):\n    model.load_weights(f\"./model_best_{split_i}.h5\")\n    fig, ax = plt.subplots(5, 2, figsize=(16,20))\n    fig.suptitle(f\"split {split_i}\")\n    for i, sample in enumerate(vdataset.take(5)):\n        inpts = _preprocess_img(sample['audio_spec'], False)\n        show_spectrogram(sample, ax[i,0], True)\n        sid = sample['species_id']\n        cams = grad_cam(model, tf.expand_dims(inpts, 0), sid)\n        ax[i,1].set_title(\"grad-cam\")\n        ax[i,1].imshow(inpts[::-1,:,0], aspect='auto', interpolation='nearest', cmap='magma')\n        ax[i,1].imshow(cv2.resize(cams[0][::-1], (WIDTH, HEIGHT)), cmap='magma', aspect='auto', interpolation='nearest', alpha=0.5)\n    plt.savefig(f\"./gradcams/split_{split_i}.png\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}