{"cells":[{"metadata":{},"cell_type":"markdown","source":"************"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow_addons as tfa\nimport numpy as np\nfrom pathlib import Path\nimport io\nimport matplotlib.pyplot as plt\nimport soundfile as sf\nimport librosa\nfrom kaggle_datasets import KaggleDatasets\nfrom tqdm import tqdm\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\nimport seaborn as sns\nfrom IPython.display import Audio\nimport cv2\n\ntf.__version__","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"cfg = {\n    'parse_params': {\n        'cut_time': 10,\n    },\n    'data_params': {\n        'sample_time': 6, # assert 60 % sample_time == 0\n        'spec_fmax': 24000.0,\n        'spec_fmin': 40.0,\n        'spec_mel': 224,\n        'mel_power': 2,\n        'img_shape': (224, 512)\n    },\n    'model_params': {\n        'batchsize_per_tpu': 16,\n        'iteration_per_epoch': 64,\n        'epoch': 15,\n        'arch': tf.keras.applications.ResNet50,\n        'arch_preprocess': tf.keras.applications.resnet50.preprocess_input,\n        'freeze_to': 0,  # Freeze to backbone.layers[:freeze_to]. If None, all layers in the backbone will be freezed.\n        'loss': {\n            'fn': tfa.losses.SigmoidFocalCrossEntropy,\n            'params': {},\n        },\n        'optim': {\n            'fn': tfa.optimizers.RectifiedAdam,\n            'params': {'lr': 1e-3, 'total_steps': 15*64, 'warmup_proportion': 0.3, 'min_lr': 1e-6},\n        },\n        'mixup': False\n    }\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# detect and init the TPU\ntpu = tf.distribute.cluster_resolver.TPUClusterResolver()\ntf.config.experimental_connect_to_cluster(tpu)\ntf.tpu.experimental.initialize_tpu_system(tpu)\nprint(\"All devices: \", tf.config.list_logical_devices('TPU'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"strategy = tf.distribute.experimental.TPUStrategy(tpu)\nAUTOTUNE = tf.data.experimental.AUTOTUNE\nGCS_DS_PATH = KaggleDatasets().get_gcs_path()\n\nTRAIN_TFREC = GCS_DS_PATH + \"/tfrecords/train\"\nTEST_TFREC = GCS_DS_PATH + \"/tfrecords/test\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CUT = cfg['parse_params']['cut_time']\nSR = 48000     # all wave's sample rate may be 48k\n\nTIME = cfg['data_params']['sample_time']\n\nFMAX = cfg['data_params']['spec_fmax']\nFMIN = cfg['data_params']['spec_fmin']\nN_MEL = cfg['data_params']['spec_mel']\n\nHEIGHT, WIDTH = cfg['data_params']['img_shape']\n\nCLASS_N = 24","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Explore the tfrecords, Create dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"raw_dataset = tf.data.TFRecordDataset([TRAIN_TFREC + '/00-148.tfrec'])\nraw_dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## parse tfrecords"},{"metadata":{"trusted":true},"cell_type":"code","source":"feature_description = {\n    'recording_id': tf.io.FixedLenFeature([], tf.string, default_value=''),\n    'audio_wav': tf.io.FixedLenFeature([], tf.string, default_value=''),\n    'label_info': tf.io.FixedLenFeature([], tf.string, default_value=''),\n}\nparse_dtype = {\n    'audio_wav': tf.float32,\n    'recording_id': tf.string,\n    'species_id': tf.int32,\n    'songtype_id': tf.int32,\n    't_min': tf.float32,\n    'f_min': tf.float32,\n    't_max': tf.float32,\n    'f_max':tf.float32,\n    'is_tp': tf.int32\n}\n\n@tf.function\ndef _parse_function(example_proto):\n    sample = tf.io.parse_single_example(example_proto, feature_description)\n    wav, _ = tf.audio.decode_wav(sample['audio_wav'], desired_channels=1) # mono\n    label_info = tf.strings.split(sample['label_info'], sep='\"')[1]\n    labels = tf.strings.split(label_info, sep=';')\n    \n    @tf.function\n    def _cut_audio(label):\n        items = tf.strings.split(label, sep=',')\n        spid = tf.squeeze(tf.strings.to_number(items[0], tf.int32))\n        soid = tf.squeeze(tf.strings.to_number(items[1], tf.int32))\n        tmin = tf.squeeze(tf.strings.to_number(items[2]))\n        fmin = tf.squeeze(tf.strings.to_number(items[3]))\n        tmax = tf.squeeze(tf.strings.to_number(items[4]))\n        fmax = tf.squeeze(tf.strings.to_number(items[5]))\n        tp = tf.squeeze(tf.strings.to_number(items[6], tf.int32))\n\n        tmax_s = tmax * tf.cast(SR, tf.float32)\n        tmin_s = tmin * tf.cast(SR, tf.float32)\n        cut_s = tf.cast(CUT * SR, tf.float32)\n        all_s = tf.cast(60 * SR, tf.float32)\n        tsize_s = tmax_s - tmin_s\n        cut_min = tf.cast(\n            tf.maximum(0.0, \n                tf.minimum(tmin_s - (cut_s - tsize_s) / 2,\n                           tf.minimum(tmax_s + (cut_s - tsize_s) / 2, all_s) - cut_s)\n            ), tf.int32\n        )\n        cut_max = cut_min + CUT * SR\n        \n        _sample = {\n            'audio_wav': tf.reshape(wav[cut_min:cut_max], [CUT*SR]),\n            'recording_id': sample['recording_id'],\n            'species_id': spid,\n            'songtype_id': soid,\n            't_min': tmin - tf.cast(cut_min, tf.float32)/tf.cast(SR, tf.float32),\n            'f_min': fmin,\n            't_max': tmax - tf.cast(cut_min, tf.float32)/tf.cast(SR, tf.float32),\n            'f_max': fmax,\n            'is_tp': tp\n        }\n        return _sample\n    \n    samples = tf.map_fn(_cut_audio, labels, dtype=parse_dtype)\n    return samples\n\nparsed_dataset = raw_dataset.map(_parse_function).unbatch()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef _cut_wav(x):\n    # random cut in training\n    cut_min = tf.random.uniform([], maxval=tf.minimum((CUT-TIME)*SR, tf.cast(x['t_max']*SR, tf.int32)), dtype=tf.int32)\n    cut_max = cut_min + TIME * SR\n    cutwave = tf.reshape(x['audio_wav'][cut_min:cut_max], [TIME*SR])\n    y = {}\n    y.update(x)\n    y['audio_wav'] = cutwave\n    y['t_min'] = tf.maximum(0.0, x['t_min'] - tf.cast(cut_min, tf.float32) / SR)\n    y['t_max'] = tf.maximum(0.0, x['t_max'] - tf.cast(cut_min, tf.float32) / SR)\n    return y\n    \n@tf.function\ndef _cut_wav_val(x):\n    # center crop in validation\n    cut_min = tf.minimum((CUT-TIME)*SR // 2, tf.cast((x['t_min'] + x['t_max']) / 2 * SR, tf.int32))\n    cut_max = cut_min + TIME * SR\n    cutwave = tf.reshape(x['audio_wav'][cut_min:cut_max], [TIME*SR])\n    y = {}\n    y.update(x)\n    y['audio_wav'] = cutwave\n    y['t_min'] = tf.maximum(0.0, x['t_min'] - tf.cast(cut_min, tf.float32) / SR)\n    y['t_max'] = tf.maximum(0.0, x['t_max'] - tf.cast(cut_min, tf.float32) / SR)\n    return y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef _filtTP(x):\n    return x['is_tp'] == 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_wav(sample, ax):\n    wav = sample[\"audio_wav\"].numpy()\n    rate = SR\n    ax.plot(np.arange(len(wav)) / rate, wav)\n    ax.set_title(\n        sample[\"recording_id\"].numpy().decode()\n        + (\"/%d\" % sample[\"species_id\"])\n        + (\"TP\" if sample[\"is_tp\"] else \"FP\"))\n\n    return Audio((wav * 2**15).astype(np.int16), rate=rate)\n\nfig, ax = plt.subplots(figsize=(15, 3))\nshow_wav(next(iter(parsed_dataset)), ax)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## create mel-spectrogram"},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef _wav_to_spec(x):\n    mel_power = cfg['data_params']['mel_power']\n    \n    stfts = tf.signal.stft(x[\"audio_wav\"], frame_length=2048, frame_step=512, fft_length=2048)\n    spectrograms = tf.abs(stfts) ** mel_power\n\n    # Warp the linear scale spectrograms into the mel-scale.\n    num_spectrogram_bins = stfts.shape[-1]\n    lower_edge_hertz, upper_edge_hertz, num_mel_bins = FMIN, FMAX, N_MEL\n    \n    linear_to_mel_weight_matrix = tf.signal.linear_to_mel_weight_matrix(\n      num_mel_bins, num_spectrogram_bins, SR, lower_edge_hertz,\n      upper_edge_hertz)\n    mel_spectrograms = tf.tensordot(\n      spectrograms, linear_to_mel_weight_matrix, 1)\n    mel_spectrograms.set_shape(spectrograms.shape[:-1].concatenate(\n      linear_to_mel_weight_matrix.shape[-1:]))\n\n    # Compute a stabilized log to get log-magnitude mel-scale spectrograms.\n    log_mel_spectrograms = tf.math.log(mel_spectrograms + 1e-6)\n\n    y = {\n        'audio_spec': tf.transpose(log_mel_spectrograms), # (num_mel_bins, frames)\n    }\n    y.update(x)\n    return y\n\nspec_dataset = parsed_dataset.filter(_filtTP).map(_cut_wav).map(_wav_to_spec)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(12,5))\nfor i, s in enumerate(spec_dataset.take(3)):\n    plt.subplot(1,3,i+1)\n    plt.imshow(s['audio_spec'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import librosa.display\nimport matplotlib.patches as patches\n\ndef show_spectrogram(sample, ax, showlabel=False):\n    S_dB = sample[\"audio_spec\"].numpy()\n    img = librosa.display.specshow(S_dB, x_axis='time',\n                             y_axis='mel', sr=SR,\n                             fmax=FMAX, fmin=FMIN, ax=ax, cmap='magma')\n    ax.set(title=f'Mel-frequency spectrogram of {sample[\"recording_id\"].numpy().decode()}')\n    sid, fmin, fmax, tmin, tmax, istp = (\n            sample[\"species_id\"], sample[\"f_min\"], sample[\"f_max\"], sample[\"t_min\"], sample[\"t_max\"], sample[\"is_tp\"])\n    ec = '#00ff00' if istp == 1 else '#0000ff'\n    ax.add_patch(\n        patches.Rectangle(xy=(tmin, fmin), width=tmax-tmin, height=fmax-fmin, ec=ec, fill=False)\n    )\n\n    if showlabel:\n        ax.text(tmin, fmax, \n        f\"{sid.numpy().item()} {'tp' if istp == 1 else 'fp'}\",\n        horizontalalignment='left', verticalalignment='bottom', color=ec, fontsize=16)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(15,3))\nshow_spectrogram(next(iter(spec_dataset)), ax, showlabel=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# in validation, annotations will come to the center\nfig, ax = plt.subplots(figsize=(15,3))\nshow_spectrogram(next(iter(parsed_dataset.filter(_filtTP).map(_cut_wav_val).map(_wav_to_spec))), ax, showlabel=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for sample in spec_dataset.take(5):\n    fig, ax = plt.subplots(figsize=(15,3))\n    show_spectrogram(sample, ax, showlabel=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## create labels"},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef _create_annot(x):\n    targ = tf.one_hot(x[\"species_id\"], CLASS_N, on_value=x[\"is_tp\"], off_value=0)\n    \n    return {\n        'input': x[\"audio_spec\"],\n        'target': tf.cast(targ, tf.float32)\n    }\n\nannot_dataset = spec_dataset.map(_create_annot)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef _preprocess_img(x, training=False):\n    image = tf.expand_dims(x, axis=-1)\n    image = tf.image.resize(image, [HEIGHT, WIDTH])\n    image = tf.image.per_image_standardization(image)\n    \n    @tf.function\n    def _specaugment(image):\n        ERASE_TIME = 50\n        ERASE_MEL = 16\n        image = tf.expand_dims(image, axis=0)\n        xoff = tf.random.uniform([2], minval=ERASE_TIME//2, maxval=WIDTH-ERASE_TIME//2, dtype=tf.int32)\n        xsize = tf.random.uniform([2], minval=ERASE_TIME//2, maxval=ERASE_TIME, dtype=tf.int32)\n        yoff = tf.random.uniform([2], minval=ERASE_MEL//2, maxval=HEIGHT-ERASE_MEL//2, dtype=tf.int32)\n        ysize = tf.random.uniform([2], minval=ERASE_MEL//2, maxval=ERASE_MEL, dtype=tf.int32)\n        image = tfa.image.cutout(image, [HEIGHT, xsize[0]], offset=[HEIGHT//2, xoff[0]])\n        image = tfa.image.cutout(image, [HEIGHT, xsize[1]], offset=[HEIGHT//2, xoff[1]])\n        image = tfa.image.cutout(image, [ysize[0], WIDTH], offset=[yoff[0], WIDTH//2])\n        image = tfa.image.cutout(image, [ysize[1], WIDTH], offset=[yoff[1], WIDTH//2])\n        image = tf.squeeze(image, axis=0)\n        return image\n    \n    if training:\n        # gaussian\n        gau = tf.keras.layers.GaussianNoise(0.3)\n        image = tf.cond(tf.random.uniform([]) < 0.5, lambda: gau(image, training=True), lambda: image)\n        # brightness\n        image = tf.image.random_brightness(image, 0.2)\n        # specaugment\n        image = tf.cond(tf.random.uniform([]) < 0.5, lambda: _specaugment(image), lambda: image)\n        \n    image = (image - tf.reduce_min(image)) / (tf.reduce_max(image) - tf.reduce_min(image)) * 255.0 # rescale to [0, 255]\n    image = tf.image.grayscale_to_rgb(image)\n    image = cfg['model_params']['arch_preprocess'](image)\n\n    return image\n\n@tf.function\ndef _preprocess(x):\n    image = _preprocess_img(x['input'], True)\n    return (image, x[\"target\"])\n\n@tf.function\ndef _preprocess_val(x):\n    image = _preprocess_img(x['input'], False)\n    return (image, x[\"target\"])\n\n@tf.function\ndef _preprocess_test(x):\n    image = _preprocess_img(x['audio_spec'], False)\n    return (image, x[\"recording_id\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for inp, targ in annot_dataset.map(_preprocess).take(2):\n    plt.imshow(inp.numpy()[:,:,0])\n    t = targ.numpy()\n    if t.sum() == 0:\n        plt.title(f'FP')\n    else:\n        plt.title(f'{t.nonzero()[0]}')\n    plt.colorbar()\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"def create_model():\n    with strategy.scope():\n        backbone = cfg['model_params']['arch'](include_top=False, weights='imagenet')\n        \n        if cfg['model_params']['freeze_to'] is None:\n            for layer in backbone.layers:\n                layer.trainable = False\n        else:\n            for layer in backbone.layers[:cfg['model_params']['freeze_to']]:\n                layer.trainable = False\n\n        head = tf.keras.Sequential([\n            tf.keras.layers.GlobalAveragePooling2D(),\n            tf.keras.layers.BatchNormalization(),\n            tf.keras.layers.Dropout(0.4),\n            tf.keras.layers.Dense(1024, activation='relu', kernel_initializer=tf.keras.initializers.he_normal()),\n            tf.keras.layers.BatchNormalization(),\n            tf.keras.layers.Dropout(0.4),\n            tf.keras.layers.Dense(CLASS_N, bias_initializer=tf.keras.initializers.Constant(-2.))])\n        model = tf.keras.Sequential([backbone, head])\n    return model\n\nmodel = create_model()\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef _mixup(inp, targ):\n    indice = tf.range(len(inp))\n    indice = tf.random.shuffle(indice)\n    sinp = tf.gather(inp, indice, axis=0)\n    starg = tf.gather(targ, indice, axis=0)\n    \n    alpha = 0.2\n    t = tf.compat.v1.distributions.Beta(alpha, alpha).sample([len(inp)])\n    tx = tf.reshape(t, [-1, 1, 1, 1])\n    ty = tf.reshape(t, [-1, 1])\n    x = inp * tx + sinp * (1-tx)\n    y = targ * ty + starg * (1-ty)\n#     y = tf.minimum(targ + starg, 1.0) # for multi-label???\n    return x, y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tfrecs = sorted(tf.io.gfile.glob(TRAIN_TFREC + '/*.tfrec'))\nparsed_trainval = (tf.data.TFRecordDataset(tfrecs, num_parallel_reads=AUTOTUNE)\n                    .map(_parse_function, num_parallel_calls=AUTOTUNE).unbatch()\n                    .filter(_filtTP).enumerate())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Stratified 5-Fold"},{"metadata":{"trusted":true},"cell_type":"code","source":"indices = []\nspid = []\nrecid = []\n\nfor i, sample in tqdm(parsed_trainval.prefetch(AUTOTUNE)):\n    indices.append(i.numpy())\n    spid.append(sample['species_id'].numpy())\n    recid.append(sample['recording_id'].numpy().decode())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"table = pd.DataFrame({'indices': indices, 'species_id': spid, 'recording_id': recid})\ntable","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"skf = StratifiedKFold(n_splits=5, random_state=42, shuffle=True)\nsplits = list(skf.split(table.index, table.species_id))\n\nplt.hist([table.loc[splits[0][0], 'species_id'], table.loc[splits[0][1], 'species_id']], bins=CLASS_N,stacked=True)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_idx_filter(indice):\n    @tf.function\n    def _filt(i, x):\n        return tf.reduce_any(indice == i)\n    return _filt\n\n@tf.function\ndef _remove_idx(i, x):\n    return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Other setup"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_train_dataset(batchsize, train_idx):\n    global parsed_trainval\n    parsed_train = (parsed_trainval\n                    .filter(create_idx_filter(train_idx))\n                    .map(_remove_idx))\n    \n    dataset = (parsed_train.cache()\n        .shuffle(len(train_idx))\n        .repeat()\n        .map(_cut_wav, num_parallel_calls=AUTOTUNE)\n        .map(_wav_to_spec, num_parallel_calls=AUTOTUNE)\n        .map(_create_annot, num_parallel_calls=AUTOTUNE)\n        .map(_preprocess, num_parallel_calls=AUTOTUNE)\n        .batch(batchsize))\n\n    if cfg['model_params']['mixup']:\n        dataset = (dataset.map(_mixup, num_parallel_calls=AUTOTUNE)\n                    .prefetch(AUTOTUNE))\n    else:\n        dataset = dataset.prefetch(AUTOTUNE)\n    return dataset\n\ndef create_val_dataset(batchsize, val_idx):\n    global parsed_trainval\n    parsed_val = (parsed_trainval\n                  .filter(create_idx_filter(val_idx))\n                  .map(_remove_idx))\n\n    vdataset = (parsed_val\n        .map(_cut_wav_val, num_parallel_calls=AUTOTUNE)\n        .map(_wav_to_spec, num_parallel_calls=AUTOTUNE)\n        .map(_create_annot, num_parallel_calls=AUTOTUNE)\n        .map(_preprocess_val, num_parallel_calls=AUTOTUNE)\n        .batch(8*strategy.num_replicas_in_sync)\n        .cache())\n    return vdataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Metrics"},{"metadata":{"trusted":true},"cell_type":"code","source":"# from https://www.kaggle.com/carlthome/l-lrap-metric-for-tf-keras\n@tf.function\ndef _one_sample_positive_class_precisions(example):\n    y_true, y_pred = example\n\n    retrieved_classes = tf.argsort(y_pred, direction='DESCENDING')\n    class_rankings = tf.argsort(retrieved_classes)\n    retrieved_class_true = tf.gather(y_true, retrieved_classes)\n    retrieved_cumulative_hits = tf.math.cumsum(tf.cast(retrieved_class_true, tf.float32))\n\n    idx = tf.where(y_true)[:, 0]\n    i = tf.boolean_mask(class_rankings, y_true)\n    r = tf.gather(retrieved_cumulative_hits, i)\n    c = 1 + tf.cast(i, tf.float32)\n    precisions = r / c\n\n    dense = tf.scatter_nd(idx[:, None], precisions, [y_pred.shape[0]])\n    return dense\n\nclass LWLRAP(tf.keras.metrics.Metric):\n    def __init__(self, num_classes, name='lwlrap'):\n        super().__init__(name=name)\n\n        self._precisions = self.add_weight(\n            name='per_class_cumulative_precision',\n            shape=[num_classes],\n            initializer='zeros',\n        )\n\n        self._counts = self.add_weight(\n            name='per_class_cumulative_count',\n            shape=[num_classes],\n            initializer='zeros',\n        )\n\n    def update_state(self, y_true, y_pred, sample_weight=None):\n        precisions = tf.map_fn(\n            fn=_one_sample_positive_class_precisions,\n            elems=(y_true, y_pred),\n            dtype=(tf.float32),\n        )\n\n        increments = tf.cast(precisions > 0, tf.float32)\n        total_increments = tf.reduce_sum(increments, axis=0)\n        total_precisions = tf.reduce_sum(precisions, axis=0)\n\n        self._precisions.assign_add(total_precisions)\n        self._counts.assign_add(total_increments)        \n\n    def result(self):\n        per_class_lwlrap = self._precisions / tf.maximum(self._counts, 1.0)\n        per_class_weight = self._counts / tf.reduce_sum(self._counts)\n        overall_lwlrap = tf.reduce_sum(per_class_lwlrap * per_class_weight)\n        return overall_lwlrap\n\n    def reset_states(self):\n        self._precisions.assign(self._precisions * 0)\n        self._counts.assign(self._counts * 0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Testset and Inference function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def _parse_function_test(example_proto):\n    sample = tf.io.parse_single_example(example_proto, feature_description)\n    wav, _ = tf.audio.decode_wav(sample['audio_wav'], desired_channels=1) # mono\n    \n    @tf.function\n    def _cut_audio(i):\n        _sample = {\n            'audio_wav': tf.reshape(wav[i*SR*TIME:(i+1)*SR*TIME], [SR*TIME]),\n            'recording_id': sample['recording_id']\n        }\n        return _sample\n\n    return tf.map_fn(_cut_audio, tf.range(60//TIME), dtype={\n        'audio_wav': tf.float32,\n        'recording_id': tf.string\n    })\n\ndef inference(model):\n    tdataset = (tf.data.TFRecordDataset(tf.io.gfile.glob(TEST_TFREC + '/*.tfrec'), num_parallel_reads=AUTOTUNE)\n        .map(_parse_function_test, num_parallel_calls=AUTOTUNE).unbatch()\n        .map(_wav_to_spec, num_parallel_calls=AUTOTUNE)\n        .map(_preprocess_test, num_parallel_calls=AUTOTUNE)\n        .batch(128*(60//TIME)).prefetch(AUTOTUNE))\n    \n    rec_ids = []\n    probs = []\n    for inp, rec_id in tqdm(tdataset):\n        with strategy.scope():\n            pred = model.predict_on_batch(tf.reshape(inp, [-1, HEIGHT, WIDTH, 3]))\n            prob = tf.sigmoid(pred)\n            prob = tf.reduce_max(tf.reshape(prob, [-1, 60//TIME, CLASS_N]), axis=1)\n\n        rec_id_stack = tf.reshape(rec_id, [-1, 60//TIME])\n        for rec in rec_id.numpy():\n            assert len(np.unique(rec)) == 1\n        rec_ids.append(rec_id_stack.numpy()[:,0])\n        probs.append(prob.numpy())\n        \n    crec_ids = np.concatenate(rec_ids)\n    cprobs = np.concatenate(probs)\n    \n    sub = pd.DataFrame({\n        'recording_id': list(map(lambda x: x.decode(), crec_ids.tolist())),\n        **{f's{i}': cprobs[:,i] for i in range(CLASS_N)}\n    })\n    sub = sub.sort_values('recording_id')\n    return sub","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Now start training!"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_history(history, name):\n    plt.figure(figsize=(8,3))\n    plt.subplot(1,2,1)\n    plt.plot(history.history[\"loss\"])\n    plt.plot(history.history[\"val_loss\"])\n    plt.legend(['Train', 'Test'], loc='upper left')\n    plt.title(\"loss\")\n    # plt.yscale('log')\n\n    plt.subplot(1,2,2)\n    plt.plot(history.history[\"lwlrap\"])\n    plt.plot(history.history[\"val_lwlrap\"])\n    plt.legend(['Train', 'Test'], loc='upper left')\n    plt.title(\"metric\")\n\n    plt.savefig(name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_and_inference(splits, split_id):\n    batchsize = cfg['model_params']['batchsize_per_tpu'] * strategy.num_replicas_in_sync\n    print(\"batchsize\", batchsize)\n    loss_fn = cfg['model_params']['loss']['fn'](from_logits=True, **cfg['model_params']['loss']['params'])\n\n    idx_train_tf = tf.constant(splits[split_id][0])\n    idx_val_tf = tf.constant(splits[split_id][1])\n\n    dataset = create_train_dataset(batchsize, idx_train_tf)\n    vdataset = create_val_dataset(batchsize, idx_val_tf)\n    \n    optimizer = cfg['model_params']['optim']['fn'](**cfg['model_params']['optim']['params'])\n    model = create_model()\n    with strategy.scope():\n        model.compile(optimizer=optimizer, loss=loss_fn, metrics=[LWLRAP(CLASS_N)])\n        \n    history = model.fit(dataset,\n                        steps_per_epoch=cfg['model_params']['iteration_per_epoch'],\n                        epochs=cfg['model_params']['epoch'],\n                        validation_data=vdataset,\n                        callbacks=[\n                            tf.keras.callbacks.ReduceLROnPlateau(\n                                'val_lwlrap', patience=10\n                            ),\n                            tf.keras.callbacks.ModelCheckpoint(\n                                filepath='model_best_%d.h5' % split_id,\n                                save_weights_only=True,\n                                monitor='val_lwlrap',\n                                mode='max',\n                                save_best_only=True),\n                        ])\n    plot_history(history, 'history_%d.png' % split_id)\n    \n    ### inference ###\n    model.load_weights('model_best_%d.h5' % split_id)\n    return inference(model), history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"# train and inference\n# sub, _ = train_and_inference(splits, 0)\n\n# N-fold ensemble\nsub = sum(\n    map(\n        lambda i: train_and_inference(splits, i)[0].set_index('recording_id'),\n        range(len(splits))\n    )\n).reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Show Grad-CAM\nLet's take a look at Grad-CAM and check if my model is trained properly"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = create_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# modified https://github.com/sicara/tf-explain/blob/8dff129ff7b1012dba2761a61e3c3e68e9ecbec2/tf_explain/core/grad_cam.py\n\"\"\"MIT License\n\nCopyright (c) 2019 SICARA\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\"\"\"\n\ndef grad_cam(model, inputs, class_index):\n    x = tf.keras.Input((None, None, 3))\n    conv_y = model.get_layer(index=0)(x, training=False)\n    y = model.get_layer(index=1)(conv_y, training=False)\n    grad_model = tf.keras.Model(x, [conv_y, y])\n    \n    with tf.GradientTape() as tape:\n        conv_outputs, predictions = grad_model(inputs, training=False)\n        loss = predictions[:, class_index]\n\n    grads = tape.gradient(loss, conv_outputs)\n    \n    cams = []\n    for grad, output in zip(grads, conv_outputs):\n        weights = tf.reduce_mean(grad, axis=(0, 1))\n        cam = tf.reduce_sum(tf.multiply(weights, output), axis=-1).numpy()\n        cams.append(cam)\n    return cams","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_idx = splits[0][1]\nparsed_val = (parsed_trainval\n              .filter(create_idx_filter(val_idx))\n              .map(_remove_idx))\n\nvdataset = (parsed_val\n    .map(_cut_wav_val)\n    .map(_wav_to_spec))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## before training"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(5, 2, figsize=(16,20))\nfor i, sample in enumerate(vdataset.take(5)):\n    inpts = _preprocess_img(sample['audio_spec'], False)\n    show_spectrogram(sample, ax[i,0], True)\n    sid = sample['species_id']\n    cams = grad_cam(model, tf.expand_dims(inpts, 0), sid)\n    ax[i,1].set_title(\"grad-cam\")\n    ax[i,1].imshow(inpts[::-1,:,0], aspect='auto', interpolation='nearest', cmap='magma')\n    ax[i,1].imshow(cv2.resize(cams[0][::-1], (WIDTH, HEIGHT)), cmap='magma', aspect='auto', interpolation='nearest', alpha=0.5)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## after training"},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_weights(\"./model_best_0.h5\")\nfig, ax = plt.subplots(5, 2, figsize=(16,20))\nfor i, sample in enumerate(vdataset.take(5)):\n    inpts = _preprocess_img(sample['audio_spec'], False)\n    show_spectrogram(sample, ax[i,0], True)\n    sid = sample['species_id']\n    cams = grad_cam(model, tf.expand_dims(inpts, 0), sid)\n    ax[i,1].set_title(\"grad-cam\")\n    ax[i,1].imshow(inpts[::-1,:,0], aspect='auto', interpolation='nearest', cmap='magma')\n    ax[i,1].imshow(cv2.resize(cams[0][::-1], (WIDTH, HEIGHT)), cmap='magma', aspect='auto', interpolation='nearest', alpha=0.5)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"👍"},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir gradcams","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for split_i in range(5):\n    model.load_weights(f\"./model_best_{split_i}.h5\")\n    fig, ax = plt.subplots(5, 2, figsize=(16,20))\n    fig.suptitle(f\"split {i}\")\n    for i, sample in enumerate(vdataset.take(5)):\n        inpts = _preprocess_img(sample['audio_spec'], False)\n        show_spectrogram(sample, ax[i,0], True)\n        sid = sample['species_id']\n        cams = grad_cam(model, tf.expand_dims(inpts, 0), sid)\n        ax[i,1].set_title(\"grad-cam\")\n        ax[i,1].imshow(inpts[::-1,:,0], aspect='auto', interpolation='nearest', cmap='magma')\n        ax[i,1].imshow(cv2.resize(cams[0][::-1], (WIDTH, HEIGHT)), cmap='magma', aspect='auto', interpolation='nearest', alpha=0.5)\n    plt.savefig(f\"./gradcams/split_{split_i}.png\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}