{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"* enable 8 TPUs\n* ResNet50 backbone\n    * from v3: more accurate initial bias in the final layer\n* mixup disabled\n* use only TP samples\n* cut 6sec around annotation.\n* Stratified 5-fold and ensemble (just summing up probabilities)\n    * from v3: clip-wise splitting instead of annotation-wise\n* from v2: Grad-CAM visualization\n* augmentation\n    * gaussian noise ($\\sigma=0.3$ while the image variance is 1)\n    * random_brightness\n    * specaugment\n* v5: support TF2.4","metadata":{}},{"cell_type":"code","source":"!pip install iterative-stratification","metadata":{"execution":{"iopub.status.busy":"2021-11-08T19:41:35.751832Z","iopub.execute_input":"2021-11-08T19:41:35.752168Z","iopub.status.idle":"2021-11-08T19:41:45.598300Z","shell.execute_reply.started":"2021-11-08T19:41:35.752088Z","shell.execute_reply":"2021-11-08T19:41:45.597483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow_addons as tfa\nimport numpy as np\nfrom pathlib import Path\nimport io\nimport matplotlib.pyplot as plt\nimport soundfile as sf\nimport librosa\nfrom kaggle_datasets import KaggleDatasets\nfrom tqdm import tqdm\nimport pandas as pd\n# from sklearn.model_selection import StratifiedKFold\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold\nimport seaborn as sns\nfrom IPython.display import Audio\nimport cv2\n\ntf.__version__","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-08T19:41:45.600128Z","iopub.execute_input":"2021-11-08T19:41:45.600393Z","iopub.status.idle":"2021-11-08T19:41:53.736924Z","shell.execute_reply.started":"2021-11-08T19:41:45.600362Z","shell.execute_reply":"2021-11-08T19:41:53.736073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cfg = {\n    'parse_params': {\n        'cut_time': 10,\n    },\n    'data_params': {\n        'sample_time': 6, # assert 60 % sample_time == 0\n        'spec_fmax': 24000.0,\n        'spec_fmin': 40.0,\n        'spec_mel': 224,\n        'mel_power': 2,\n        'img_shape': (224, 512)\n    },\n    'model_params': {\n        'batchsize_per_tpu': 16,\n        'iteration_per_epoch': 64,\n        'epoch': 15,\n        'arch': tf.keras.applications.ResNet50,\n        'arch_preprocess': tf.keras.applications.resnet50.preprocess_input,\n        'freeze_to': 0,  # Freeze to backbone.layers[:freeze_to]. If None, all layers in the backbone will be freezed.\n        'loss': {\n            'fn': tfa.losses.SigmoidFocalCrossEntropy,\n            'params': {},\n        },\n        'optim': {\n            'fn': tfa.optimizers.RectifiedAdam,\n            'params': {'lr': 1e-3, 'total_steps': 15*64, 'warmup_proportion': 0.3, 'min_lr': 1e-6},\n        },\n        'mixup': False\n    }\n}","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2021-11-08T19:41:58.181489Z","iopub.execute_input":"2021-11-08T19:41:58.182124Z","iopub.status.idle":"2021-11-08T19:41:58.191845Z","shell.execute_reply.started":"2021-11-08T19:41:58.182086Z","shell.execute_reply":"2021-11-08T19:41:58.190837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# detect and init the TPU\ntpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect()\nstrategy = tf.distribute.TPUStrategy(tpu)\nprint(\"Number of devices: \", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T19:41:59.204306Z","iopub.execute_input":"2021-11-08T19:41:59.204581Z","iopub.status.idle":"2021-11-08T19:42:04.502745Z","shell.execute_reply.started":"2021-11-08T19:41:59.204553Z","shell.execute_reply":"2021-11-08T19:42:04.502077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTOTUNE = tf.data.experimental.AUTOTUNE\nGCS_DS_PATH = KaggleDatasets().get_gcs_path()\n\nTRAIN_TFREC = GCS_DS_PATH + \"/tfrecords/train\"\nTEST_TFREC = GCS_DS_PATH + \"/tfrecords/test\"","metadata":{"execution":{"iopub.status.busy":"2021-11-08T19:42:04.503812Z","iopub.execute_input":"2021-11-08T19:42:04.504056Z","iopub.status.idle":"2021-11-08T19:42:04.901170Z","shell.execute_reply.started":"2021-11-08T19:42:04.504015Z","shell.execute_reply":"2021-11-08T19:42:04.900449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CUT = cfg['parse_params']['cut_time']\nSR = 48000     # all wave's sample rate may be 48k\n\nTIME = cfg['data_params']['sample_time']\n\nFMAX = cfg['data_params']['spec_fmax']\nFMIN = cfg['data_params']['spec_fmin']\nN_MEL = cfg['data_params']['spec_mel']\n\nHEIGHT, WIDTH = cfg['data_params']['img_shape']\n\nCLASS_N = 24","metadata":{"execution":{"iopub.status.busy":"2021-11-08T19:42:57.222068Z","iopub.execute_input":"2021-11-08T19:42:57.222579Z","iopub.status.idle":"2021-11-08T19:42:57.228783Z","shell.execute_reply.started":"2021-11-08T19:42:57.222531Z","shell.execute_reply":"2021-11-08T19:42:57.228138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Explore the tfrecords, Create dataset","metadata":{}},{"cell_type":"code","source":"raw_dataset = tf.data.TFRecordDataset([TRAIN_TFREC + '/00-148.tfrec'])\nraw_dataset","metadata":{"execution":{"iopub.status.busy":"2021-11-08T19:43:00.893245Z","iopub.execute_input":"2021-11-08T19:43:00.893806Z","iopub.status.idle":"2021-11-08T19:43:00.914563Z","shell.execute_reply.started":"2021-11-08T19:43:00.893759Z","shell.execute_reply":"2021-11-08T19:43:00.913549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## parse tfrecords","metadata":{}},{"cell_type":"code","source":"feature_description = {\n    'recording_id': tf.io.FixedLenFeature([], tf.string, default_value=''),\n    'audio_wav': tf.io.FixedLenFeature([], tf.string, default_value=''),\n    'label_info': tf.io.FixedLenFeature([], tf.string, default_value=''),\n}\nparse_dtype = {\n    'audio_wav': tf.float32,\n    'recording_id': tf.string,\n    'species_id': tf.int32,\n    'songtype_id': tf.int32,\n    't_min': tf.float32,\n    'f_min': tf.float32,\n    't_max': tf.float32,\n    'f_max':tf.float32,\n    'is_tp': tf.int32\n}\n\n@tf.function\ndef _parse_function(example_proto):\n    sample = tf.io.parse_single_example(example_proto, feature_description)\n    wav, _ = tf.audio.decode_wav(sample['audio_wav'], desired_channels=1) # mono\n    label_info = tf.strings.split(sample['label_info'], sep='\"')[1]\n    labels = tf.strings.split(label_info, sep=';')\n    \n    @tf.function\n    def _cut_audio(label):\n        items = tf.strings.split(label, sep=',')\n        spid = tf.squeeze(tf.strings.to_number(items[0], tf.int32))\n        soid = tf.squeeze(tf.strings.to_number(items[1], tf.int32))\n        tmin = tf.squeeze(tf.strings.to_number(items[2]))\n        fmin = tf.squeeze(tf.strings.to_number(items[3]))\n        tmax = tf.squeeze(tf.strings.to_number(items[4]))\n        fmax = tf.squeeze(tf.strings.to_number(items[5]))\n        tp = tf.squeeze(tf.strings.to_number(items[6], tf.int32))\n\n        tmax_s = tmax * tf.cast(SR, tf.float32)\n        tmin_s = tmin * tf.cast(SR, tf.float32)\n        cut_s = tf.cast(CUT * SR, tf.float32)\n        all_s = tf.cast(60 * SR, tf.float32)\n        tsize_s = tmax_s - tmin_s\n        cut_min = tf.cast(\n            tf.maximum(0.0, \n                tf.minimum(tmin_s - (cut_s - tsize_s) / 2,\n                           tf.minimum(tmax_s + (cut_s - tsize_s) / 2, all_s) - cut_s)\n            ), tf.int32\n        )\n        cut_max = cut_min + CUT * SR\n        \n        _sample = {\n            'audio_wav': tf.reshape(wav[cut_min:cut_max], [CUT*SR]),\n            'recording_id': sample['recording_id'],\n            'species_id': spid,\n            'songtype_id': soid,\n            't_min': tmin - tf.cast(cut_min, tf.float32)/tf.cast(SR, tf.float32),\n            'f_min': fmin,\n            't_max': tmax - tf.cast(cut_min, tf.float32)/tf.cast(SR, tf.float32),\n            'f_max': fmax,\n            'is_tp': tp\n        }\n        return _sample\n    \n    samples = tf.map_fn(_cut_audio, labels, dtype=parse_dtype)\n    return samples\n\nparsed_dataset = raw_dataset.map(_parse_function).unbatch()","metadata":{"execution":{"iopub.status.busy":"2021-11-08T19:43:01.795029Z","iopub.execute_input":"2021-11-08T19:43:01.795580Z","iopub.status.idle":"2021-11-08T19:43:02.495621Z","shell.execute_reply.started":"2021-11-08T19:43:01.795536Z","shell.execute_reply":"2021-11-08T19:43:02.494671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@tf.function\ndef _cut_wav(x):\n    # random cut in training\n    cut_min = tf.random.uniform([], maxval=tf.minimum((CUT-TIME)*SR, tf.cast(x['t_max']*SR, tf.int32)), dtype=tf.int32)\n    cut_max = cut_min + TIME * SR\n    cutwave = tf.reshape(x['audio_wav'][cut_min:cut_max], [TIME*SR])\n    y = {}\n    y.update(x)\n    y['audio_wav'] = cutwave\n    y['t_min'] = tf.maximum(0.0, x['t_min'] - tf.cast(cut_min, tf.float32) / SR)\n    y['t_max'] = tf.maximum(0.0, x['t_max'] - tf.cast(cut_min, tf.float32) / SR)\n    return y\n    \n@tf.function\ndef _cut_wav_val(x):\n    # center crop in validation\n    cut_min = tf.minimum((CUT-TIME)*SR // 2, tf.cast((x['t_min'] + x['t_max']) / 2 * SR, tf.int32))\n    cut_max = cut_min + TIME * SR\n    cutwave = tf.reshape(x['audio_wav'][cut_min:cut_max], [TIME*SR])\n    y = {}\n    y.update(x)\n    y['audio_wav'] = cutwave\n    y['t_min'] = tf.maximum(0.0, x['t_min'] - tf.cast(cut_min, tf.float32) / SR)\n    y['t_max'] = tf.maximum(0.0, x['t_max'] - tf.cast(cut_min, tf.float32) / SR)\n    return y","metadata":{"execution":{"iopub.status.busy":"2021-11-08T19:43:03.799035Z","iopub.execute_input":"2021-11-08T19:43:03.799315Z","iopub.status.idle":"2021-11-08T19:43:03.812570Z","shell.execute_reply.started":"2021-11-08T19:43:03.799288Z","shell.execute_reply":"2021-11-08T19:43:03.811568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@tf.function\ndef _filtTP(x):\n    return x['is_tp'] == 1","metadata":{"execution":{"iopub.status.busy":"2021-11-08T19:43:05.628483Z","iopub.execute_input":"2021-11-08T19:43:05.628810Z","iopub.status.idle":"2021-11-08T19:43:05.633639Z","shell.execute_reply.started":"2021-11-08T19:43:05.628777Z","shell.execute_reply":"2021-11-08T19:43:05.632986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_wav(sample, ax):\n    wav = sample[\"audio_wav\"].numpy()\n    rate = SR\n    ax.plot(np.arange(len(wav)) / rate, wav)\n    ax.set_title(\n        sample[\"recording_id\"].numpy().decode()\n        + (\"/%d\" % sample[\"species_id\"])\n        + (\"TP\" if sample[\"is_tp\"] else \"FP\"))\n\n    return Audio((wav * 2**15).astype(np.int16), rate=rate)\n\nfig, ax = plt.subplots(figsize=(15, 3))\nshow_wav(next(iter(parsed_dataset)), ax)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T19:43:06.961671Z","iopub.execute_input":"2021-11-08T19:43:06.962537Z","iopub.status.idle":"2021-11-08T19:43:10.281316Z","shell.execute_reply.started":"2021-11-08T19:43:06.962502Z","shell.execute_reply":"2021-11-08T19:43:10.280398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## create mel-spectrogram","metadata":{}},{"cell_type":"code","source":"@tf.function\ndef _wav_to_spec(x):\n    mel_power = cfg['data_params']['mel_power']\n    \n    stfts = tf.signal.stft(x[\"audio_wav\"], frame_length=2048, frame_step=512, fft_length=2048)\n    spectrograms = tf.abs(stfts) ** mel_power\n\n    # Warp the linear scale spectrograms into the mel-scale.\n    num_spectrogram_bins = stfts.shape[-1]\n    lower_edge_hertz, upper_edge_hertz, num_mel_bins = FMIN, FMAX, N_MEL\n    \n    linear_to_mel_weight_matrix = tf.signal.linear_to_mel_weight_matrix(\n      num_mel_bins, num_spectrogram_bins, SR, lower_edge_hertz,\n      upper_edge_hertz)\n    mel_spectrograms = tf.tensordot(\n      spectrograms, linear_to_mel_weight_matrix, 1)\n    mel_spectrograms.set_shape(spectrograms.shape[:-1].concatenate(\n      linear_to_mel_weight_matrix.shape[-1:]))\n\n    # Compute a stabilized log to get log-magnitude mel-scale spectrograms.\n    log_mel_spectrograms = tf.math.log(mel_spectrograms + 1e-6)\n\n    y = {\n        'audio_spec': tf.transpose(log_mel_spectrograms), # (num_mel_bins, frames)\n    }\n    y.update(x)\n    return y\n\nspec_dataset = parsed_dataset.filter(_filtTP).map(_cut_wav).map(_wav_to_spec)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T19:43:33.663489Z","iopub.execute_input":"2021-11-08T19:43:33.663814Z","iopub.status.idle":"2021-11-08T19:43:34.307997Z","shell.execute_reply.started":"2021-11-08T19:43:33.663781Z","shell.execute_reply":"2021-11-08T19:43:34.307075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,5))\nfor i, s in enumerate(spec_dataset.take(3)):\n    plt.subplot(1,3,i+1)\n    plt.imshow(s['audio_spec'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-08T19:43:35.325458Z","iopub.execute_input":"2021-11-08T19:43:35.325802Z","iopub.status.idle":"2021-11-08T19:43:36.447931Z","shell.execute_reply.started":"2021-11-08T19:43:35.325766Z","shell.execute_reply":"2021-11-08T19:43:36.446361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import librosa.display\nimport matplotlib.patches as patches\n\ndef show_spectrogram(sample, ax, showlabel=False):\n    S_dB = sample[\"audio_spec\"].numpy()\n    img = librosa.display.specshow(S_dB, x_axis='time',\n                             y_axis='mel', sr=SR,\n                             fmax=FMAX, fmin=FMIN, ax=ax, cmap='magma')\n    ax.set(title=f'Mel-frequency spectrogram of {sample[\"recording_id\"].numpy().decode()}')\n    sid, fmin, fmax, tmin, tmax, istp = (\n            sample[\"species_id\"], sample[\"f_min\"], sample[\"f_max\"], sample[\"t_min\"], sample[\"t_max\"], sample[\"is_tp\"])\n    ec = '#00ff00' if istp == 1 else '#0000ff'\n    ax.add_patch(\n        patches.Rectangle(xy=(tmin, fmin), width=tmax-tmin, height=fmax-fmin, ec=ec, fill=False)\n    )\n\n    if showlabel:\n        ax.text(tmin, fmax, \n        f\"{sid.numpy().item()} {'tp' if istp == 1 else 'fp'}\",\n        horizontalalignment='left', verticalalignment='bottom', color=ec, fontsize=16)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T19:43:39.887079Z","iopub.execute_input":"2021-11-08T19:43:39.887391Z","iopub.status.idle":"2021-11-08T19:43:39.900277Z","shell.execute_reply.started":"2021-11-08T19:43:39.887348Z","shell.execute_reply":"2021-11-08T19:43:39.899391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(15,3))\nshow_spectrogram(next(iter(spec_dataset)), ax, showlabel=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T19:43:40.168371Z","iopub.execute_input":"2021-11-08T19:43:40.169041Z","iopub.status.idle":"2021-11-08T19:43:40.992810Z","shell.execute_reply.started":"2021-11-08T19:43:40.169002Z","shell.execute_reply":"2021-11-08T19:43:40.991794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# in validation, annotations will come to the center\nfig, ax = plt.subplots(figsize=(15,3))\nshow_spectrogram(next(iter(parsed_dataset.filter(_filtTP).map(_cut_wav_val).map(_wav_to_spec))), ax, showlabel=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T19:43:44.685549Z","iopub.execute_input":"2021-11-08T19:43:44.685884Z","iopub.status.idle":"2021-11-08T19:43:45.742983Z","shell.execute_reply.started":"2021-11-08T19:43:44.685850Z","shell.execute_reply":"2021-11-08T19:43:45.742056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for sample in spec_dataset.take(5):\n    fig, ax = plt.subplots(figsize=(15,3))\n    show_spectrogram(sample, ax, showlabel=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T19:43:45.744668Z","iopub.execute_input":"2021-11-08T19:43:45.745014Z","iopub.status.idle":"2021-11-08T19:43:49.090603Z","shell.execute_reply.started":"2021-11-08T19:43:45.744980Z","shell.execute_reply":"2021-11-08T19:43:49.089631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## create labels","metadata":{}},{"cell_type":"code","source":"@tf.function\ndef _create_annot(x):\n    targ = tf.one_hot(x[\"species_id\"], CLASS_N, on_value=x[\"is_tp\"], off_value=0)\n    \n    return {\n        'input': x[\"audio_spec\"],\n        'target': tf.cast(targ, tf.float32)\n    }\n\nannot_dataset = spec_dataset.map(_create_annot)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T19:43:49.092145Z","iopub.execute_input":"2021-11-08T19:43:49.092388Z","iopub.status.idle":"2021-11-08T19:43:49.174166Z","shell.execute_reply.started":"2021-11-08T19:43:49.092360Z","shell.execute_reply":"2021-11-08T19:43:49.173334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## proprocessing and data augmentation\n\nIn training, I use\n\n* gaussian noise\n* random brightness\n* specaugment","metadata":{}},{"cell_type":"code","source":"@tf.function\ndef _preprocess_img(x, training=False):\n    image = tf.expand_dims(x, axis=-1)\n    image = tf.image.resize(image, [HEIGHT, WIDTH])\n    image = tf.image.per_image_standardization(image)\n    \n    @tf.function\n    def _specaugment(image):\n        ERASE_TIME = 40\n        ERASE_MEL = 20\n        ERASE_TIME_N = 1\n        ERASE_MEL_N = 1\n        image = tf.expand_dims(image, axis=0)\n        xoff = tf.random.uniform([ERASE_TIME_N], minval=ERASE_TIME//2, maxval=WIDTH-ERASE_TIME//2, dtype=tf.int32)\n        xsize_half = tf.random.uniform([ERASE_TIME_N], minval=ERASE_TIME//4, maxval=ERASE_TIME//2, dtype=tf.int32)\n        yoff = tf.random.uniform([ERASE_MEL_N], minval=ERASE_MEL//2, maxval=HEIGHT-ERASE_MEL//2, dtype=tf.int32)\n        ysize_half = tf.random.uniform([ERASE_MEL_N], minval=ERASE_MEL//4, maxval=ERASE_MEL//2, dtype=tf.int32)\n        for i in range(ERASE_TIME_N):\n            image = tfa.image.cutout(image, [HEIGHT, xsize_half[i]*2], offset=[HEIGHT//2, xoff[i]])\n        for i in range(ERASE_MEL_N):\n            image = tfa.image.cutout(image, [ysize_half[i]*2, WIDTH], offset=[yoff[i], WIDTH//2])\n        return tf.squeeze(image, axis=0)\n    \n    if training:\n        # gaussian\n        gau = tf.keras.layers.GaussianNoise(0.3)\n        image = tf.cond(tf.random.uniform([]) < 0.5, lambda: gau(image, training=True), lambda: image)\n        # brightness\n        image = tf.image.random_brightness(image, 0.2)\n        # specaugment\n        image = tf.cond(tf.random.uniform([]) < 0.5, lambda: _specaugment(image), lambda: image, name='spec_cond')\n        \n    image = (image - tf.reduce_min(image)) / (tf.reduce_max(image) - tf.reduce_min(image)) * 255.0 # rescale to [0, 255]\n    image = tf.image.grayscale_to_rgb(image)\n    image = cfg['model_params']['arch_preprocess'](image)\n\n    return image\n\n@tf.function\ndef _preprocess(x):\n    image = _preprocess_img(x['input'], True)\n    return (image, x[\"target\"])\n\n@tf.function\ndef _preprocess_val(x):\n    image = _preprocess_img(x['input'], False)\n    return (image, x[\"target\"])\n\n@tf.function\ndef _preprocess_test(x):\n    image = _preprocess_img(x['audio_spec'], False)\n    return (image, x[\"recording_id\"])","metadata":{"execution":{"iopub.status.busy":"2021-11-08T19:57:20.578857Z","iopub.execute_input":"2021-11-08T19:57:20.579233Z","iopub.status.idle":"2021-11-08T19:57:20.600235Z","shell.execute_reply.started":"2021-11-08T19:57:20.579196Z","shell.execute_reply":"2021-11-08T19:57:20.599380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(16, 4))\nfor i, (inp, targ) in enumerate(annot_dataset.map(_preprocess).take(6)):\n    plt.subplot(2,3,i+1)\n    plt.imshow(inp.numpy()[:,:,0])\n    t = targ.numpy()\n    if t.sum() == 0:\n        plt.title(f'FP')\n    else:\n        plt.title(f'{t.nonzero()[0]}')\n    plt.colorbar()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-08T19:57:22.704922Z","iopub.execute_input":"2021-11-08T19:57:22.705242Z","iopub.status.idle":"2021-11-08T19:57:25.700883Z","shell.execute_reply.started":"2021-11-08T19:57:22.705201Z","shell.execute_reply":"2021-11-08T19:57:25.699983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"def create_model():\n    with strategy.scope():\n        backbone = cfg['model_params']['arch'](include_top=False, weights='imagenet')\n        \n        if cfg['model_params']['freeze_to'] is None:\n            for layer in backbone.layers:\n                layer.trainable = False\n        else:\n            for layer in backbone.layers[:cfg['model_params']['freeze_to']]:\n                layer.trainable = False\n                \n        # negative bias for stability (Section 4.1 in \"Focal Loss\" (https://arxiv.org/abs/1708.02002))\n        prior = 1/CLASS_N\n        pi = -np.log((1-prior) / prior) # sigmoid(pi) == prior\n\n        head = tf.keras.Sequential([\n            tf.keras.layers.GlobalAveragePooling2D(),\n            tf.keras.layers.BatchNormalization(),\n            tf.keras.layers.Dropout(0.4),\n            tf.keras.layers.Dense(1024, activation='relu', kernel_initializer=tf.keras.initializers.he_normal()),\n            tf.keras.layers.BatchNormalization(),\n            tf.keras.layers.Dropout(0.4),\n            tf.keras.layers.Dense(CLASS_N, bias_initializer=tf.keras.initializers.Constant(pi))\n        ])\n        model = tf.keras.Sequential([backbone, head])\n    return model\n\nmodel = create_model()\nmodel.summary()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-11-08T19:45:51.826848Z","iopub.execute_input":"2021-11-08T19:45:51.827682Z","iopub.status.idle":"2021-11-08T19:46:02.802317Z","shell.execute_reply.started":"2021-11-08T19:45:51.827642Z","shell.execute_reply":"2021-11-08T19:46:02.800585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@tf.function\ndef _mixup(inp, targ):\n    indice = tf.range(len(inp))\n    indice = tf.random.shuffle(indice)\n    sinp = tf.gather(inp, indice, axis=0)\n    starg = tf.gather(targ, indice, axis=0)\n    \n    alpha = 0.2\n    t = tf.compat.v1.distributions.Beta(alpha, alpha).sample([len(inp)])\n    tx = tf.reshape(t, [-1, 1, 1, 1])\n    ty = tf.reshape(t, [-1, 1])\n    x = inp * tx + sinp * (1-tx)\n    y = targ * ty + starg * (1-ty)\n#     y = tf.minimum(targ + starg, 1.0) # for multi-label???\n    return x, y","metadata":{"execution":{"iopub.status.busy":"2021-11-08T19:46:02.805861Z","iopub.execute_input":"2021-11-08T19:46:02.806107Z","iopub.status.idle":"2021-11-08T19:46:02.814004Z","shell.execute_reply.started":"2021-11-08T19:46:02.806078Z","shell.execute_reply":"2021-11-08T19:46:02.813104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tfrecs = sorted(tf.io.gfile.glob(TRAIN_TFREC + '/*.tfrec'))\nparsed_trainval = (tf.data.TFRecordDataset(tfrecs, num_parallel_reads=AUTOTUNE)\n                    .map(_parse_function, num_parallel_calls=AUTOTUNE).unbatch()\n                    .filter(_filtTP))","metadata":{"execution":{"iopub.status.busy":"2021-11-08T19:46:02.815124Z","iopub.execute_input":"2021-11-08T19:46:02.815408Z","iopub.status.idle":"2021-11-08T19:46:02.948348Z","shell.execute_reply.started":"2021-11-08T19:46:02.815379Z","shell.execute_reply":"2021-11-08T19:46:02.946542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Stratified 5-Fold","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/rfcx-species-audio-detection/train_tp.csv\")\n\ntable = df.groupby('recording_id')['species_id'].apply(\n    lambda x: tf.scatter_nd(tf.expand_dims(x, 1), np.ones_like(x), shape=[CLASS_N]).numpy()\n).reset_index()\n\nskf = MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=0)\nidx_splits = list(skf.split(table.recording_id, np.stack(table.species_id.to_numpy())))\nsplits = list(map(lambda xs: (table.recording_id[xs[0]].to_numpy(), table.recording_id[xs[1]].to_numpy()), idx_splits))","metadata":{"execution":{"iopub.status.busy":"2021-11-08T19:46:06.851705Z","iopub.execute_input":"2021-11-08T19:46:06.852020Z","iopub.status.idle":"2021-11-08T19:46:08.945107Z","shell.execute_reply.started":"2021-11-08T19:46:06.851989Z","shell.execute_reply":"2021-11-08T19:46:08.943940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_table = df.set_index('recording_id')\nplt.hist([_table.loc[splits[0][0], 'species_id'], _table.loc[splits[0][1], 'species_id']],\n         bins=CLASS_N,stacked=True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-08T19:46:08.946756Z","iopub.execute_input":"2021-11-08T19:46:08.947032Z","iopub.status.idle":"2021-11-08T19:46:09.227466Z","shell.execute_reply.started":"2021-11-08T19:46:08.947001Z","shell.execute_reply":"2021-11-08T19:46:09.226557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_recid_filter(recids):\n    @tf.function\n    def _filt(x):\n        return tf.reduce_any(recids == x['recording_id'])\n    return _filt","metadata":{"execution":{"iopub.status.busy":"2021-11-08T19:46:09.229059Z","iopub.execute_input":"2021-11-08T19:46:09.229304Z","iopub.status.idle":"2021-11-08T19:46:09.235430Z","shell.execute_reply.started":"2021-11-08T19:46:09.229275Z","shell.execute_reply":"2021-11-08T19:46:09.234800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Other setup","metadata":{}},{"cell_type":"code","source":"def create_train_dataset(batchsize, train_recids):\n    global parsed_trainval\n    print(\"train split: %d\" % len(train_recids))\n    parsed_train = (parsed_trainval\n                    .filter(create_recid_filter(train_recids)))\n\n    dataset = (parsed_train.cache()\n        .shuffle(len(train_recids))\n        .repeat()\n        .map(_cut_wav, num_parallel_calls=AUTOTUNE)\n        .map(_wav_to_spec, num_parallel_calls=AUTOTUNE)\n        .map(_create_annot, num_parallel_calls=AUTOTUNE)\n        .map(_preprocess, num_parallel_calls=AUTOTUNE)\n        .batch(batchsize))\n\n    if cfg['model_params']['mixup']:\n        dataset = (dataset.map(_mixup, num_parallel_calls=AUTOTUNE)\n                    .prefetch(AUTOTUNE))\n    else:\n        dataset = dataset.prefetch(AUTOTUNE)\n    return dataset\n\ndef create_val_dataset(batchsize, val_recids):\n    global parsed_trainval\n    print(\"val split: %d\" % len(val_recids))\n    parsed_val = (parsed_trainval\n                  .filter(create_recid_filter(val_recids)))\n\n    vdataset = (parsed_val\n        .map(_cut_wav_val, num_parallel_calls=AUTOTUNE)\n        .map(_wav_to_spec, num_parallel_calls=AUTOTUNE)\n        .map(_create_annot, num_parallel_calls=AUTOTUNE)\n        .map(_preprocess_val, num_parallel_calls=AUTOTUNE)\n        .batch(8*strategy.num_replicas_in_sync)\n        .cache())\n    return vdataset","metadata":{"execution":{"iopub.status.busy":"2021-11-08T19:46:10.038541Z","iopub.execute_input":"2021-11-08T19:46:10.039448Z","iopub.status.idle":"2021-11-08T19:46:10.050572Z","shell.execute_reply.started":"2021-11-08T19:46:10.039387Z","shell.execute_reply":"2021-11-08T19:46:10.049696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Metrics","metadata":{}},{"cell_type":"code","source":"# from https://www.kaggle.com/carlthome/l-lrap-metric-for-tf-keras\n@tf.function\ndef _one_sample_positive_class_precisions(example):\n    y_true, y_pred = example\n\n    retrieved_classes = tf.argsort(y_pred, direction='DESCENDING')\n    class_rankings = tf.argsort(retrieved_classes)\n    retrieved_class_true = tf.gather(y_true, retrieved_classes)\n    retrieved_cumulative_hits = tf.math.cumsum(tf.cast(retrieved_class_true, tf.float32))\n\n    idx = tf.where(y_true)[:, 0]\n    i = tf.boolean_mask(class_rankings, y_true)\n    r = tf.gather(retrieved_cumulative_hits, i)\n    c = 1 + tf.cast(i, tf.float32)\n    precisions = r / c\n\n    dense = tf.scatter_nd(idx[:, None], precisions, [y_pred.shape[0]])\n    return dense\n\nclass LWLRAP(tf.keras.metrics.Metric):\n    def __init__(self, num_classes, name='lwlrap'):\n        super().__init__(name=name)\n\n        self._precisions = self.add_weight(\n            name='per_class_cumulative_precision',\n            shape=[num_classes],\n            initializer='zeros',\n        )\n\n        self._counts = self.add_weight(\n            name='per_class_cumulative_count',\n            shape=[num_classes],\n            initializer='zeros',\n        )\n\n    def update_state(self, y_true, y_pred, sample_weight=None):\n        precisions = tf.map_fn(\n            fn=_one_sample_positive_class_precisions,\n            elems=(y_true, y_pred),\n            dtype=(tf.float32),\n        )\n\n        increments = tf.cast(precisions > 0, tf.float32)\n        total_increments = tf.reduce_sum(increments, axis=0)\n        total_precisions = tf.reduce_sum(precisions, axis=0)\n\n        self._precisions.assign_add(total_precisions)\n        self._counts.assign_add(total_increments)        \n\n    def result(self):\n        per_class_lwlrap = self._precisions / tf.maximum(self._counts, 1.0)\n        per_class_weight = self._counts / tf.reduce_sum(self._counts)\n        overall_lwlrap = tf.reduce_sum(per_class_lwlrap * per_class_weight)\n        return overall_lwlrap\n\n    def reset_states(self):\n        self._precisions.assign(self._precisions * 0)\n        self._counts.assign(self._counts * 0)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T19:46:32.457516Z","iopub.execute_input":"2021-11-08T19:46:32.457963Z","iopub.status.idle":"2021-11-08T19:46:32.473629Z","shell.execute_reply.started":"2021-11-08T19:46:32.457924Z","shell.execute_reply":"2021-11-08T19:46:32.472747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Testset and Inference function","metadata":{}},{"cell_type":"code","source":"def _parse_function_test(example_proto):\n    sample = tf.io.parse_single_example(example_proto, feature_description)\n    wav, _ = tf.audio.decode_wav(sample['audio_wav'], desired_channels=1) # mono\n    \n    @tf.function\n    def _cut_audio(i):\n        _sample = {\n            'audio_wav': tf.reshape(wav[i*SR*TIME:(i+1)*SR*TIME], [SR*TIME]),\n            'recording_id': sample['recording_id']\n        }\n        return _sample\n\n    return tf.map_fn(_cut_audio, tf.range(60//TIME), dtype={\n        'audio_wav': tf.float32,\n        'recording_id': tf.string\n    })\n\ndef inference(model):\n    tdataset = (tf.data.TFRecordDataset(tf.io.gfile.glob(TEST_TFREC + '/*.tfrec'), num_parallel_reads=AUTOTUNE)\n        .map(_parse_function_test, num_parallel_calls=AUTOTUNE).unbatch()\n        .map(_wav_to_spec, num_parallel_calls=AUTOTUNE)\n        .map(_preprocess_test, num_parallel_calls=AUTOTUNE)\n        .batch(128*(60//TIME)).prefetch(AUTOTUNE))\n    \n    rec_ids = []\n    probs = []\n    for inp, rec_id in tqdm(tdataset):\n        with strategy.scope():\n            pred = model.predict_on_batch(tf.reshape(inp, [-1, HEIGHT, WIDTH, 3]))\n            prob = tf.sigmoid(pred)\n            prob = tf.reduce_max(tf.reshape(prob, [-1, 60//TIME, CLASS_N]), axis=1)\n\n        rec_id_stack = tf.reshape(rec_id, [-1, 60//TIME])\n        for rec in rec_id.numpy():\n            assert len(np.unique(rec)) == 1\n        rec_ids.append(rec_id_stack.numpy()[:,0])\n        probs.append(prob.numpy())\n        \n    crec_ids = np.concatenate(rec_ids)\n    cprobs = np.concatenate(probs)\n    \n    sub = pd.DataFrame({\n        'recording_id': list(map(lambda x: x.decode(), crec_ids.tolist())),\n        **{f's{i}': cprobs[:,i] for i in range(CLASS_N)}\n    })\n    sub = sub.sort_values('recording_id')\n    return sub","metadata":{"execution":{"iopub.status.busy":"2021-11-08T19:46:33.353292Z","iopub.execute_input":"2021-11-08T19:46:33.353620Z","iopub.status.idle":"2021-11-08T19:46:33.369451Z","shell.execute_reply.started":"2021-11-08T19:46:33.353588Z","shell.execute_reply":"2021-11-08T19:46:33.368755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Now start training!","metadata":{}},{"cell_type":"code","source":"def plot_history(history, name):\n    plt.figure(figsize=(8,3))\n    plt.subplot(1,2,1)\n    plt.plot(history.history[\"loss\"])\n    plt.plot(history.history[\"val_loss\"])\n    plt.legend(['Train', 'Test'], loc='upper left')\n    plt.title(\"loss\")\n    # plt.yscale('log')\n\n    plt.subplot(1,2,2)\n    plt.plot(history.history[\"lwlrap\"])\n    plt.plot(history.history[\"val_lwlrap\"])\n    plt.legend(['Train', 'Test'], loc='upper left')\n    plt.title(\"metric\")\n\n    plt.savefig(name)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T19:46:34.185242Z","iopub.execute_input":"2021-11-08T19:46:34.185619Z","iopub.status.idle":"2021-11-08T19:46:34.197700Z","shell.execute_reply.started":"2021-11-08T19:46:34.185576Z","shell.execute_reply":"2021-11-08T19:46:34.196751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_and_inference(splits, split_id):\n    batchsize = cfg['model_params']['batchsize_per_tpu'] * strategy.num_replicas_in_sync\n    print(\"batchsize\", batchsize)\n    loss_fn = cfg['model_params']['loss']['fn'](from_logits=True, **cfg['model_params']['loss']['params'])\n\n    idx_train_tf = tf.constant(splits[split_id][0])\n    idx_val_tf = tf.constant(splits[split_id][1])\n\n    dataset = create_train_dataset(batchsize, idx_train_tf)\n    vdataset = create_val_dataset(batchsize, idx_val_tf)\n    \n    optimizer = cfg['model_params']['optim']['fn'](**cfg['model_params']['optim']['params'])\n    model = create_model()\n    with strategy.scope():\n        model.compile(optimizer=optimizer, loss=loss_fn, metrics=[LWLRAP(CLASS_N)])\n\n    cp_options = tf.train.CheckpointOptions(experimental_io_device=\"/job:localhost\")\n    history = model.fit(dataset,\n                        steps_per_epoch=cfg['model_params']['iteration_per_epoch'],\n                        epochs=cfg['model_params']['epoch'],\n                        validation_data=vdataset,\n                        callbacks=[\n                            tf.keras.callbacks.ModelCheckpoint(\n                                filepath='model_best_%d.h5' % split_id,\n                                save_weights_only=True,\n                                monitor='val_lwlrap',\n                                mode='max',\n                                save_best_only=True,\n                                options=cp_options,\n                            ),\n                        ])\n    plot_history(history, 'history_%d.png' % split_id)\n    \n    ### inference ###\n    model.load_weights('model_best_%d.h5' % split_id, options=cp_options)\n    return inference(model), history","metadata":{"execution":{"iopub.status.busy":"2021-11-08T19:46:34.869012Z","iopub.execute_input":"2021-11-08T19:46:34.869311Z","iopub.status.idle":"2021-11-08T19:46:34.881348Z","shell.execute_reply.started":"2021-11-08T19:46:34.869279Z","shell.execute_reply":"2021-11-08T19:46:34.880122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train and inference\n# sub, _ = train_and_inference(splits, 0)\n\n# N-fold ensemble\nsub = sum(\n    map(\n        lambda i: train_and_inference(splits, i)[0].set_index('recording_id'),\n        range(len(splits))\n    )\n).reset_index()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-11-08T19:46:35.794102Z","iopub.execute_input":"2021-11-08T19:46:35.794530Z","iopub.status.idle":"2021-11-08T19:53:03.925894Z","shell.execute_reply.started":"2021-11-08T19:46:35.794500Z","shell.execute_reply":"2021-11-08T19:53:03.924872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.describe()","metadata":{"execution":{"iopub.status.busy":"2021-11-08T18:57:11.082444Z","iopub.execute_input":"2021-11-08T18:57:11.083207Z","iopub.status.idle":"2021-11-08T18:57:11.170391Z","shell.execute_reply.started":"2021-11-08T18:57:11.083161Z","shell.execute_reply":"2021-11-08T18:57:11.169529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Show Grad-CAM\nLet's take a look at Grad-CAM and check if my model is trained properly","metadata":{}},{"cell_type":"code","source":"model = create_model()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# modified https://github.com/sicara/tf-explain/blob/8dff129ff7b1012dba2761a61e3c3e68e9ecbec2/tf_explain/core/grad_cam.py\n\"\"\"MIT License\n\nCopyright (c) 2019 SICARA\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\"\"\"\n\ndef grad_cam(model, inputs, class_index):\n    x = tf.keras.Input((None, None, 3))\n    conv_y = model.get_layer(index=0)(x, training=False)\n    y = model.get_layer(index=1)(conv_y, training=False)\n    grad_model = tf.keras.Model(x, [conv_y, y])\n    \n    with tf.GradientTape() as tape:\n        conv_outputs, predictions = grad_model(inputs, training=False)\n        loss = predictions[:, class_index]\n\n    grads = tape.gradient(loss, conv_outputs)\n    \n    cams = []\n    for grad, output in zip(grads, conv_outputs):\n        weights = tf.reduce_mean(grad, axis=(0, 1))\n        cam = tf.reduce_sum(tf.multiply(weights, output), axis=-1).numpy()\n        cams.append(cam)\n    return cams","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_recids = splits[0][1]\nparsed_val = (parsed_trainval\n              .filter(create_recid_filter(val_recids)))\n\nvdataset = (parsed_val\n    .map(_cut_wav_val)\n    .map(_wav_to_spec))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## before training","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(5, 2, figsize=(16,20))\nfor i, sample in enumerate(vdataset.take(5)):\n    inpts = _preprocess_img(sample['audio_spec'], False)\n    show_spectrogram(sample, ax[i,0], True)\n    sid = sample['species_id']\n    cams = grad_cam(model, tf.expand_dims(inpts, 0), sid)\n    ax[i,1].set_title(\"grad-cam\")\n    ax[i,1].imshow(inpts[::-1,:,0], aspect='auto', interpolation='nearest', cmap='magma')\n    ax[i,1].imshow(cv2.resize(cams[0][::-1], (WIDTH, HEIGHT)), cmap='magma', aspect='auto', interpolation='nearest', alpha=0.5)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## after training","metadata":{}},{"cell_type":"code","source":"model.load_weights(\"./model_best_0.h5\")\nfig, ax = plt.subplots(5, 2, figsize=(16,20))\nfor i, sample in enumerate(vdataset.take(5)):\n    inpts = _preprocess_img(sample['audio_spec'], False)\n    show_spectrogram(sample, ax[i,0], True)\n    sid = sample['species_id']\n    cams = grad_cam(model, tf.expand_dims(inpts, 0), sid)\n    ax[i,1].set_title(\"grad-cam\")\n    ax[i,1].imshow(inpts[::-1,:,0], aspect='auto', interpolation='nearest', cmap='magma')\n    ax[i,1].imshow(cv2.resize(cams[0][::-1], (WIDTH, HEIGHT)), cmap='magma', aspect='auto', interpolation='nearest', alpha=0.5)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"👍","metadata":{}},{"cell_type":"code","source":"!mkdir gradcams","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for split_i in range(5):\n    model.load_weights(f\"./model_best_{split_i}.h5\")\n    fig, ax = plt.subplots(5, 2, figsize=(16,20))\n    fig.suptitle(f\"split {split_i}\")\n    for i, sample in enumerate(vdataset.take(5)):\n        inpts = _preprocess_img(sample['audio_spec'], False)\n        show_spectrogram(sample, ax[i,0], True)\n        sid = sample['species_id']\n        cams = grad_cam(model, tf.expand_dims(inpts, 0), sid)\n        ax[i,1].set_title(\"grad-cam\")\n        ax[i,1].imshow(inpts[::-1,:,0], aspect='auto', interpolation='nearest', cmap='magma')\n        ax[i,1].imshow(cv2.resize(cams[0][::-1], (WIDTH, HEIGHT)), cmap='magma', aspect='auto', interpolation='nearest', alpha=0.5)\n    plt.savefig(f\"./gradcams/split_{split_i}.png\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}