{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\n\nCOLAB=False\nmodels_path=''\n\nfrom kaggle_datasets import KaggleDatasets\nGCS_DS_PATH = KaggleDatasets().get_gcs_path('rfcx-species-audio-detection')\nprint (GCS_DS_PATH)","metadata":{"id":"BHRl-lsznqCE","papermill":{"duration":0.617654,"end_time":"2021-01-09T21:35:23.39096","exception":false,"start_time":"2021-01-09T21:35:22.773306","status":"completed"},"tags":[],"outputId":"c8763935-f874-457e-8494-23c3e6378e03","execution":{"iopub.status.busy":"2021-05-27T20:18:39.387722Z","iopub.execute_input":"2021-05-27T20:18:39.388146Z","iopub.status.idle":"2021-05-27T20:18:40.354978Z","shell.execute_reply.started":"2021-05-27T20:18:39.388063Z","shell.execute_reply":"2021-05-27T20:18:40.353997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -U tensorflow==2.4\n!pip install -q tensorflow_io==0.17.0\nimport tensorflow_io as tfio\nimport tensorflow as tf\ntf.__version__","metadata":{"execution":{"iopub.status.busy":"2021-05-27T20:19:04.598229Z","iopub.execute_input":"2021-05-27T20:19:04.598586Z","iopub.status.idle":"2021-05-27T20:20:13.946531Z","shell.execute_reply.started":"2021-05-27T20:19:04.598557Z","shell.execute_reply":"2021-05-27T20:20:13.945409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\n!pip install image-classifiers\n!pip install tensorflow_addons\nimport tensorflow_addons as tfa\nimport numpy as np\nfrom pathlib import Path\nimport io\nimport matplotlib.pyplot as plt\n!pip install soundfile\nimport soundfile as sf\nimport librosa\nfrom tqdm import tqdm\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\nimport seaborn as sns\nfrom IPython.display import Audio\nfrom classification_models.keras import Classifiers","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","id":"eWsId8ltno6U","papermill":{"duration":30.534452,"end_time":"2021-01-09T21:35:54.222871","exception":false,"start_time":"2021-01-09T21:35:23.688419","status":"completed"},"tags":[],"outputId":"e74f669b-7a18-40cb-e96b-a7ae78b334c1","execution":{"iopub.status.busy":"2021-05-27T20:23:10.33921Z","iopub.execute_input":"2021-05-27T20:23:10.339577Z","iopub.status.idle":"2021-05-27T20:23:33.184235Z","shell.execute_reply.started":"2021-05-27T20:23:10.33955Z","shell.execute_reply":"2021-05-27T20:23:33.183249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from classification_models.keras import Classifiers","metadata":{"id":"JfNnWfoBPrN0","papermill":{"duration":0.048743,"end_time":"2021-01-09T21:35:54.312825","exception":false,"start_time":"2021-01-09T21:35:54.264082","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-27T20:24:01.148287Z","iopub.execute_input":"2021-05-27T20:24:01.148632Z","iopub.status.idle":"2021-05-27T20:24:01.153968Z","shell.execute_reply.started":"2021-05-27T20:24:01.148604Z","shell.execute_reply":"2021-05-27T20:24:01.153165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#SEED = 42\nimport random \nSEED = random.randint(0, 10000)\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    \nseed_everything(SEED)","metadata":{"id":"N1fcA3x8Ps4J","papermill":{"duration":0.052421,"end_time":"2021-01-09T21:35:54.406369","exception":false,"start_time":"2021-01-09T21:35:54.353948","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-27T20:24:02.93098Z","iopub.execute_input":"2021-05-27T20:24:02.931382Z","iopub.status.idle":"2021-05-27T20:24:02.937724Z","shell.execute_reply.started":"2021-05-27T20:24:02.931348Z","shell.execute_reply":"2021-05-27T20:24:02.936479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from https://github.com/qubvel/classification_models\nResNet18, preprocess_input = Classifiers.get('resnet18')","metadata":{"id":"1MRmH14VPzBg","papermill":{"duration":0.04859,"end_time":"2021-01-09T21:35:54.496437","exception":false,"start_time":"2021-01-09T21:35:54.447847","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-27T20:24:15.876904Z","iopub.execute_input":"2021-05-27T20:24:15.87728Z","iopub.status.idle":"2021-05-27T20:24:15.881997Z","shell.execute_reply.started":"2021-05-27T20:24:15.877247Z","shell.execute_reply":"2021-05-27T20:24:15.880784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cfg = {\n    'parse_params': {\n        'cut_time': 10,\n    },\n    'data_params': {\n        'sample_time': 6, # assert 60 % sample_time == 0\n        'spec_fmax': 24000.0,\n        'spec_fmin': 40.0,\n        'spec_mel': 300, \n        'mel_power': 2,\n        'img_shape': (300, 670)\n    },\n    'model_params': {\n        'batchsize_per_tpu': 8,\n        'iteration_per_epoch': 128,\n        'epoch': 25,\n        'arch': ResNet18,\n        'arch_preprocess': preprocess_input,\n        'freeze_to': 0,  # Freeze to backbone.layers[:freeze_to]. If None, all layers in the backbone will be freezed.\n        'loss': {\n            'fn': tfa.losses.SigmoidFocalCrossEntropy,\n            'params': {},\n        },\n        'optim': {\n            'fn': tfa.optimizers.RectifiedAdam,\n            'params': {'lr': 2e-3, 'total_steps': 18*64, 'warmup_proportion': 0.3, 'min_lr': 1e-6},\n        },\n        'mixup': True # False\n    }\n}","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","id":"Sb376cw6no6W","papermill":{"duration":0.053605,"end_time":"2021-01-09T21:35:54.591979","exception":false,"start_time":"2021-01-09T21:35:54.538374","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-27T20:24:22.043024Z","iopub.execute_input":"2021-05-27T20:24:22.043386Z","iopub.status.idle":"2021-05-27T20:24:22.050475Z","shell.execute_reply.started":"2021-05-27T20:24:22.043357Z","shell.execute_reply":"2021-05-27T20:24:22.049676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# detect and init the TPU\ntpu = tf.distribute.cluster_resolver.TPUClusterResolver()\ntf.config.experimental_connect_to_cluster(tpu)\ntf.tpu.experimental.initialize_tpu_system(tpu)\nprint(\"All devices: \", tf.config.list_logical_devices('TPU'))\ntpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)","metadata":{"id":"AilneWzono6Y","papermill":{"duration":4.107262,"end_time":"2021-01-09T21:35:58.741599","exception":false,"start_time":"2021-01-09T21:35:54.634337","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-27T20:24:31.6676Z","iopub.execute_input":"2021-05-27T20:24:31.66795Z","iopub.status.idle":"2021-05-27T20:24:37.888468Z","shell.execute_reply.started":"2021-05-27T20:24:31.667912Z","shell.execute_reply":"2021-05-27T20:24:37.887374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTOTUNE = tf.data.experimental.AUTOTUNE\n\nTRAIN_TFREC = GCS_DS_PATH + \"/tfrecords/train\"\nTEST_TFREC = GCS_DS_PATH + \"/tfrecords/test\"","metadata":{"id":"XXTFtxKbno6Z","papermill":{"duration":0.049283,"end_time":"2021-01-09T21:35:58.833563","exception":false,"start_time":"2021-01-09T21:35:58.78428","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-27T20:24:40.057997Z","iopub.execute_input":"2021-05-27T20:24:40.058393Z","iopub.status.idle":"2021-05-27T20:24:40.062796Z","shell.execute_reply.started":"2021-05-27T20:24:40.058357Z","shell.execute_reply":"2021-05-27T20:24:40.061628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CUT = cfg['parse_params']['cut_time']\nSR = 48000     # all wave's sample rate may be 48k\n\nTIME = cfg['data_params']['sample_time']\n\nFMAX = cfg['data_params']['spec_fmax']\nFMIN = cfg['data_params']['spec_fmin']\nN_MEL = cfg['data_params']['spec_mel']\n\nHEIGHT, WIDTH = cfg['data_params']['img_shape']\n\nCLASS_N = 24","metadata":{"id":"3BbfyLC4no6a","papermill":{"duration":0.052816,"end_time":"2021-01-09T21:35:58.928689","exception":false,"start_time":"2021-01-09T21:35:58.875873","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-27T20:24:42.106294Z","iopub.execute_input":"2021-05-27T20:24:42.106858Z","iopub.status.idle":"2021-05-27T20:24:42.111789Z","shell.execute_reply.started":"2021-05-27T20:24:42.106789Z","shell.execute_reply":"2021-05-27T20:24:42.111006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Explore the tfrecords, Create dataset","metadata":{"id":"qqI_Qymnno6a","papermill":{"duration":0.042825,"end_time":"2021-01-09T21:35:59.014075","exception":false,"start_time":"2021-01-09T21:35:58.97125","status":"completed"},"tags":[]}},{"cell_type":"code","source":"raw_dataset = tf.data.TFRecordDataset([TRAIN_TFREC + '/00-148.tfrec'])\nraw_dataset","metadata":{"id":"X30m1K0_no6a","papermill":{"duration":0.066747,"end_time":"2021-01-09T21:35:59.122717","exception":false,"start_time":"2021-01-09T21:35:59.05597","status":"completed"},"tags":[],"outputId":"83a47a77-436a-4399-87a4-ec8520bddbb8","execution":{"iopub.status.busy":"2021-05-27T20:24:44.762109Z","iopub.execute_input":"2021-05-27T20:24:44.762624Z","iopub.status.idle":"2021-05-27T20:24:44.78139Z","shell.execute_reply.started":"2021-05-27T20:24:44.762582Z","shell.execute_reply":"2021-05-27T20:24:44.780042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## parse tfrecords","metadata":{"id":"j88p-qsdno6b","papermill":{"duration":0.042477,"end_time":"2021-01-09T21:35:59.207562","exception":false,"start_time":"2021-01-09T21:35:59.165085","status":"completed"},"tags":[]}},{"cell_type":"code","source":"feature_description = {\n    'recording_id': tf.io.FixedLenFeature([], tf.string, default_value=''),\n    'audio_wav': tf.io.FixedLenFeature([], tf.string, default_value=''),\n    'label_info': tf.io.FixedLenFeature([], tf.string, default_value=''),\n}\nparse_dtype = {\n    'audio_wav': tf.float32,\n    'recording_id': tf.string,\n    'species_id': tf.int32,\n    'songtype_id': tf.int32,\n    't_min': tf.float32,\n    'f_min': tf.float32,\n    't_max': tf.float32,\n    'f_max':tf.float32,\n    'is_tp': tf.int32\n}\n\n@tf.function\ndef _parse_function(example_proto):\n    sample = tf.io.parse_single_example(example_proto, feature_description)\n    wav, _ = tf.audio.decode_wav(sample['audio_wav'], desired_channels=1) # mono\n    label_info = tf.strings.split(sample['label_info'], sep='\"')[1]\n    labels = tf.strings.split(label_info, sep=';')\n    \n    @tf.function\n    def _cut_audio(label):\n        items = tf.strings.split(label, sep=',')\n        spid = tf.squeeze(tf.strings.to_number(items[0], tf.int32))\n        soid = tf.squeeze(tf.strings.to_number(items[1], tf.int32))\n        tmin = tf.squeeze(tf.strings.to_number(items[2]))\n        fmin = tf.squeeze(tf.strings.to_number(items[3]))\n        tmax = tf.squeeze(tf.strings.to_number(items[4]))\n        fmax = tf.squeeze(tf.strings.to_number(items[5]))\n        tp = tf.squeeze(tf.strings.to_number(items[6], tf.int32))\n\n        tmax_s = tmax * tf.cast(SR, tf.float32)\n        tmin_s = tmin * tf.cast(SR, tf.float32)\n        cut_s = tf.cast(CUT * SR, tf.float32)\n        all_s = tf.cast(60 * SR, tf.float32)\n        tsize_s = tmax_s - tmin_s\n        cut_min = tf.cast(\n            tf.maximum(0.0, \n                tf.minimum(tmin_s - (cut_s - tsize_s) / 2,\n                           tf.minimum(tmax_s + (cut_s - tsize_s) / 2, all_s) - cut_s)\n            ), tf.int32\n        )\n        cut_max = cut_min + CUT * SR\n        \n        _sample = {\n            'audio_wav': tf.reshape(wav[cut_min:cut_max], [CUT*SR]),\n            'recording_id': sample['recording_id'],\n            'species_id': spid,\n            'songtype_id': soid,\n            't_min': tmin - tf.cast(cut_min, tf.float32)/tf.cast(SR, tf.float32),\n            'f_min': fmin,\n            't_max': tmax - tf.cast(cut_min, tf.float32)/tf.cast(SR, tf.float32),\n            'f_max': fmax,\n            'is_tp': tp\n        }\n        return _sample\n    \n    samples = tf.map_fn(_cut_audio, labels, dtype=parse_dtype)\n    return samples\n\nparsed_dataset = raw_dataset.map(_parse_function).unbatch()","metadata":{"id":"tIPNzl0-no6c","papermill":{"duration":1.014249,"end_time":"2021-01-09T21:36:00.26432","exception":false,"start_time":"2021-01-09T21:35:59.250071","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-27T20:24:47.618132Z","iopub.execute_input":"2021-05-27T20:24:47.618515Z","iopub.status.idle":"2021-05-27T20:24:48.285093Z","shell.execute_reply.started":"2021-05-27T20:24:47.618487Z","shell.execute_reply":"2021-05-27T20:24:48.28424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@tf.function\ndef _cut_wav(x):\n    # random cut in training\n    cut_min = tf.random.uniform([], maxval=(CUT-TIME)*SR, dtype=tf.int32)\n    cut_max = cut_min + TIME * SR\n    cutwave = tf.reshape(x['audio_wav'][cut_min:cut_max], [TIME*SR])\n    y = {}\n    y.update(x)\n    y['audio_wav'] = cutwave\n    y['t_min'] = tf.maximum(0.0, x['t_min'] - tf.cast(cut_min, tf.float32) / SR)\n    y['t_max'] = tf.maximum(0.0, x['t_max'] - tf.cast(cut_min, tf.float32) / SR)\n    return y\n    \n@tf.function\ndef _cut_wav_val(x):\n    # center crop in validation\n    cut_min = (CUT-TIME)*SR // 2\n    cut_max = cut_min + TIME * SR\n    cutwave = tf.reshape(x['audio_wav'][cut_min:cut_max], [TIME*SR])\n    y = {}\n    y.update(x)\n    y['audio_wav'] = cutwave\n    y['t_min'] = tf.maximum(0.0, x['t_min'] - tf.cast(cut_min, tf.float32) / SR)\n    y['t_max'] = tf.maximum(0.0, x['t_max'] - tf.cast(cut_min, tf.float32) / SR)\n    return y","metadata":{"id":"oVgyg_71no6c","papermill":{"duration":0.058759,"end_time":"2021-01-09T21:36:00.365537","exception":false,"start_time":"2021-01-09T21:36:00.306778","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-27T20:24:54.307219Z","iopub.execute_input":"2021-05-27T20:24:54.307578Z","iopub.status.idle":"2021-05-27T20:24:54.318177Z","shell.execute_reply.started":"2021-05-27T20:24:54.307545Z","shell.execute_reply":"2021-05-27T20:24:54.317122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@tf.function\ndef _filtTP(x):\n    return x['is_tp'] == 1","metadata":{"id":"aM_ujnk1no6e","papermill":{"duration":0.050968,"end_time":"2021-01-09T21:36:00.461049","exception":false,"start_time":"2021-01-09T21:36:00.410081","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-27T20:25:02.506805Z","iopub.execute_input":"2021-05-27T20:25:02.507183Z","iopub.status.idle":"2021-05-27T20:25:02.511617Z","shell.execute_reply.started":"2021-05-27T20:25:02.50713Z","shell.execute_reply":"2021-05-27T20:25:02.510719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_wav(sample, ax):\n    wav = sample[\"audio_wav\"].numpy()\n    rate = SR\n    ax.plot(np.arange(len(wav)) / rate, wav)\n    ax.set_title(\n        sample[\"recording_id\"].numpy().decode()\n        + (\"/%d\" % sample[\"species_id\"])\n        + (\"TP\" if sample[\"is_tp\"] else \"FP\"))\n\n    return Audio((wav * 2**15).astype(np.int16), rate=rate)\n\nfig, ax = plt.subplots(figsize=(15, 3))\nshow_wav(next(iter(parsed_dataset)), ax)","metadata":{"id":"akJEbFH1no6e","papermill":{"duration":4.619345,"end_time":"2021-01-09T21:36:05.122875","exception":false,"start_time":"2021-01-09T21:36:00.50353","status":"completed"},"tags":[],"outputId":"26d03357-5a7b-4013-a053-d4f988bf2df4","execution":{"iopub.status.busy":"2021-05-27T20:25:04.465831Z","iopub.execute_input":"2021-05-27T20:25:04.466387Z","iopub.status.idle":"2021-05-27T20:25:09.140258Z","shell.execute_reply.started":"2021-05-27T20:25:04.46634Z","shell.execute_reply":"2021-05-27T20:25:09.139206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating Mel-spectrograms","metadata":{"id":"F9NjSFe9no6e","papermill":{"duration":0.066656,"end_time":"2021-01-09T21:36:05.262746","exception":false,"start_time":"2021-01-09T21:36:05.19609","status":"completed"},"tags":[]}},{"cell_type":"code","source":"@tf.function\ndef _wav_to_spec(x):\n    mel_power = cfg['data_params']['mel_power']\n    \n    stfts = tf.signal.stft(x[\"audio_wav\"], frame_length=2048, frame_step=512, fft_length=2048)\n    spectrograms = tf.abs(stfts) ** mel_power\n\n    # Warp the linear scale spectrograms into the mel-scale.\n    num_spectrogram_bins = stfts.shape[-1]\n    lower_edge_hertz, upper_edge_hertz, num_mel_bins = FMIN, FMAX, N_MEL\n    \n    linear_to_mel_weight_matrix = tf.signal.linear_to_mel_weight_matrix(\n      num_mel_bins, num_spectrogram_bins, SR, lower_edge_hertz,\n      upper_edge_hertz)\n    mel_spectrograms = tf.tensordot(\n      spectrograms, linear_to_mel_weight_matrix, 1)\n    mel_spectrograms.set_shape(spectrograms.shape[:-1].concatenate(\n      linear_to_mel_weight_matrix.shape[-1:]))\n\n    # Compute a stabilized log to get log-magnitude mel-scale spectrograms.\n    log_mel_spectrograms = tf.math.log(mel_spectrograms + 1e-6)\n\n    y = {\n        'audio_spec': tf.transpose(log_mel_spectrograms), # (num_mel_bins, frames)\n    }\n    y.update(x)\n    return y\n\nspec_dataset = parsed_dataset.filter(_filtTP).map(_cut_wav).map(_wav_to_spec)","metadata":{"id":"AqvbHCAIno6e","papermill":{"duration":0.792062,"end_time":"2021-01-09T21:36:06.120507","exception":false,"start_time":"2021-01-09T21:36:05.328445","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-27T20:25:11.415016Z","iopub.execute_input":"2021-05-27T20:25:11.415392Z","iopub.status.idle":"2021-05-27T20:25:11.971478Z","shell.execute_reply.started":"2021-05-27T20:25:11.415363Z","shell.execute_reply":"2021-05-27T20:25:11.970364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Creating MFCCs\n@tf.function\ndef _wav_to_mfcc(x):\n    mel_power = cfg['data_params']['mel_power']\n    \n    stfts = tf.signal.stft(x[\"audio_wav\"], frame_length=2048, frame_step=512, fft_length=2048)\n    spectrograms = tf.abs(stfts) ** mel_power\n\n    # Warp the linear scale spectrograms into the mel-scale.\n    num_spectrogram_bins = stfts.shape[-1]\n    lower_edge_hertz, upper_edge_hertz, num_mel_bins = FMIN, FMAX, N_MEL\n    \n    linear_to_mel_weight_matrix = tf.signal.linear_to_mel_weight_matrix(\n      num_mel_bins, num_spectrogram_bins, SR, lower_edge_hertz,\n      upper_edge_hertz)\n    mel_spectrograms = tf.tensordot(\n      spectrograms, linear_to_mel_weight_matrix, 1)\n    mel_spectrograms.set_shape(spectrograms.shape[:-1].concatenate(\n      linear_to_mel_weight_matrix.shape[-1:]))\n\n    # Compute a stabilized log to get log-magnitude mel-scale spectrograms.\n    log_mel_spectrograms = tf.math.log(mel_spectrograms + 1e-6)\n    \n    # Compute MFCCs\n    mfccs = tf.signal.mfccs_from_log_mel_spectrograms(log_mel_spectrograms)\n\n    y = {\n        'audio_spec': tf.transpose(mfccs), # (num_mel_bins, frames)\n    }\n    y.update(x)\n    return y\n\nMFCC_dataset = parsed_dataset.filter(_filtTP).map(_cut_wav).map(_wav_to_mfcc)","metadata":{"execution":{"iopub.status.busy":"2021-05-27T20:25:15.806225Z","iopub.execute_input":"2021-05-27T20:25:15.806601Z","iopub.status.idle":"2021-05-27T20:25:16.408232Z","shell.execute_reply.started":"2021-05-27T20:25:15.806569Z","shell.execute_reply":"2021-05-27T20:25:16.407267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,5))\nfor i, s in enumerate(spec_dataset.take(1)):\n    plt.subplot(1,3,i+1)\n    plt.imshow(s['audio_spec'])\nplt.show()","metadata":{"id":"raVnTPizno6f","papermill":{"duration":1.07994,"end_time":"2021-01-09T21:36:07.267082","exception":false,"start_time":"2021-01-09T21:36:06.187142","status":"completed"},"tags":[],"outputId":"3aed166e-9635-4ed0-e587-b83d75aa3e21","execution":{"iopub.status.busy":"2021-05-27T20:26:41.785254Z","iopub.execute_input":"2021-05-27T20:26:41.785794Z","iopub.status.idle":"2021-05-27T20:26:42.291846Z","shell.execute_reply.started":"2021-05-27T20:26:41.78575Z","shell.execute_reply":"2021-05-27T20:26:42.290854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import librosa.display\nimport matplotlib.patches as patches\n\ndef show_spectrogram(sample, ax, showlabel=False):\n    S_dB = sample[\"audio_spec\"].numpy()\n    img = librosa.display.specshow(S_dB, x_axis='time',\n                             y_axis='mel', sr=SR,\n                             fmax=FMAX, fmin=FMIN, ax=ax, cmap='magma')\n    ax.set(title=f'Mel-frequency spectrogram of {sample[\"recording_id\"].numpy().decode()}')\n    sid, fmin, fmax, tmin, tmax, istp = (\n            sample[\"species_id\"], sample[\"f_min\"], sample[\"f_max\"], sample[\"t_min\"], sample[\"t_max\"], sample[\"is_tp\"])\n    ec = '#00ff00' if istp == 1 else '#0000ff'\n    ax.add_patch(\n        patches.Rectangle(xy=(tmin, fmin), width=tmax-tmin, height=fmax-fmin, ec=ec, fill=False)\n    )\n\n    if showlabel:\n        ax.text(tmin, fmax, \n        f\"{sid.numpy().item()} {'tp' if istp == 1 else 'fp'}\",\n        horizontalalignment='left', verticalalignment='bottom', color=ec, fontsize=16)","metadata":{"id":"2-xvLYjvno6f","papermill":{"duration":0.087578,"end_time":"2021-01-09T21:36:07.426129","exception":false,"start_time":"2021-01-09T21:36:07.338551","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-26T08:48:33.213921Z","iopub.status.idle":"2021-05-26T08:48:33.214318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(15,3))\nshow_spectrogram(next(iter(spec_dataset)), ax, showlabel=True)","metadata":{"id":"2_mRhMM_no6g","papermill":{"duration":0.913987,"end_time":"2021-01-09T21:36:08.409008","exception":false,"start_time":"2021-01-09T21:36:07.495021","status":"completed"},"tags":[],"outputId":"5937d326-ff80-4228-ab1e-220cfeadc43c","execution":{"iopub.status.busy":"2021-05-26T08:48:33.215187Z","iopub.status.idle":"2021-05-26T08:48:33.215577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# in validation, annotations will come to the center\nfig, ax = plt.subplots(figsize=(15,3))\nshow_spectrogram(next(iter(parsed_dataset.filter(_filtTP).map(_cut_wav_val).map(_wav_to_spec))), ax, showlabel=True)","metadata":{"id":"rlGItkKPno6g","papermill":{"duration":1.110584,"end_time":"2021-01-09T21:36:09.598219","exception":false,"start_time":"2021-01-09T21:36:08.487635","status":"completed"},"tags":[],"outputId":"883916b0-b827-41df-c0f0-37199141649b","execution":{"iopub.status.busy":"2021-05-26T08:48:33.21643Z","iopub.status.idle":"2021-05-26T08:48:33.216874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for sample in spec_dataset.take(5):\n    fig, ax = plt.subplots(figsize=(15,3))\n    show_spectrogram(sample, ax, showlabel=True)","metadata":{"id":"mLhy_N0Eno6g","papermill":{"duration":2.878388,"end_time":"2021-01-09T21:36:12.560883","exception":false,"start_time":"2021-01-09T21:36:09.682495","status":"completed"},"tags":[],"outputId":"4f826dfb-6822-4c56-d698-160ddcbc51b4","execution":{"iopub.status.busy":"2021-05-26T08:48:33.217856Z","iopub.status.idle":"2021-05-26T08:48:33.21825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating Labels","metadata":{"id":"EvMUWxsOno6g","papermill":{"duration":0.113472,"end_time":"2021-01-09T21:36:12.790501","exception":false,"start_time":"2021-01-09T21:36:12.677029","status":"completed"},"tags":[]}},{"cell_type":"code","source":"@tf.function\ndef _create_annot(x):\n    targ = tf.one_hot(x[\"species_id\"], CLASS_N, on_value=x[\"is_tp\"], off_value=0)\n    \n    return {\n        'input': x[\"audio_spec\"],\n        'target': tf.cast(targ, tf.float32)\n    }\n\nannot_dataset = spec_dataset.map(_create_annot) #MFCC_dataset/spec_dataset","metadata":{"id":"eFq1HMXzno6g","papermill":{"duration":0.236236,"end_time":"2021-01-09T21:36:13.142922","exception":false,"start_time":"2021-01-09T21:36:12.906686","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-26T08:48:33.219109Z","iopub.status.idle":"2021-05-26T08:48:33.219505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocessing and Data Augmentation\n* gaussian noise\n* random brightness\n* specaugment","metadata":{"id":"pqAFORpJno6g","papermill":{"duration":0.117771,"end_time":"2021-01-09T21:36:13.377783","exception":false,"start_time":"2021-01-09T21:36:13.260012","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#@tf.function\ndef _preprocess_img(x, training=False, test=False):\n    image = tf.expand_dims(x, axis=-1)\n    image = tf.image.resize(image, [HEIGHT, WIDTH])\n    image = tf.image.per_image_standardization(image)\n    \n    @tf.function\n    def _specaugment(image):\n        ERASE_TIME = 50\n        ERASE_MEL = 16\n        image = tf.squeeze(image, axis=2)\n        image = tfio.experimental.audio.time_mask(image, param=ERASE_TIME)\n        image = tfio.experimental.audio.freq_mask(image, param=ERASE_MEL)\n        image = tf.expand_dims(image, axis=2)\n        return image\n    \n    if training:\n        # gaussian\n        gau = tf.keras.layers.GaussianNoise(0.3)\n        image = tf.cond(tf.random.uniform([]) < 0.5, lambda: gau(image, training=True), lambda: image)\n        # brightness\n        image = tf.image.random_brightness(image, 0.2)\n        # random left right flip\n        image = tf.image.random_flip_left_right(image)\n        # specaugment\n        image = tf.cond(tf.random.uniform([]) < 0.5, lambda: _specaugment(image), lambda: image)\n        \n    if test:\n        # specaugment\n        image = tf.cond(tf.random.uniform([]) < 0.5, lambda: _specaugment(image), lambda: image)\n        \n    image = (image - tf.reduce_min(image)) / (tf.reduce_max(image) - tf.reduce_min(image)) * 255.0 # rescale to [0, 255]\n    image = tf.image.grayscale_to_rgb(image)\n    image = cfg['model_params']['arch_preprocess'](image)\n\n    return image\n\n@tf.function\ndef _preprocess(x):\n    image = _preprocess_img(x['input'], training=True, test=False)\n    return (image, x[\"target\"])\n\n@tf.function\ndef _preprocess_val(x):\n    image = _preprocess_img(x['input'], training=False, test=False)\n    return (image, x[\"target\"])\n\n@tf.function\ndef _preprocess_test(x):\n    image = _preprocess_img(x['audio_spec'], training=False, test=True)\n    return (image, x[\"recording_id\"])","metadata":{"id":"0eOtbTC6no6g","papermill":{"duration":0.144025,"end_time":"2021-01-09T21:36:13.637544","exception":false,"start_time":"2021-01-09T21:36:13.493519","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-26T08:48:33.220434Z","iopub.status.idle":"2021-05-26T08:48:33.220918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for inp, targ in annot_dataset.map(_preprocess).take(1):\n    plt.imshow(inp.numpy()[:,:,0])\n    t = targ.numpy()\n    if t.sum() == 0:\n        plt.title(f'FP')\n    else:\n        plt.title(f'{t.nonzero()[0]}')\n    plt.colorbar()\n    plt.show()","metadata":{"id":"sbmYhwW3no6h","papermill":{"duration":4.479235,"end_time":"2021-01-09T21:36:18.229277","exception":false,"start_time":"2021-01-09T21:36:13.750042","status":"completed"},"tags":[],"outputId":"7c4b68b9-f462-4241-f966-4037ae2f1f13","execution":{"iopub.status.busy":"2021-05-26T08:48:33.221964Z","iopub.status.idle":"2021-05-26T08:48:33.222372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{"id":"WBslx-DLno6h","papermill":{"duration":0.120958,"end_time":"2021-01-09T21:36:18.473258","exception":false,"start_time":"2021-01-09T21:36:18.3523","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from tensorflow.keras.layers import *\nfrom tensorflow.keras import losses, models, optimizers\nfrom tensorflow.keras.optimizers import Adam\ndef create_model():\n    #with strategy.scope():\n    #backbone = cfg['model_params']['arch'](include_top=False, weights='imagenet')\n    \n    def Classifier(shape_):\n\n        backbone = cfg['model_params']['arch']((shape_), include_top=False, weights='imagenet')\n    \n    \n        \n        def cbr(x, out_layer, kernel, stride, dilation):\n            x = Conv2D(out_layer, kernel_size=kernel, dilation_rate=dilation, strides=stride, padding=\"same\")(x)\n            x = BatchNormalization()(x)\n            x = Activation(\"relu\")(x)\n            return x\n\n        def wave_block(x, filters, kernel_size, n):\n            dilation_rates = [2**i for i in range(n)]\n            x = Conv2D(filters = filters,\n                       kernel_size = 1,\n                       padding = 'same')(x)\n            res_x = x\n            for dilation_rate in dilation_rates:\n                tanh_out = Conv2D(filters = filters,\n                                  kernel_size = kernel_size,\n                                  padding = 'same', \n                                  activation = 'tanh', \n                                  dilation_rate = dilation_rate)(x)\n                sigm_out = Conv2D(filters = filters,\n                                  kernel_size = kernel_size,\n                                  padding = 'same',\n                                  activation = 'sigmoid', \n                                  dilation_rate = dilation_rate)(x)\n                x = Multiply()([tanh_out, sigm_out])\n                x = Conv2D(filters = filters,\n                           kernel_size = 1,\n                           padding = 'same')(x)\n                res_x = Add()([res_x, x])\n            return res_x\n\n        \n        #out1\n        def wavenet(layer):\n          \n          x = cbr(layer, 192, 7, 1, 1)\n          x = BatchNormalization()(x)\n          x = wave_block(x, 192, 3, 1)\n          x = cbr(x, 96, 7, 1, 1)\n          x = BatchNormalization()(x)\n          x = wave_block(x, 96, 3, 1)\n          x = cbr(x, 48, 5, 1, 1)\n          x = BatchNormalization()(x)\n          x = wave_block(x, 48, 3, 1)  \n          return x\n\n        def wavenet1(layer):\n          \n          x = cbr(layer, 4, 7, 1, 1)\n          x = BatchNormalization()(x)\n          x = wave_block(x, 3, 3, 1)\n          x = cbr(x, 3, 7, 1, 1)\n          x = BatchNormalization()(x)\n          x = wave_block(x, 16, 3, 1)\n          x = cbr(x, 3, 5, 1, 1)\n          return x\n        #x = BatchNormalization()(x)\n        \n        x0 = backbone#model\n        print('1')\n        #backbone.summary()\n        x1 = tf.keras.layers.GlobalAveragePooling2D()(x0.layers[-1].output)  #-3,-7,-9,-15  for EF5    \n        #x2 = tf.keras.layers.GlobalAveragePooling2D()(x0.layers[-3].output) # 803,799,797,791 for EF7\n        x3 = tf.keras.layers.GlobalAveragePooling2D()(x0.layers[-7].output)\n        #x4 = tf.keras.layers.GlobalAveragePooling2D()(x0.layers[-12].output)\n        x5 = tf.keras.layers.GlobalAveragePooling2D()(x0.layers[-18].output)\n        print('2')\n        x1=wavenet(x0.layers[-1].output)\n        x3=wavenet(x0.layers[-7].output)\n        x5=wavenet(x0.layers[-18].output)\n\n        x1 = tf.keras.layers.GlobalAveragePooling2D()(x1)\n        x3 = tf.keras.layers.GlobalAveragePooling2D()(x3)\n        x5 = tf.keras.layers.GlobalAveragePooling2D()(x5)\n       \n        \n        \n        print('4')\n        #x =  tf.concat([x1,x2,x3,x4,x5],axis = 1)\n       \n        x =  tf.concat([x1,x3,x5],axis = 1)\n      \n        x = tf.keras.layers.Dropout(0.7)(x)\n        x = tf.keras.layers.Dense(192)(x)\n        #x =  tf.keras.layers.BatchNormalization()(x)          \n        x = tf.keras.layers.Dropout(0.4)(x)\n        #x =  margin([x , label])\n        \n        output = tf.keras.layers.Softmax(dtype='float32')(x)\n        output =tf.keras.layers.Dense(CLASS_N)(x)\n        print('5')\n        model = tf.keras.models.Model(inputs = x0.input, outputs = output)\n        #model.compile(optimizer=optimizer, loss=loss_fn, metrics=[LWLRAP(CLASS_N)])\n        \n\n        \n        \n        return model\n    return Classifier([HEIGHT,WIDTH,3])\n\n\nmodel = create_model()\nmodel.summary()\n","metadata":{"_kg_hide-output":true,"id":"QLtMr4yYno6h","papermill":{"duration":20.442591,"end_time":"2021-01-09T21:36:39.03362","exception":false,"start_time":"2021-01-09T21:36:18.591029","status":"completed"},"tags":[],"outputId":"97b5be21-65f2-417a-9759-059de510d598","execution":{"iopub.status.busy":"2021-05-26T08:48:33.22332Z","iopub.status.idle":"2021-05-26T08:48:33.223777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@tf.function\ndef _mixup(inp, targ):\n    indice = tf.range(len(inp))\n    indice = tf.random.shuffle(indice)\n    sinp = tf.gather(inp, indice, axis=0)\n    starg = tf.gather(targ, indice, axis=0)\n    \n    alpha = 0.2\n    t = tf.compat.v1.distributions.Beta(alpha, alpha).sample([len(inp)])\n    tx = tf.reshape(t, [-1, 1, 1, 1])\n    ty = tf.reshape(t, [-1, 1])\n    x = inp * tx + sinp * (1-tx)\n    y = targ * ty + starg * (1-ty)\n#     y = tf.minimum(targ + starg, 1.0) # for multi-label???\n    return x, y","metadata":{"id":"KQpVSGDdW90z","papermill":{"duration":0.177644,"end_time":"2021-01-09T21:36:39.726737","exception":false,"start_time":"2021-01-09T21:36:39.549093","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-26T08:48:33.224583Z","iopub.status.idle":"2021-05-26T08:48:33.225008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tfrecs = sorted(tf.io.gfile.glob(TRAIN_TFREC + '/*.tfrec'))\nparsed_trainval = (tf.data.TFRecordDataset(tfrecs, num_parallel_reads=AUTOTUNE)\n                    .map(_parse_function, num_parallel_calls=AUTOTUNE).unbatch()\n                    .filter(_filtTP).enumerate())","metadata":{"id":"unyKxlyLno6i","papermill":{"duration":0.384839,"end_time":"2021-01-09T21:36:40.271949","exception":false,"start_time":"2021-01-09T21:36:39.88711","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-26T08:48:33.225806Z","iopub.status.idle":"2021-05-26T08:48:33.226192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Stratified 5-Fold","metadata":{"id":"sFHUKIJ3no6j","papermill":{"duration":0.149661,"end_time":"2021-01-09T21:36:40.583152","exception":false,"start_time":"2021-01-09T21:36:40.433491","status":"completed"},"tags":[]}},{"cell_type":"code","source":"indices = []\nspid = []\nrecid = []\n\nfor i, sample in tqdm(parsed_trainval.prefetch(AUTOTUNE)):\n    indices.append(i.numpy())\n    spid.append(sample['species_id'].numpy())\n    recid.append(sample['recording_id'].numpy().decode())","metadata":{"id":"DTYFe70gno6j","papermill":{"duration":41.49283,"end_time":"2021-01-09T21:37:22.216352","exception":false,"start_time":"2021-01-09T21:36:40.723522","status":"completed"},"tags":[],"outputId":"275925f0-4d11-4361-cbf0-67e31377346e","execution":{"iopub.status.busy":"2021-05-26T08:48:33.227064Z","iopub.status.idle":"2021-05-26T08:48:33.227485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"table = pd.DataFrame({'indices': indices, 'species_id': spid, 'recording_id': recid})\ntable","metadata":{"id":"TxE06jxIno6j","papermill":{"duration":0.208764,"end_time":"2021-01-09T21:37:22.606773","exception":false,"start_time":"2021-01-09T21:37:22.398009","status":"completed"},"tags":[],"outputId":"480429ab-36f6-42bd-99bb-f042bf2010d5","execution":{"iopub.status.busy":"2021-05-26T08:48:33.228239Z","iopub.status.idle":"2021-05-26T08:48:33.228671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=5, random_state=SEED, shuffle=True)\nsplits = list(skf.split(table.index, table.species_id))\n\nplt.hist([table.loc[splits[0][0], 'species_id'], table.loc[splits[0][1], 'species_id']], bins=CLASS_N,stacked=True)\nplt.show()","metadata":{"id":"23tYZQu8no6k","papermill":{"duration":0.437587,"end_time":"2021-01-09T21:37:23.227143","exception":false,"start_time":"2021-01-09T21:37:22.789556","status":"completed"},"tags":[],"outputId":"4afb7007-3ccc-4161-f70a-fe6e15313e4a","execution":{"iopub.status.busy":"2021-05-26T08:48:33.229566Z","iopub.status.idle":"2021-05-26T08:48:33.230005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_idx_filter(indice):\n    @tf.function\n    def _filt(i, x):\n        return tf.reduce_any(indice == i)\n    return _filt\n\n@tf.function\ndef _remove_idx(i, x):\n    return x","metadata":{"id":"RF0AECQ-no6k","papermill":{"duration":0.194203,"end_time":"2021-01-09T21:37:23.607365","exception":false,"start_time":"2021-01-09T21:37:23.413162","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-26T08:48:33.230842Z","iopub.status.idle":"2021-05-26T08:48:33.231243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Other setup","metadata":{"id":"i7a8ks-Ano6k","papermill":{"duration":0.182871,"end_time":"2021-01-09T21:37:23.974522","exception":false,"start_time":"2021-01-09T21:37:23.791651","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def create_train_dataset(batchsize, train_idx):\n    global parsed_trainval\n    parsed_train = (parsed_trainval\n                    .filter(create_idx_filter(train_idx))\n                    .map(_remove_idx))\n    \n    dataset = (parsed_train.cache()\n        .shuffle(len(train_idx))\n        .repeat()\n        .map(_cut_wav, num_parallel_calls=AUTOTUNE)\n        #.map(_wav_to_mfcc, num_parallel_calls=AUTOTUNE)\n        .map(_wav_to_spec, num_parallel_calls=AUTOTUNE)\n        .map(_create_annot, num_parallel_calls=AUTOTUNE)\n        .map(_preprocess, num_parallel_calls=AUTOTUNE)#.map(_preprocess, num_parallel_calls=AUTOTUNE)\n        .batch(batchsize))\n\n    if cfg['model_params']['mixup']:\n        dataset = (dataset.map(_mixup, num_parallel_calls=AUTOTUNE)\n                    .prefetch(AUTOTUNE))\n    else:\n        dataset = dataset.prefetch(AUTOTUNE)\n    return dataset\n\ndef create_val_dataset(batchsize, val_idx):\n    global parsed_trainval\n    parsed_val = (parsed_trainval\n                  .filter(create_idx_filter(val_idx))\n                  .map(_remove_idx))\n\n    vdataset = (parsed_val\n        .map(_cut_wav_val, num_parallel_calls=AUTOTUNE)\n        #.map(_wav_to_mfcc, num_parallel_calls=AUTOTUNE)\n        .map(_wav_to_spec, num_parallel_calls=AUTOTUNE)\n        .map(_create_annot, num_parallel_calls=AUTOTUNE)\n        .map(_preprocess_val, num_parallel_calls=AUTOTUNE)\n        .batch(8*tpu_strategy.num_replicas_in_sync)\n        .cache())\n    return vdataset","metadata":{"id":"CFQRHfgbno6k","papermill":{"duration":0.224694,"end_time":"2021-01-09T21:37:24.381498","exception":false,"start_time":"2021-01-09T21:37:24.156804","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-26T08:48:33.232214Z","iopub.status.idle":"2021-05-26T08:48:33.232602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Metrics","metadata":{"id":"42cDABkLno6l","papermill":{"duration":0.185235,"end_time":"2021-01-09T21:37:24.816152","exception":false,"start_time":"2021-01-09T21:37:24.630917","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# from https://www.kaggle.com/carlthome/l-lrap-metric-for-tf-keras\n@tf.function\ndef _one_sample_positive_class_precisions(example):\n    y_true, y_pred = example\n\n    retrieved_classes = tf.argsort(y_pred, direction='DESCENDING')\n    class_rankings = tf.argsort(retrieved_classes)\n    retrieved_class_true = tf.gather(y_true, retrieved_classes)\n    retrieved_cumulative_hits = tf.math.cumsum(tf.cast(retrieved_class_true, tf.float32))\n\n    idx = tf.where(y_true)[:, 0]\n    i = tf.boolean_mask(class_rankings, y_true)\n    r = tf.gather(retrieved_cumulative_hits, i)\n    c = 1 + tf.cast(i, tf.float32)\n    precisions = r / c\n\n    dense = tf.scatter_nd(idx[:, None], precisions, [y_pred.shape[0]])\n    return dense\n\nclass LWLRAP(tf.keras.metrics.Metric):\n    def __init__(self, num_classes, name='lwlrap'):\n        super().__init__(name=name)\n\n        self._precisions = self.add_weight(\n            name='per_class_cumulative_precision',\n            shape=[num_classes],\n            initializer='zeros',\n        )\n\n        self._counts = self.add_weight(\n            name='per_class_cumulative_count',\n            shape=[num_classes],\n            initializer='zeros',\n        )\n\n    def update_state(self, y_true, y_pred, sample_weight=None):\n        precisions = tf.map_fn(\n            fn=_one_sample_positive_class_precisions,\n            elems=(y_true, y_pred),\n            dtype=(tf.float32),\n        )\n\n        increments = tf.cast(precisions > 0, tf.float32)\n        total_increments = tf.reduce_sum(increments, axis=0)\n        total_precisions = tf.reduce_sum(precisions, axis=0)\n\n        self._precisions.assign_add(total_precisions)\n        self._counts.assign_add(total_increments)        \n\n    def result(self):\n        per_class_lwlrap = self._precisions / tf.maximum(self._counts, 1.0)\n        per_class_weight = self._counts / tf.reduce_sum(self._counts)\n        overall_lwlrap = tf.reduce_sum(per_class_lwlrap * per_class_weight)\n        return overall_lwlrap\n\n    def reset_states(self):\n        self._precisions.assign(self._precisions * 0)\n        self._counts.assign(self._counts * 0)","metadata":{"id":"FQy9n6v5no6l","papermill":{"duration":0.208871,"end_time":"2021-01-09T21:37:25.215546","exception":false,"start_time":"2021-01-09T21:37:25.006675","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-26T08:48:33.233474Z","iopub.status.idle":"2021-05-26T08:48:33.233886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Testset and Inference function","metadata":{"id":"gxnPqFiGno6l","papermill":{"duration":0.182913,"end_time":"2021-01-09T21:37:25.585031","exception":false,"start_time":"2021-01-09T21:37:25.402118","status":"completed"},"tags":[]}},{"cell_type":"code","source":"\ndef _parse_function_test(example_proto):\n    sample = tf.io.parse_single_example(example_proto, feature_description)\n    wav, _ = tf.audio.decode_wav(sample['audio_wav'], desired_channels=1) # mono\n    \n    @tf.function\n    def _cut_audio(i):\n        _sample = {\n            'audio_wav': tf.reshape(wav[i*SR*TIME:(i+1)*SR*TIME], [SR*TIME]),\n            'recording_id': sample['recording_id']\n        }\n        return _sample\n\n    return tf.map_fn(_cut_audio, tf.range(60//TIME), dtype={\n        'audio_wav': tf.float32,\n        'recording_id': tf.string\n    })\n\ndef inference(model):\n    tdataset = (tf.data.TFRecordDataset(tf.io.gfile.glob(TEST_TFREC + '/*.tfrec'), num_parallel_reads=AUTOTUNE)\n        .map(_parse_function_test, num_parallel_calls=AUTOTUNE).unbatch()\n        #.map(_wav_to_mfcc, num_parallel_calls=AUTOTUNE)\n        .map(_wav_to_spec, num_parallel_calls=AUTOTUNE)\n        .map(_preprocess_test, num_parallel_calls=AUTOTUNE)\n        .batch(128*(60//TIME)).prefetch(AUTOTUNE))\n    \n    rec_ids = []\n    probs = []\n    for inp, rec_id in tqdm(tdataset):\n        with tpu_strategy.scope():\n            pred = model.predict_on_batch(tf.reshape(inp, [-1, HEIGHT, WIDTH, 3]))\n            prob = tf.sigmoid(pred)\n            prob = tf.reduce_max(tf.reshape(prob, [-1, 60//TIME, CLASS_N]), axis=1)\n\n        rec_id_stack = tf.reshape(rec_id, [-1, 60//TIME])\n        for rec in rec_id.numpy():\n            assert len(np.unique(rec)) == 1\n        rec_ids.append(rec_id_stack.numpy()[:,0])\n        probs.append(prob.numpy())\n        \n    crec_ids = np.concatenate(rec_ids)\n    cprobs = np.concatenate(probs)\n    \n    sub = pd.DataFrame({\n        'recording_id': list(map(lambda x: x.decode(), crec_ids.tolist())),\n        **{f's{i}': cprobs[:,i] for i in range(CLASS_N)}\n    })\n    sub = sub.sort_values('recording_id')\n\n    return sub","metadata":{"id":"8gfYp9llno6l","papermill":{"duration":0.212578,"end_time":"2021-01-09T21:37:25.982537","exception":false,"start_time":"2021-01-09T21:37:25.769959","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-26T08:48:33.23491Z","iopub.status.idle":"2021-05-26T08:48:33.235324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{"id":"LYFaqUomno6l","papermill":{"duration":0.189942,"end_time":"2021-01-09T21:37:26.363494","exception":false,"start_time":"2021-01-09T21:37:26.173552","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def plot_history(history, name):\n    plt.figure(figsize=(8,3))\n    plt.subplot(1,2,1)\n    plt.plot(history.history[\"loss\"])\n    plt.plot(history.history[\"val_loss\"])\n    plt.legend(['Train', 'Test'], loc='upper left')\n    plt.title(\"loss\")\n    # plt.yscale('log')\n\n    plt.subplot(1,2,2)\n    plt.plot(history.history[\"lwlrap\"])\n    plt.plot(history.history[\"val_lwlrap\"])\n    plt.legend(['Train', 'Test'], loc='upper left')\n    plt.title(\"metric\")\n\n    plt.savefig(name)\n    hist_df = pd.DataFrame(history.history) \n    hist_df.to_csv(models_path+f\"history_{name}.csv\", index=False)","metadata":{"id":"zsCQF4Bwno6l","papermill":{"duration":0.198689,"end_time":"2021-01-09T21:37:26.746839","exception":false,"start_time":"2021-01-09T21:37:26.54815","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-26T08:48:33.236126Z","iopub.status.idle":"2021-05-26T08:48:33.236521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_and_inference(splits, split_id):\n    print(split_id)\n\n    batchsize = cfg['model_params']['batchsize_per_tpu'] * tpu_strategy.num_replicas_in_sync\n    print(\"batchsize\", batchsize)\n    loss_fn = cfg['model_params']['loss']['fn'](from_logits=True, **cfg['model_params']['loss']['params'])\n\n    idx_train_tf = tf.constant(splits[split_id][0])\n    idx_val_tf = tf.constant(splits[split_id][1])\n\n    dataset = create_train_dataset(batchsize, idx_train_tf)\n    vdataset = create_val_dataset(batchsize, idx_val_tf)\n    \n    optimizer = cfg['model_params']['optim']['fn'](**cfg['model_params']['optim']['params'])\n    \n    with tpu_strategy.scope():\n        model = create_model()\n        model.compile(optimizer=optimizer, loss=loss_fn, metrics=[LWLRAP(CLASS_N), 'accuracy'])\n    \n    if split_id not in (10,10):#For convenience: If your Colab shuts down for some reason, you can always download your already trained models from Google Drive\n\n        history = model.fit(dataset,\n                          steps_per_epoch=cfg['model_params']['iteration_per_epoch'],\n                          epochs=cfg['model_params']['epoch'],\n                          validation_data=vdataset,\n                          callbacks=[\n                              tf.keras.callbacks.ReduceLROnPlateau(\n                                  'val_lwlrap', patience=10\n                              ),  \n                              tf.keras.callbacks.ModelCheckpoint(\n                                  filepath=models_path+'model_best_%d.h5' % split_id,\n                                  save_weights_only=True,\n                                  monitor='val_lwlrap',\n                                  mode='max',\n                                  save_best_only=True),\n                          ]\n                          )\n        plot_history(history, 'history_%d.png' % split_id)\n        best_score = max(history.history['val_lwlrap'])\n        best_accuracy = max(history.history['accuracy'])\n        print(\"Best Accuracy: \",best_accuracy)\n        print (\"Best Score: \",best_score)\n    ### inference ###\n    \n    model.load_weights(models_path+'model_best_%d.h5' % split_id)\n    sub=inference(model)\n    del model\n    gc.collect()\n    return sub,best_score","metadata":{"id":"yxIyk8Y9no6l","papermill":{"duration":0.206752,"end_time":"2021-01-09T21:37:27.140072","exception":false,"start_time":"2021-01-09T21:37:26.93332","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-26T08:48:33.237472Z","iopub.status.idle":"2021-05-26T08:48:33.237898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train and inference\n#sub, _ = train_and_inference(splits, 0)\n\n# N-fold ensemble\nprint(SEED)\ntrain_n=0\ndf = pd.DataFrame(columns=[\"train_n\",'split_id','best_score','CSV','SEED'])\nfor split_id in range(len(splits)):\n    sub, best_score=train_and_inference(splits, split_id)\n    sub.set_index('recording_id').to_csv(models_path+f\"submission_train_n_{train_n}_split_id_{split_id}.csv\", index=False)\n    df = df.append({'train_n': train_n,'split_id': split_id,'best_score': best_score,'CSV': f\"submission_train_n_{train_n}_split_id_{split_id}.csv\",'SEED':SEED}, ignore_index=True)\ndf.to_csv(models_path+f\"train_n_{train_n}.csv\", index=False)\n\n","metadata":{"id":"UZyI6qPpno6m","papermill":{"duration":1562.915291,"end_time":"2021-01-09T22:03:30.239268","exception":false,"start_time":"2021-01-09T21:37:27.323977","status":"completed"},"tags":[],"outputId":"2aa80d7e-a8ef-4be5-b486-4bd595d72742","execution":{"iopub.status.busy":"2021-05-26T08:48:33.238984Z","iopub.status.idle":"2021-05-26T08:48:33.239384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}