{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Tensorflow Tpu birdcall training","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## About\n\nIn this notebook, I try residual network as per this tutorial https://keras.io/examples/audio/speaker_recognition_using_cnn/\n\n\nI want to thanks following kernel authors for such a nice public kernel\nhttps://www.kaggle.com/ttahara/training-birdsong-baseline-resnest50-fast\nhttps://www.kaggle.com/hidehisaarai1213/inference-pytorch-birdcall-resnet-baseline\n","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Prepare","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### import libraries","execution_count":null},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"!pip install efficientnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow.keras.backend as K\n# import efficientnet.tfkeras as efn\nfrom kaggle_datasets import KaggleDatasets\n\nimport os\nimport shutil\nimport numpy as np\n\nimport tensorflow as tf\nfrom tensorflow import keras\n\nfrom pathlib import Path\nfrom IPython.display import display, Audio\n\nseed_value = 42\ntf.random.set_seed(seed_value)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DEVICE = \"tpu\"\nif DEVICE == \"tpu\":\n    print(\"Connecting for tpu\")\n    try :\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print(\"Running on tpu\", tpu.master())\n    except ValueError:\n        print(\"could not connect TPU\")\n        tpu = None\n    if tpu:\n        try:\n            print(\"Initialize TPU...\")\n            tf.config.experimental_connect_to_cluster(tpu)\n            tf.tpu.experimental.initialize_tpu_system(tpu)\n            strategy = tf.distribute.experimental.TPUStrategy(tpu)\n            print('TPU initialized')\n        except _:\n            print('Failed to initialize TPU!')\n    else:\n        DEVICE = 'GPU'\nif DEVICE != 'tpu':\n    print('Using default strategy for CPU and single GPU')\n    strategy = tf.distribute.get_strategy()\n\nif DEVICE == 'GPU':\n    print('Num GPUs Available: ',\n          len(tf.config.experimental.list_physical_devices('GPU')))\n\nprint('REPLICAS: ', strategy.num_replicas_in_sync)\nAUTO = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport gc\nimport time\nimport shutil\nimport random\nimport warnings\nimport typing as tp\nfrom pathlib import Path\nfrom contextlib import contextmanager\n\nimport yaml\nfrom joblib import delayed, Parallel\n\nimport cv2\nimport librosa\nimport audioread\nimport soundfile as sf\n\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn.metrics import f1_score\nfrom sklearn.model_selection import StratifiedKFold\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.data as data\n\n# import pytorch_pfn_extras as ppe\n# from pytorch_pfn_extras.training import extensions as ppe_extensions\n\npd.options.display.max_rows = 500\npd.options.display.max_columns = 500","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DEVICE","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### read data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"ROOT = Path.cwd().parent\nINPUT_ROOT = ROOT / \"input\"\nRAW_DATA = INPUT_ROOT / \"birdsong-recognition\"\nTRAIN_AUDIO_DIR = RAW_DATA / \"train_audio\"\n\n\nTRAIN_RESAMPLED_AUDIO_DIRS = [\n  INPUT_ROOT / \"birdsong-resampled-train-audio-{:0>2}\".format(i)  for i in range(5)\n]\ntrain = pd.read_csv(TRAIN_RESAMPLED_AUDIO_DIRS[0] / \"train_mod.csv\")\nTEST_AUDIO_DIR = RAW_DATA / \"test_audio\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train = pd.read_csv(RAW_DATA / \"train.csv\")\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### settings","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"settings_str = \"\"\"\nglobals:\n  seed: 1213\n  device: cuda\n  num_epochs: 5\n  output_dir: /kaggle/training_output/\n  use_fold: 0\n  target_sr: 32000\n\ndataset:\n  name: SpectrogramDataset\n  params:\n    img_size: 224\n    melspectrogram_parameters:\n      n_mels: 128\n      fmin: 20\n      fmax: 16000\n    \nsplit:\n  name: StratifiedKFold\n  params:\n    n_splits: 5\n    random_state: 42\n    shuffle: True\n\nloader:\n  train:\n    batch_size: 50\n    shuffle: True\n    num_workers: 2\n    pin_memory: True\n    drop_last: True\n  val:\n    batch_size: 50\n    shuffle: False\n    num_workers: 2\n    pin_memory: True\n    drop_last: False\n\nmodel:\n  name: resnest50_fast_1s1x64d\n  params:\n    pretrained: True\n    n_classes: 264\n\nloss:\n  name: BCEWithLogitsLoss\n  params: {}\n\noptimizer:\n  name: Adam\n  params:\n    lr: 0.001\n\nscheduler:\n  name: CosineAnnealingLR\n  params:\n    T_max: 10\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"settings = yaml.safe_load(settings_str)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not torch.cuda.is_available():\n    settings[\"globals\"][\"device\"] = \"cpu\"","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":false,"trusted":true},"cell_type":"code","source":"for k, v in settings.items():\n    print(\"[{}]\".format(k))\n    print(v)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Definition","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Dataset\n* forked from: https://github.com/koukyo1994/kaggle-birdcall-resnet-baseline-training/blob/master/src/dataset.py\n* modified partialy\n","execution_count":null},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"BIRD_CODE = {\n    'aldfly': 0, 'ameavo': 1, 'amebit': 2, 'amecro': 3, 'amegfi': 4,\n    'amekes': 5, 'amepip': 6, 'amered': 7, 'amerob': 8, 'amewig': 9,\n    'amewoo': 10, 'amtspa': 11, 'annhum': 12, 'astfly': 13, 'baisan': 14,\n    'baleag': 15, 'balori': 16, 'banswa': 17, 'barswa': 18, 'bawwar': 19,\n    'belkin1': 20, 'belspa2': 21, 'bewwre': 22, 'bkbcuc': 23, 'bkbmag1': 24,\n    'bkbwar': 25, 'bkcchi': 26, 'bkchum': 27, 'bkhgro': 28, 'bkpwar': 29,\n    'bktspa': 30, 'blkpho': 31, 'blugrb1': 32, 'blujay': 33, 'bnhcow': 34,\n    'boboli': 35, 'bongul': 36, 'brdowl': 37, 'brebla': 38, 'brespa': 39,\n    'brncre': 40, 'brnthr': 41, 'brthum': 42, 'brwhaw': 43, 'btbwar': 44,\n    'btnwar': 45, 'btywar': 46, 'buffle': 47, 'buggna': 48, 'buhvir': 49,\n    'bulori': 50, 'bushti': 51, 'buwtea': 52, 'buwwar': 53, 'cacwre': 54,\n    'calgul': 55, 'calqua': 56, 'camwar': 57, 'cangoo': 58, 'canwar': 59,\n    'canwre': 60, 'carwre': 61, 'casfin': 62, 'caster1': 63, 'casvir': 64,\n    'cedwax': 65, 'chispa': 66, 'chiswi': 67, 'chswar': 68, 'chukar': 69,\n    'clanut': 70, 'cliswa': 71, 'comgol': 72, 'comgra': 73, 'comloo': 74,\n    'commer': 75, 'comnig': 76, 'comrav': 77, 'comred': 78, 'comter': 79,\n    'comyel': 80, 'coohaw': 81, 'coshum': 82, 'cowscj1': 83, 'daejun': 84,\n    'doccor': 85, 'dowwoo': 86, 'dusfly': 87, 'eargre': 88, 'easblu': 89,\n    'easkin': 90, 'easmea': 91, 'easpho': 92, 'eastow': 93, 'eawpew': 94,\n    'eucdov': 95, 'eursta': 96, 'evegro': 97, 'fiespa': 98, 'fiscro': 99,\n    'foxspa': 100, 'gadwal': 101, 'gcrfin': 102, 'gnttow': 103, 'gnwtea': 104,\n    'gockin': 105, 'gocspa': 106, 'goleag': 107, 'grbher3': 108, 'grcfly': 109,\n    'greegr': 110, 'greroa': 111, 'greyel': 112, 'grhowl': 113, 'grnher': 114,\n    'grtgra': 115, 'grycat': 116, 'gryfly': 117, 'haiwoo': 118, 'hamfly': 119,\n    'hergul': 120, 'herthr': 121, 'hoomer': 122, 'hoowar': 123, 'horgre': 124,\n    'horlar': 125, 'houfin': 126, 'houspa': 127, 'houwre': 128, 'indbun': 129,\n    'juntit1': 130, 'killde': 131, 'labwoo': 132, 'larspa': 133, 'lazbun': 134,\n    'leabit': 135, 'leafly': 136, 'leasan': 137, 'lecthr': 138, 'lesgol': 139,\n    'lesnig': 140, 'lesyel': 141, 'lewwoo': 142, 'linspa': 143, 'lobcur': 144,\n    'lobdow': 145, 'logshr': 146, 'lotduc': 147, 'louwat': 148, 'macwar': 149,\n    'magwar': 150, 'mallar3': 151, 'marwre': 152, 'merlin': 153, 'moublu': 154,\n    'mouchi': 155, 'moudov': 156, 'norcar': 157, 'norfli': 158, 'norhar2': 159,\n    'normoc': 160, 'norpar': 161, 'norpin': 162, 'norsho': 163, 'norwat': 164,\n    'nrwswa': 165, 'nutwoo': 166, 'olsfly': 167, 'orcwar': 168, 'osprey': 169,\n    'ovenbi1': 170, 'palwar': 171, 'pasfly': 172, 'pecsan': 173, 'perfal': 174,\n    'phaino': 175, 'pibgre': 176, 'pilwoo': 177, 'pingro': 178, 'pinjay': 179,\n    'pinsis': 180, 'pinwar': 181, 'plsvir': 182, 'prawar': 183, 'purfin': 184,\n    'pygnut': 185, 'rebmer': 186, 'rebnut': 187, 'rebsap': 188, 'rebwoo': 189,\n    'redcro': 190, 'redhea': 191, 'reevir1': 192, 'renpha': 193, 'reshaw': 194,\n    'rethaw': 195, 'rewbla': 196, 'ribgul': 197, 'rinduc': 198, 'robgro': 199,\n    'rocpig': 200, 'rocwre': 201, 'rthhum': 202, 'ruckin': 203, 'rudduc': 204,\n    'rufgro': 205, 'rufhum': 206, 'rusbla': 207, 'sagspa1': 208, 'sagthr': 209,\n    'savspa': 210, 'saypho': 211, 'scatan': 212, 'scoori': 213, 'semplo': 214,\n    'semsan': 215, 'sheowl': 216, 'shshaw': 217, 'snobun': 218, 'snogoo': 219,\n    'solsan': 220, 'sonspa': 221, 'sora': 222, 'sposan': 223, 'spotow': 224,\n    'stejay': 225, 'swahaw': 226, 'swaspa': 227, 'swathr': 228, 'treswa': 229,\n    'truswa': 230, 'tuftit': 231, 'tunswa': 232, 'veery': 233, 'vesspa': 234,\n    'vigswa': 235, 'warvir': 236, 'wesblu': 237, 'wesgre': 238, 'weskin': 239,\n    'wesmea': 240, 'wessan': 241, 'westan': 242, 'wewpew': 243, 'whbnut': 244,\n    'whcspa': 245, 'whfibi': 246, 'whtspa': 247, 'whtswi': 248, 'wilfly': 249,\n    'wilsni1': 250, 'wiltur': 251, 'winwre3': 252, 'wlswar': 253, 'wooduc': 254,\n    'wooscj2': 255, 'woothr': 256, 'y00475': 257, 'yebfly': 258, 'yebsap': 259,\n    'yehbla': 260, 'yelwar': 261, 'yerwar': 262, 'yetvir': 263\n}\n\nINV_BIRD_CODE = {v: k for k, v in BIRD_CODE.items()}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tmp_list = []\nfor audio_d in TRAIN_RESAMPLED_AUDIO_DIRS:\n    if not audio_d.exists():\n        continue\n    for ebird_d in audio_d.iterdir():\n        if ebird_d.is_file():\n            continue\n        for wav_f in ebird_d.iterdir():\n            tmp_list.append([ebird_d.name, wav_f.name, wav_f.as_posix()])\n            \ntrain_wav_path_exist = pd.DataFrame(\n    tmp_list, columns=[\"ebird_code\", \"resampled_filename\", \"file_path\"])\n\ndel tmp_list\n\ntrain_all = pd.merge(\n    train, train_wav_path_exist, on=[\"ebird_code\", \"resampled_filename\"], how=\"inner\")\n\nprint(train.shape)\nprint(train_wav_path_exist.shape)\nprint(train_all.shape)\n\ndel train\ndel train_wav_path_exist\n\nskf = StratifiedKFold(**settings[\"split\"][\"params\"])\n\ntrain_all[\"fold\"] = -1\nfor fold_id, (train_index, val_index) in enumerate(skf.split(train_all, train_all[\"ebird_code\"])):\n    train_all.iloc[val_index, -1] = fold_id\n    \n\nif DEVICE == 'tpu':\n    from kaggle_datasets import KaggleDatasets\n    GCS_PATH = KaggleDatasets().get_gcs_path('birdsong-resampled-train-audio-00')\n    GCS_PATH1 = KaggleDatasets().get_gcs_path('birdsong-resampled-train-audio-01')\n    GCS_PATH2 = KaggleDatasets().get_gcs_path('birdsong-resampled-train-audio-02')\n    GCS_PATH3 = KaggleDatasets().get_gcs_path('birdsong-resampled-train-audio-03')\n    GCS_PATH4 = KaggleDatasets().get_gcs_path('birdsong-resampled-train-audio-04')\n    train_all['file_path'] = train_all['file_path'].str.replace('/kaggle/input/birdsong-resampled-train-audio-00', GCS_PATH)\n    train_all['file_path'] = train_all['file_path'].str.replace('/kaggle/input/birdsong-resampled-train-audio-01', GCS_PATH1)\n    train_all['file_path'] = train_all['file_path'].str.replace('/kaggle/input/birdsong-resampled-train-audio-02', GCS_PATH2)\n    train_all['file_path'] = train_all['file_path'].str.replace('/kaggle/input/birdsong-resampled-train-audio-03', GCS_PATH3)\n    train_all['file_path'] = train_all['file_path'].str.replace('/kaggle/input/birdsong-resampled-train-audio-04', GCS_PATH4)\n    \nuse_fold = settings[\"globals\"][\"use_fold\"]\ntrain_file_list = train_all.query(\"fold != @use_fold\")[[\"file_path\", \"ebird_code\"]]\nval_file_list = train_all.query(\"fold == @use_fold\")[[\"file_path\", \"ebird_code\"]]\n\nprint(\"[fold {}] train: {}, val: {}\".format(use_fold, len(train_file_list), len(val_file_list)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_file_list.ebird_code = train_file_list.ebird_code.map(BIRD_CODE)\nval_file_list.ebird_code = val_file_list.ebird_code.map(BIRD_CODE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !pip install -q tensorflow-io \n# # > /dev/null","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import tensorflow_io as tfio","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def paths_and_labels_to_dataset(audio_paths, labels):\n    \"\"\"Constructs a dataset of audios and labels.\"\"\"\n    path_ds = tf.data.Dataset.from_tensor_slices(audio_paths)\n    audio_ds = path_ds.map(lambda x: path_to_audio(x))\n    label_ds = tf.data.Dataset.from_tensor_slices(labels)\n    return tf.data.Dataset.zip((audio_ds, label_ds))\n\n\ndef path_to_audio(path):\n    \"\"\"Reads and decodes an audio file.\"\"\"\n    audio = tf.io.read_file(path)\n    audio, _ = tf.audio.decode_wav(audio, 1\n                                   , settings[\"globals\"]['target_sr'] * 5\n                                  )    \n    \n    return audio\n\ndef audio_to_fft(audio, y):\n    # Since tf.signal.fft applies FFT on the innermost dimension,\n    # we need to squeeze the dimensions and then expand them again\n    # after FFT\n    audio = tf.squeeze(audio, axis=-1)\n    fft = tf.signal.fft(\n        tf.cast(tf.complex(real=audio, imag=tf.zeros_like(audio)), tf.complex64)\n    )\n    fft = tf.expand_dims(fft, axis=-1)\n\n    # Return the absolute value of the first half of the FFT\n    # which represents the positive frequencies\n    return tf.math.abs(fft[:, : (audio.shape[1] // 2), :]), y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SHUFFLE_SEED = 42\n\n# Create 2 datasets, one for training and the other for validation\ntrain_ds = paths_and_labels_to_dataset(train_file_list['file_path'].values.tolist(), train_file_list['ebird_code'].values.tolist())\ntrain_ds = train_ds.shuffle(buffer_size=settings['loader']['train']['batch_size'] * 8, seed=SHUFFLE_SEED).batch(\n    settings['loader']['train']['batch_size']\n)\n\nvalid_ds = paths_and_labels_to_dataset(val_file_list['file_path'].values.tolist(), val_file_list['ebird_code'].values.tolist() )\nvalid_ds = valid_ds.shuffle(buffer_size=settings['loader']['val']['batch_size'] * 8, seed=SHUFFLE_SEED).batch(32)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Transform audio wave to the frequency domain using `audio_to_fft`\ntrain_ds = train_ds.map(\n    audio_to_fft, num_parallel_calls=tf.data.experimental.AUTOTUNE\n)\ntrain_ds = train_ds.prefetch(tf.data.experimental.AUTOTUNE)\n\nvalid_ds = valid_ds.map(\n    audio_to_fft, num_parallel_calls=tf.data.experimental.AUTOTUNE\n)\nvalid_ds = valid_ds.prefetch(tf.data.experimental.AUTOTUNE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def residual_block(x, filters, conv_num=3, activation=\"relu\"):\n    # Shortcut\n    s = keras.layers.Conv1D(filters, 1, padding=\"same\")(x)\n    s = keras.layers.BatchNormalization()(s)\n    for i in range(conv_num - 1):\n        x = keras.layers.Conv1D(filters, 3, padding=\"same\")(x)\n        x = keras.layers.BatchNormalization()(x)\n        x = keras.layers.Activation(activation)(x)\n    x = keras.layers.Conv1D(filters, 3, padding=\"same\")(x)\n    x = keras.layers.Add()([x, s])\n    x = keras.layers.Activation(activation)(x)\n    return keras.layers.MaxPool1D(pool_size=2, strides=2)(x)\n\n\ndef build_model(input_shape, num_classes):\n    inputs = keras.layers.Input(shape=input_shape, name=\"input\")\n\n    x = residual_block(inputs, 16, 2)\n    x = residual_block(x, 32, 2)\n    x = residual_block(x, 64, 3)\n    x = residual_block(x, 128, 3)\n    x = residual_block(x, 128, 3)\n\n    x = keras.layers.AveragePooling1D(pool_size=3, strides=3)(x)\n    x = keras.layers.Flatten()(x)\n    x = keras.layers.Dense(256, activation=\"relu\")(x)\n    x = keras.layers.Dense(128, activation=\"relu\")(x)\n\n    outputs = keras.layers.Dense(num_classes, activation=\"softmax\", name=\"output\")(x)\n\n    return keras.models.Model(inputs=inputs, outputs=outputs)\ndef compile_new_model():    \n    with strategy.scope():\n        model = build_model((settings['globals']['target_sr']*5 // 2, 1), settings['model']['params']['n_classes'])\n\n        model.summary()\n\n        # Compile the model using Adam's default learning rate\n        model.compile(\n            optimizer=\"Adam\", loss=\"sparse_categorical_crossentropy\", metrics=[\"accuracy\"]\n        )\n        return model\n# Add callbacks:\n# 'EarlyStopping' to stop training when the model is not enhancing anymore\n# 'ModelCheckPoint' to always keep the model that has the best val_accuracy\nmodel_save_filename = \"model.h5\"\n\nearlystopping_cb = keras.callbacks.EarlyStopping(patience=10, restore_best_weights=True)\nmdlcheckpoint_cb = keras.callbacks.ModelCheckpoint(\n    model_save_filename, monitor=\"val_accuracy\", save_best_only=True\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = compile_new_model()\nhistory = model.fit(\n    train_ds,\n    epochs=settings['globals']['num_epochs'],\n    validation_data=valid_ds,\n    callbacks=[earlystopping_cb, mdlcheckpoint_cb],\n)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}