{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Tensorflow birdcall vggish","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## About\n\nIn this notebook, I try to implement vggish model of tensorflow. \n\n\nI want to thanks following kernel authors for such a nice public kernel\nhttps://www.kaggle.com/ttahara/training-birdsong-baseline-resnest50-fast\nhttps://www.kaggle.com/hidehisaarai1213/inference-pytorch-birdcall-resnet-baseline\n","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Prepare","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### import libraries","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install numpy resampy tensorflow tf_slim six soundfile","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow.keras.backend as K\n# import efficientnet.tfkeras as efn\nfrom kaggle_datasets import KaggleDatasets\n\nimport os\nimport shutil\nimport numpy as np\n\nimport tensorflow as tf\nfrom tensorflow import keras\n\nfrom pathlib import Path\nfrom IPython.display import display, Audio\n\nseed_value = 42\ntf.random.set_seed(seed_value)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DEVICE = \"tpu\"\nif DEVICE == \"tpu\":\n    print(\"Connecting for tpu\")\n    try :\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print(\"Running on tpu\", tpu.master())\n    except ValueError:\n        print(\"could not connect TPU\")\n        tpu = None\n    if tpu:\n        try:\n            print(\"Initialize TPU...\")\n            tf.config.experimental_connect_to_cluster(tpu)\n            tf.tpu.experimental.initialize_tpu_system(tpu)\n            strategy = tf.distribute.experimental.TPUStrategy(tpu)\n            print('TPU initialized')\n        except _:\n            print('Failed to initialize TPU!')\n    else:\n        DEVICE = 'GPU'\nif DEVICE != 'tpu':\n    print('Using default strategy for CPU and single GPU')\n    strategy = tf.distribute.get_strategy()\n\nif DEVICE == 'GPU':\n    print('Num GPUs Available: ',\n          len(tf.config.experimental.list_physical_devices('GPU')))\n\nprint('REPLICAS: ', strategy.num_replicas_in_sync)\nAUTO = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport gc\nimport time\nimport shutil\nimport random\nimport warnings\nimport typing as tp\nfrom pathlib import Path\nfrom contextlib import contextmanager\n\nimport yaml\nfrom joblib import delayed, Parallel\n\nimport cv2\nimport librosa\nimport audioread\nimport soundfile as sf\n\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn.metrics import f1_score\nfrom sklearn.model_selection import StratifiedKFold\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.data as data\n\n# import pytorch_pfn_extras as ppe\n# from pytorch_pfn_extras.training import extensions as ppe_extensions\n\npd.options.display.max_rows = 500\npd.options.display.max_columns = 500","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUM_FRAMES = 96  # Frames in input mel-spectrogram patch.\nNUM_BANDS = 64  # Frequency bands in input mel-spectrogram patch.\nEMBEDDING_SIZE = 128  # Size of embedding layer.\n\n# Hyperparameters used in feature and example generation.\nSAMPLE_RATE = 16000\nSTFT_WINDOW_LENGTH_SECONDS = 0.025\nSTFT_HOP_LENGTH_SECONDS = 0.010\nNUM_MEL_BINS = NUM_BANDS\nMEL_MIN_HZ = 125\nMEL_MAX_HZ = 7500\nLOG_OFFSET = 0.01  # Offset used for stabilized log of input mel-spectrogram.\nEXAMPLE_WINDOW_SECONDS = 0.96  # Each example contains 96 10ms frames\nEXAMPLE_HOP_SECONDS = 0.96     # with zero overlap.\n\n# Parameters used for embedding postprocessing.\nPCA_EIGEN_VECTORS_NAME = 'pca_eigen_vectors'\nPCA_MEANS_NAME = 'pca_means'\nQUANTIZE_MIN_VAL = -2.0\nQUANTIZE_MAX_VAL = +2.0\n\n# Hyperparameters used in training.\nINIT_STDDEV = 0.01  # Standard deviation used to initialize weights.\nLEARNING_RATE = 1e-4  # Learning rate for the Adam optimizer.\nADAM_EPSILON = 1e-8  # Epsilon for the Adam optimizer.\n\n# Names of ops, tensors, and features.\nINPUT_OP_NAME = 'vggish/input_features'\nINPUT_TENSOR_NAME = INPUT_OP_NAME + ':0'\nOUTPUT_OP_NAME = 'vggish/embedding'\nOUTPUT_TENSOR_NAME = OUTPUT_OP_NAME + ':0'\nAUDIO_EMBEDDING_FEATURE_NAME = 'audio_embedding'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\n\n\ndef frame(data, window_length, hop_length):\n  \"\"\"Convert array into a sequence of successive possibly overlapping frames.\n  An n-dimensional array of shape (num_samples, ...) is converted into an\n  (n+1)-D array of shape (num_frames, window_length, ...), where each frame\n  starts hop_length points after the preceding one.\n  This is accomplished using stride_tricks, so the original data is not\n  copied.  However, there is no zero-padding, so any incomplete frames at the\n  end are not included.\n  Args:\n    data: np.array of dimension N >= 1.\n    window_length: Number of samples in each frame.\n    hop_length: Advance (in samples) between each window.\n  Returns:\n    (N+1)-D np.array with as many rows as there are complete frames that can be\n    extracted.\n  \"\"\"\n  num_samples = data.shape[0]\n  num_frames = 1 + int(np.floor((num_samples - window_length) / hop_length))\n  shape = (num_frames, window_length) + data.shape[1:]\n  strides = (data.strides[0] * hop_length,) + data.strides\n  return np.lib.stride_tricks.as_strided(data, shape=shape, strides=strides)\n\n\ndef periodic_hann(window_length):\n  \"\"\"Calculate a \"periodic\" Hann window.\n  The classic Hann window is defined as a raised cosine that starts and\n  ends on zero, and where every value appears twice, except the middle\n  point for an odd-length window.  Matlab calls this a \"symmetric\" window\n  and np.hanning() returns it.  However, for Fourier analysis, this\n  actually represents just over one cycle of a period N-1 cosine, and\n  thus is not compactly expressed on a length-N Fourier basis.  Instead,\n  it's better to use a raised cosine that ends just before the final\n  zero value - i.e. a complete cycle of a period-N cosine.  Matlab\n  calls this a \"periodic\" window. This routine calculates it.\n  Args:\n    window_length: The number of points in the returned window.\n  Returns:\n    A 1D np.array containing the periodic hann window.\n  \"\"\"\n  return 0.5 - (0.5 * np.cos(2 * np.pi / window_length *\n                             np.arange(window_length)))\n\n\ndef stft_magnitude(signal, fft_length,\n                   hop_length=None,\n                   window_length=None):\n  \"\"\"Calculate the short-time Fourier transform magnitude.\n  Args:\n    signal: 1D np.array of the input time-domain signal.\n    fft_length: Size of the FFT to apply.\n    hop_length: Advance (in samples) between each frame passed to FFT.\n    window_length: Length of each block of samples to pass to FFT.\n  Returns:\n    2D np.array where each row contains the magnitudes of the fft_length/2+1\n    unique values of the FFT for the corresponding frame of input samples.\n  \"\"\"\n  frames = frame(signal, window_length, hop_length)\n  # Apply frame window to each frame. We use a periodic Hann (cosine of period\n  # window_length) instead of the symmetric Hann of np.hanning (period\n  # window_length-1).\n  window = periodic_hann(window_length)\n  windowed_frames = frames * window\n  return np.abs(np.fft.rfft(windowed_frames, int(fft_length)))\n\n\n# Mel spectrum constants and functions.\n_MEL_BREAK_FREQUENCY_HERTZ = 700.0\n_MEL_HIGH_FREQUENCY_Q = 1127.0\n\n\ndef hertz_to_mel(frequencies_hertz):\n  \"\"\"Convert frequencies to mel scale using HTK formula.\n  Args:\n    frequencies_hertz: Scalar or np.array of frequencies in hertz.\n  Returns:\n    Object of same size as frequencies_hertz containing corresponding values\n    on the mel scale.\n  \"\"\"\n  return _MEL_HIGH_FREQUENCY_Q * np.log(\n      1.0 + (frequencies_hertz / _MEL_BREAK_FREQUENCY_HERTZ))\n\n\ndef spectrogram_to_mel_matrix(num_mel_bins=20,\n                              num_spectrogram_bins=129,\n                              audio_sample_rate=8000,\n                              lower_edge_hertz=125.0,\n                              upper_edge_hertz=3800.0):\n  \"\"\"Return a matrix that can post-multiply spectrogram rows to make mel.\n  Returns a np.array matrix A that can be used to post-multiply a matrix S of\n  spectrogram values (STFT magnitudes) arranged as frames x bins to generate a\n  \"mel spectrogram\" M of frames x num_mel_bins.  M = S A.\n  The classic HTK algorithm exploits the complementarity of adjacent mel bands\n  to multiply each FFT bin by only one mel weight, then add it, with positive\n  and negative signs, to the two adjacent mel bands to which that bin\n  contributes.  Here, by expressing this operation as a matrix multiply, we go\n  from num_fft multiplies per frame (plus around 2*num_fft adds) to around\n  num_fft^2 multiplies and adds.  However, because these are all presumably\n  accomplished in a single call to np.dot(), it's not clear which approach is\n  faster in Python.  The matrix multiplication has the attraction of being more\n  general and flexible, and much easier to read.\n  Args:\n    num_mel_bins: How many bands in the resulting mel spectrum.  This is\n      the number of columns in the output matrix.\n    num_spectrogram_bins: How many bins there are in the source spectrogram\n      data, which is understood to be fft_size/2 + 1, i.e. the spectrogram\n      only contains the nonredundant FFT bins.\n    audio_sample_rate: Samples per second of the audio at the input to the\n      spectrogram. We need this to figure out the actual frequencies for\n      each spectrogram bin, which dictates how they are mapped into mel.\n    lower_edge_hertz: Lower bound on the frequencies to be included in the mel\n      spectrum.  This corresponds to the lower edge of the lowest triangular\n      band.\n    upper_edge_hertz: The desired top edge of the highest frequency band.\n  Returns:\n    An np.array with shape (num_spectrogram_bins, num_mel_bins).\n  Raises:\n    ValueError: if frequency edges are incorrectly ordered or out of range.\n  \"\"\"\n  nyquist_hertz = audio_sample_rate / 2.\n  if lower_edge_hertz < 0.0:\n    raise ValueError(\"lower_edge_hertz %.1f must be >= 0\" % lower_edge_hertz)\n  if lower_edge_hertz >= upper_edge_hertz:\n    raise ValueError(\"lower_edge_hertz %.1f >= upper_edge_hertz %.1f\" %\n                     (lower_edge_hertz, upper_edge_hertz))\n  if upper_edge_hertz > nyquist_hertz:\n    raise ValueError(\"upper_edge_hertz %.1f is greater than Nyquist %.1f\" %\n                     (upper_edge_hertz, nyquist_hertz))\n  spectrogram_bins_hertz = np.linspace(0.0, nyquist_hertz, num_spectrogram_bins)\n  spectrogram_bins_mel = hertz_to_mel(spectrogram_bins_hertz)\n  # The i'th mel band (starting from i=1) has center frequency\n  # band_edges_mel[i], lower edge band_edges_mel[i-1], and higher edge\n  # band_edges_mel[i+1].  Thus, we need num_mel_bins + 2 values in\n  # the band_edges_mel arrays.\n  band_edges_mel = np.linspace(hertz_to_mel(lower_edge_hertz),\n                               hertz_to_mel(upper_edge_hertz), num_mel_bins + 2)\n  # Matrix to post-multiply feature arrays whose rows are num_spectrogram_bins\n  # of spectrogram values.\n  mel_weights_matrix = np.empty((num_spectrogram_bins, num_mel_bins))\n  for i in range(num_mel_bins):\n    lower_edge_mel, center_mel, upper_edge_mel = band_edges_mel[i:i + 3]\n    # Calculate lower and upper slopes for every spectrogram bin.\n    # Line segments are linear in the *mel* domain, not hertz.\n    lower_slope = ((spectrogram_bins_mel - lower_edge_mel) /\n                   (center_mel - lower_edge_mel))\n    upper_slope = ((upper_edge_mel - spectrogram_bins_mel) /\n                   (upper_edge_mel - center_mel))\n    # .. then intersect them with each other and zero.\n    mel_weights_matrix[:, i] = np.maximum(0.0, np.minimum(lower_slope,\n                                                          upper_slope))\n  # HTK excludes the spectrogram DC bin; make sure it always gets a zero\n  # coefficient.\n  mel_weights_matrix[0, :] = 0.0\n  return mel_weights_matrix\n\n\ndef log_mel_spectrogram(data,\n                        audio_sample_rate=8000,\n                        log_offset=0.0,\n                        window_length_secs=0.025,\n                        hop_length_secs=0.010,\n                        **kwargs):\n  \"\"\"Convert waveform to a log magnitude mel-frequency spectrogram.\n  Args:\n    data: 1D np.array of waveform data.\n    audio_sample_rate: The sampling rate of data.\n    log_offset: Add this to values when taking log to avoid -Infs.\n    window_length_secs: Duration of each window to analyze.\n    hop_length_secs: Advance between successive analysis windows.\n    **kwargs: Additional arguments to pass to spectrogram_to_mel_matrix.\n  Returns:\n    2D np.array of (num_frames, num_mel_bins) consisting of log mel filterbank\n    magnitudes for successive frames.\n  \"\"\"\n  window_length_samples = int(round(audio_sample_rate * window_length_secs))\n  hop_length_samples = int(round(audio_sample_rate * hop_length_secs))\n  fft_length = 2 ** int(np.ceil(np.log(window_length_samples) / np.log(2.0)))\n  spectrogram = stft_magnitude(\n      data,\n      fft_length=fft_length,\n      hop_length=hop_length_samples,\n      window_length=window_length_samples)\n  mel_spectrogram = np.dot(spectrogram, spectrogram_to_mel_matrix(\n      num_spectrogram_bins=spectrogram.shape[1],\n      audio_sample_rate=audio_sample_rate, **kwargs))\n  return np.log(mel_spectrogram + log_offset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _remove_silent_frames(audio, top_db):\n    indices = librosa.effects.split(audio, top_db=top_db)\n\n    trimmed_audio = []\n    for index in indices:\n        trimmed_audio.append(torch.tensor(audio[index[0]: index[1]]))\n\n    return np.concatenate(trimmed_audio, axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport resampy\nimport librosa\n\nPERIOD = 10\n\ntry:\n  import soundfile as sf\n\n  def wav_read(wav_file):\n    wav_data, sr = sf.read(wav_file)\n    return wav_data, sr\n\nexcept ImportError:\n\n  def wav_read(wav_file):\n    raise NotImplementedError('WAV file reading requires soundfile package.')\n\ndef _remove_silent_frames(audio, top_db):\n    indices = librosa.effects.split(audio, top_db=top_db)\n\n    trimmed_audio = []\n    for index in indices:\n        trimmed_audio.append(torch.tensor(audio[index[0]: index[1]]))\n\n    return np.concatenate(trimmed_audio, axis=0)\n\ndef waveform_to_examples(data, sample_rate):\n  \"\"\"Converts audio waveform into an array of examples for VGGish.\n  Args:\n    data: np.array of either one dimension (mono) or two dimensions\n      (multi-channel, with the outer dimension representing channels).\n      Each sample is generally expected to lie in the range [-1.0, +1.0],\n      although this is not required.\n    sample_rate: Sample rate of data.\n  Returns:\n    3-D np.array of shape [num_examples, num_frames, num_bands] which represents\n    a sequence of examples, each of which contains a patch of log mel\n    spectrogram, covering num_frames frames of audio and num_bands mel frequency\n    bands, where the frame length is vggish_params.STFT_HOP_LENGTH_SECONDS.\n  \"\"\"\n  # Convert to mono.\n  if len(data.shape) > 1:\n    data = np.mean(data, axis=1)\n  # Resample to the rate assumed by VGGish.\n  if sample_rate != SAMPLE_RATE:\n    data = resampy.resample(data, sample_rate, SAMPLE_RATE)\n\n  # Compute log mel spectrogram features.\n  log_mel = log_mel_spectrogram(\n      data,\n      audio_sample_rate= SAMPLE_RATE,\n      log_offset= LOG_OFFSET,\n      window_length_secs= STFT_WINDOW_LENGTH_SECONDS,\n      hop_length_secs= STFT_HOP_LENGTH_SECONDS,\n      num_mel_bins= NUM_MEL_BINS,\n      lower_edge_hertz= MEL_MIN_HZ,\n      upper_edge_hertz= MEL_MAX_HZ)\n\n  # Frame features into examples.\n  features_sample_rate = 1.0 /  STFT_HOP_LENGTH_SECONDS\n  example_window_length = int(round(\n      EXAMPLE_WINDOW_SECONDS * features_sample_rate))\n  example_hop_length = int(round(\n      EXAMPLE_HOP_SECONDS * features_sample_rate))\n  log_mel_examples = frame(\n      log_mel,\n      window_length=example_window_length,\n      hop_length=example_hop_length)\n  return log_mel_examples\n\n\ndef wavfile_to_examples(wav_file):\n  \"\"\"Convenience wrapper around waveform_to_examples() for a common WAV format.\n  Args:\n    wav_file: String path to a file, or a file-like object. The file\n    is assumed to contain WAV audio data with signed 16-bit PCM samples.\n  Returns:\n    See waveform_to_examples.\n  \"\"\"\n  wav_data, sr = wav_read(wav_file)\n  wav_data = _remove_silent_frames(wav_data, top_db=40)\n  effective_length = sr * PERIOD  \n  len_y = len(wav_data)\n\n  if len_y < effective_length:\n                new_y = np.zeros(effective_length, dtype=wav_data.dtype)\n                start = np.random.randint(effective_length - len_y)\n                new_y[start:start + len_y] = wav_data\n                wav_data = new_y.astype(np.float32)\n  elif len_y > effective_length:\n                start = np.random.randint(len_y - effective_length)\n                wav_data = wav_data[start:start + effective_length].astype(np.float32)\n  else:\n                wav_data = wav_data.astype(np.float32)\n                \n  wav_data = wav_data * 32768\n  wav_data = wav_data.astype(np.int16)\n    \n  assert wav_data.dtype == np.int16, 'Bad sample type: %r' % wav_data.dtype\n  samples = wav_data / 32768.0  # Convert to [-1.0, +1.0]\n  return waveform_to_examples(samples, sr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow.compat.v1 as tf\nimport tf_slim as slim\n\n\n\n\ndef define_vggish_slim(features_tensor=None, training=False):\n  \"\"\"Defines the VGGish TensorFlow model.\n  All ops are created in the current default graph, under the scope 'vggish/'.\n  The input is either a tensor passed in via the optional 'features_tensor'\n  argument or a placeholder created below named 'vggish/input_features'. The\n  input is expected to have dtype float32 and shape [batch_size, num_frames,\n  num_bands] where batch_size is variable and num_frames and num_bands are\n  constants, and [num_frames, num_bands] represents a log-mel-scale spectrogram\n  patch covering num_bands frequency bands and num_frames time frames (where\n  each frame step is usually 10ms). This is produced by computing the stabilized\n  log(mel-spectrogram + params.LOG_OFFSET).  The output is a tensor named\n  'vggish/embedding' which produces the pre-activation values of a 128-D\n  embedding layer, which is usually the penultimate layer when used as part of a\n  full model with a final classifier layer.\n  Args:\n    features_tensor: If not None, the tensor containing the input features.\n      If None, a placeholder input is created.\n    training: If true, all parameters are marked trainable.\n  Returns:\n    The op 'vggish/embeddings'.\n  \"\"\"\n  # Defaults:\n  # - All weights are initialized to N(0, INIT_STDDEV).\n  # - All biases are initialized to 0.\n  # - All activations are ReLU.\n  # - All convolutions are 3x3 with stride 1 and SAME padding.\n  # - All max-pools are 2x2 with stride 2 and SAME padding.\n  with slim.arg_scope([slim.conv2d, slim.fully_connected],\n                      weights_initializer=tf.truncated_normal_initializer(\n                          stddev=INIT_STDDEV),\n                      biases_initializer=tf.zeros_initializer(),\n                      activation_fn=tf.nn.relu,\n                      trainable=training), \\\n       slim.arg_scope([slim.conv2d],\n                      kernel_size=[3, 3], stride=1, padding='SAME'), \\\n       slim.arg_scope([slim.max_pool2d],\n                      kernel_size=[2, 2], stride=2, padding='SAME'), \\\n       tf.variable_scope('vggish'):\n    # Input: a batch of 2-D log-mel-spectrogram patches.\n    if features_tensor is None:\n      features_tensor = tf.placeholder(\n          tf.float32, shape=(None, NUM_FRAMES, NUM_BANDS),\n          name='input_features')\n    # Reshape to 4-D so that we can convolve a batch with conv2d().\n    net = tf.reshape(features_tensor,\n                     [-1, NUM_FRAMES, NUM_BANDS, 1])\n\n    # The VGG stack of alternating convolutions and max-pools.\n    net = slim.conv2d(net, 64, scope='conv1')\n    net = slim.max_pool2d(net, scope='pool1')\n    net = slim.conv2d(net, 128, scope='conv2')\n    net = slim.max_pool2d(net, scope='pool2')\n    net = slim.repeat(net, 2, slim.conv2d, 256, scope='conv3')\n    net = slim.max_pool2d(net, scope='pool3')\n    net = slim.repeat(net, 2, slim.conv2d, 512, scope='conv4')\n    net = slim.max_pool2d(net, scope='pool4')\n\n    # Flatten before entering fully-connected layers\n    net = slim.flatten(net)\n    net = slim.repeat(net, 2, slim.fully_connected, 4096, scope='fc1')\n    # The embedding layer.\n    net = slim.fully_connected(net, EMBEDDING_SIZE, scope='fc2',\n                               activation_fn=None)\n    return tf.identity(net, name='embedding')\n\n\ndef load_vggish_slim_checkpoint(session, checkpoint_path):\n  \"\"\"Loads a pre-trained VGGish-compatible checkpoint.\n  This function can be used as an initialization function (referred to as\n  init_fn in TensorFlow documentation) which is called in a Session after\n  initializating all variables. When used as an init_fn, this will load\n  a pre-trained checkpoint that is compatible with the VGGish model\n  definition. Only variables defined by VGGish will be loaded.\n  Args:\n    session: an active TensorFlow session.\n    checkpoint_path: path to a file containing a checkpoint that is\n      compatible with the VGGish model definition.\n  \"\"\"\n  # Get the list of names of all VGGish variables that exist in\n  # the checkpoint (i.e., all inference-mode VGGish variables).\n  with tf.Graph().as_default():\n    define_vggish_slim(training=False)\n    vggish_var_names = [v.name for v in tf.global_variables()]\n\n  # Get the list of all currently existing variables that match\n  # the list of variable names we just computed.\n  vggish_vars = [v for v in tf.global_variables() if v.name in vggish_var_names]\n\n  # Use a Saver to restore just the variables selected above.\n  saver = tf.train.Saver(vggish_vars, name='vggish_load_pretrained',\n                         write_version=1)\n  saver.restore(session, checkpoint_path)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### read data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"ROOT = Path.cwd().parent\nINPUT_ROOT = ROOT / \"input\"\nRAW_DATA = INPUT_ROOT / \"birdsong-recognition\"\nTRAIN_AUDIO_DIR = RAW_DATA / \"train_audio\"\n\n\nTRAIN_RESAMPLED_AUDIO_DIRS = [\n  INPUT_ROOT / \"birdsong-resampled-train-audio-{:0>2}\".format(i)  for i in range(5)\n]\ntrain = pd.read_csv(TRAIN_RESAMPLED_AUDIO_DIRS[0] / \"train_mod.csv\")\nTEST_AUDIO_DIR = RAW_DATA / \"test_audio\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### settings","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"settings_str = \"\"\"\nglobals:\n  seed: 1213\n  device: cuda\n  num_epochs: 5\n  output_dir: /kaggle/training_output/\n  use_fold: 0\n  target_sr: 32000\n\ndataset:\n  name: SpectrogramDataset\n  params:\n    img_size: 224\n    melspectrogram_parameters:\n      n_mels: 128\n      fmin: 20\n      fmax: 16000\n    \nsplit:\n  name: StratifiedKFold\n  params:\n    n_splits: 5\n    random_state: 42\n    shuffle: True\n\nloader:\n  train:\n    batch_size: 32\n    shuffle: True\n    num_workers: 2\n    pin_memory: True\n    drop_last: True\n  val:\n    batch_size: 32\n    shuffle: False\n    num_workers: 2\n    pin_memory: True\n    drop_last: False\n\nmodel:\n  name: resnest50_fast_1s1x64d\n  params:\n    pretrained: True\n    n_classes: 264\n\nloss:\n  name: BCEWithLogitsLoss\n  params: {}\n\noptimizer:\n  name: Adam\n  params:\n    lr: 0.001\n\nscheduler:\n  name: CosineAnnealingLR\n  params:\n    T_max: 10\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"settings = yaml.safe_load(settings_str)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not torch.cuda.is_available():\n    settings[\"globals\"][\"device\"] = \"cpu\"","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":false,"trusted":true},"cell_type":"code","source":"for k, v in settings.items():\n    print(\"[{}]\".format(k))\n    print(v)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Definition","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Dataset\n* forked from: https://github.com/koukyo1994/kaggle-birdcall-resnet-baseline-training/blob/master/src/dataset.py\n* modified partialy\n","execution_count":null},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"BIRD_CODE = {\n    'aldfly': 0, 'ameavo': 1, 'amebit': 2, 'amecro': 3, 'amegfi': 4,\n    'amekes': 5, 'amepip': 6, 'amered': 7, 'amerob': 8, 'amewig': 9,\n    'amewoo': 10, 'amtspa': 11, 'annhum': 12, 'astfly': 13, 'baisan': 14,\n    'baleag': 15, 'balori': 16, 'banswa': 17, 'barswa': 18, 'bawwar': 19,\n    'belkin1': 20, 'belspa2': 21, 'bewwre': 22, 'bkbcuc': 23, 'bkbmag1': 24,\n    'bkbwar': 25, 'bkcchi': 26, 'bkchum': 27, 'bkhgro': 28, 'bkpwar': 29,\n    'bktspa': 30, 'blkpho': 31, 'blugrb1': 32, 'blujay': 33, 'bnhcow': 34,\n    'boboli': 35, 'bongul': 36, 'brdowl': 37, 'brebla': 38, 'brespa': 39,\n    'brncre': 40, 'brnthr': 41, 'brthum': 42, 'brwhaw': 43, 'btbwar': 44,\n    'btnwar': 45, 'btywar': 46, 'buffle': 47, 'buggna': 48, 'buhvir': 49,\n    'bulori': 50, 'bushti': 51, 'buwtea': 52, 'buwwar': 53, 'cacwre': 54,\n    'calgul': 55, 'calqua': 56, 'camwar': 57, 'cangoo': 58, 'canwar': 59,\n    'canwre': 60, 'carwre': 61, 'casfin': 62, 'caster1': 63, 'casvir': 64,\n    'cedwax': 65, 'chispa': 66, 'chiswi': 67, 'chswar': 68, 'chukar': 69,\n    'clanut': 70, 'cliswa': 71, 'comgol': 72, 'comgra': 73, 'comloo': 74,\n    'commer': 75, 'comnig': 76, 'comrav': 77, 'comred': 78, 'comter': 79,\n    'comyel': 80, 'coohaw': 81, 'coshum': 82, 'cowscj1': 83, 'daejun': 84,\n    'doccor': 85, 'dowwoo': 86, 'dusfly': 87, 'eargre': 88, 'easblu': 89,\n    'easkin': 90, 'easmea': 91, 'easpho': 92, 'eastow': 93, 'eawpew': 94,\n    'eucdov': 95, 'eursta': 96, 'evegro': 97, 'fiespa': 98, 'fiscro': 99,\n    'foxspa': 100, 'gadwal': 101, 'gcrfin': 102, 'gnttow': 103, 'gnwtea': 104,\n    'gockin': 105, 'gocspa': 106, 'goleag': 107, 'grbher3': 108, 'grcfly': 109,\n    'greegr': 110, 'greroa': 111, 'greyel': 112, 'grhowl': 113, 'grnher': 114,\n    'grtgra': 115, 'grycat': 116, 'gryfly': 117, 'haiwoo': 118, 'hamfly': 119,\n    'hergul': 120, 'herthr': 121, 'hoomer': 122, 'hoowar': 123, 'horgre': 124,\n    'horlar': 125, 'houfin': 126, 'houspa': 127, 'houwre': 128, 'indbun': 129,\n    'juntit1': 130, 'killde': 131, 'labwoo': 132, 'larspa': 133, 'lazbun': 134,\n    'leabit': 135, 'leafly': 136, 'leasan': 137, 'lecthr': 138, 'lesgol': 139,\n    'lesnig': 140, 'lesyel': 141, 'lewwoo': 142, 'linspa': 143, 'lobcur': 144,\n    'lobdow': 145, 'logshr': 146, 'lotduc': 147, 'louwat': 148, 'macwar': 149,\n    'magwar': 150, 'mallar3': 151, 'marwre': 152, 'merlin': 153, 'moublu': 154,\n    'mouchi': 155, 'moudov': 156, 'norcar': 157, 'norfli': 158, 'norhar2': 159,\n    'normoc': 160, 'norpar': 161, 'norpin': 162, 'norsho': 163, 'norwat': 164,\n    'nrwswa': 165, 'nutwoo': 166, 'olsfly': 167, 'orcwar': 168, 'osprey': 169,\n    'ovenbi1': 170, 'palwar': 171, 'pasfly': 172, 'pecsan': 173, 'perfal': 174,\n    'phaino': 175, 'pibgre': 176, 'pilwoo': 177, 'pingro': 178, 'pinjay': 179,\n    'pinsis': 180, 'pinwar': 181, 'plsvir': 182, 'prawar': 183, 'purfin': 184,\n    'pygnut': 185, 'rebmer': 186, 'rebnut': 187, 'rebsap': 188, 'rebwoo': 189,\n    'redcro': 190, 'redhea': 191, 'reevir1': 192, 'renpha': 193, 'reshaw': 194,\n    'rethaw': 195, 'rewbla': 196, 'ribgul': 197, 'rinduc': 198, 'robgro': 199,\n    'rocpig': 200, 'rocwre': 201, 'rthhum': 202, 'ruckin': 203, 'rudduc': 204,\n    'rufgro': 205, 'rufhum': 206, 'rusbla': 207, 'sagspa1': 208, 'sagthr': 209,\n    'savspa': 210, 'saypho': 211, 'scatan': 212, 'scoori': 213, 'semplo': 214,\n    'semsan': 215, 'sheowl': 216, 'shshaw': 217, 'snobun': 218, 'snogoo': 219,\n    'solsan': 220, 'sonspa': 221, 'sora': 222, 'sposan': 223, 'spotow': 224,\n    'stejay': 225, 'swahaw': 226, 'swaspa': 227, 'swathr': 228, 'treswa': 229,\n    'truswa': 230, 'tuftit': 231, 'tunswa': 232, 'veery': 233, 'vesspa': 234,\n    'vigswa': 235, 'warvir': 236, 'wesblu': 237, 'wesgre': 238, 'weskin': 239,\n    'wesmea': 240, 'wessan': 241, 'westan': 242, 'wewpew': 243, 'whbnut': 244,\n    'whcspa': 245, 'whfibi': 246, 'whtspa': 247, 'whtswi': 248, 'wilfly': 249,\n    'wilsni1': 250, 'wiltur': 251, 'winwre3': 252, 'wlswar': 253, 'wooduc': 254,\n    'wooscj2': 255, 'woothr': 256, 'y00475': 257, 'yebfly': 258, 'yebsap': 259,\n    'yehbla': 260, 'yelwar': 261, 'yerwar': 262, 'yetvir': 263\n}\n\nINV_BIRD_CODE = {v: k for k, v in BIRD_CODE.items()}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tmp_list = []\nfor audio_d in TRAIN_RESAMPLED_AUDIO_DIRS:\n    if not audio_d.exists():\n        continue\n    for ebird_d in audio_d.iterdir():\n        if ebird_d.is_file():\n            continue\n        for wav_f in ebird_d.iterdir():\n            tmp_list.append([ebird_d.name, wav_f.name, wav_f.as_posix()])\n            \ntrain_wav_path_exist = pd.DataFrame(\n    tmp_list, columns=[\"ebird_code\", \"resampled_filename\", \"file_path\"])\n\ndel tmp_list\n\ntrain_all = pd.merge(\n    train, train_wav_path_exist, on=[\"ebird_code\", \"resampled_filename\"], how=\"inner\")\n\nprint(train.shape)\nprint(train_wav_path_exist.shape)\nprint(train_all.shape)\n\ndel train\ndel train_wav_path_exist\n\nskf = StratifiedKFold(**settings[\"split\"][\"params\"])\n\ntrain_all[\"fold\"] = -1\nfor fold_id, (train_index, val_index) in enumerate(skf.split(train_all, train_all[\"ebird_code\"])):\n    train_all.iloc[val_index, -1] = fold_id\n    \n\nif DEVICE == 'tpu':\n    from kaggle_datasets import KaggleDatasets\n    GCS_PATH = KaggleDatasets().get_gcs_path('birdsong-resampled-train-audio-00')\n    GCS_PATH1 = KaggleDatasets().get_gcs_path('birdsong-resampled-train-audio-01')\n    GCS_PATH2 = KaggleDatasets().get_gcs_path('birdsong-resampled-train-audio-02')\n    GCS_PATH3 = KaggleDatasets().get_gcs_path('birdsong-resampled-train-audio-03')\n    GCS_PATH4 = KaggleDatasets().get_gcs_path('birdsong-resampled-train-audio-04')\n    train_all['file_path'] = train_all['file_path'].str.replace('/kaggle/input/birdsong-resampled-train-audio-00', GCS_PATH)\n    train_all['file_path'] = train_all['file_path'].str.replace('/kaggle/input/birdsong-resampled-train-audio-01', GCS_PATH1)\n    train_all['file_path'] = train_all['file_path'].str.replace('/kaggle/input/birdsong-resampled-train-audio-02', GCS_PATH2)\n    train_all['file_path'] = train_all['file_path'].str.replace('/kaggle/input/birdsong-resampled-train-audio-03', GCS_PATH3)\n    train_all['file_path'] = train_all['file_path'].str.replace('/kaggle/input/birdsong-resampled-train-audio-04', GCS_PATH4)\n    \nuse_fold = settings[\"globals\"][\"use_fold\"]\ntrain_file_list = train_all.query(\"fold != @use_fold\")[[\"file_path\", \"ebird_code\"]]\nval_file_list = train_all.query(\"fold == @use_fold\")[[\"file_path\", \"ebird_code\"]]\n\nprint(\"[fold {}] train: {}, val: {}\".format(use_fold, len(train_file_list), len(val_file_list)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_file_list.ebird_code = train_file_list.ebird_code.map(BIRD_CODE)\nval_file_list.ebird_code = val_file_list.ebird_code.map(BIRD_CODE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from random import shuffle\n# def _get_examples_batch():\n#   \"\"\"Returns a shuffled batch of examples of all audio classes.\n#   Note that this is just a toy function because this is a simple demo intended\n#   to illustrate how the training code might work.\n#   Returns:\n#     a tuple (features, labels) where features is a NumPy array of shape\n#     [batch_size, num_frames, num_bands] where the batch_size is variable and\n#     each row is a log mel spectrogram patch of shape [num_frames, num_bands]\n#     suitable for feeding VGGish, while labels is a NumPy array of shape\n#     [batch_size, num_classes] where each row is a multi-hot label vector that\n#     provides the labels for corresponding rows in features.\n#   \"\"\"\n#   # Make a waveform for each class.\n#   num_seconds = 5\n#   sr = 44100  # Sampling rate.\n#   t = np.arange(0, num_seconds, 1 / sr)  # Time axis\n#   # Random sine wave.\n#   freq = np.random.uniform(100, 1000)\n#   sine = np.sin(2 * np.pi * freq * t)\n#   # Random constant signal.\n#   magnitude = np.random.uniform(-1, 1)\n#   const = magnitude * t\n#   # White noise.\n#   noise = np.random.normal(-1, 1, size=t.shape)\n\n#   # Make examples of each signal and corresponding labels.\n#   # Sine is class index 0, Const class index 1, Noise class index 2.\n#   sine_examples = waveform_to_examples(sine, sr)\n#   sine_labels = np.array([[1, 0, 0]] * sine_examples.shape[0])\n#   const_examples = waveform_to_examples(const, sr)\n#   const_labels = np.array([[0, 1, 0]] * const_examples.shape[0])\n#   noise_examples = waveform_to_examples(noise, sr)\n#   noise_labels = np.array([[0, 0, 1]] * noise_examples.shape[0])\n\n#   # Shuffle (example, label) pairs across all classes.\n#   all_examples = np.concatenate((sine_examples, const_examples, noise_examples))\n#   all_labels = np.concatenate((sine_labels, const_labels, noise_labels))\n#   labeled_examples = list(zip(all_examples, all_labels))\n#   shuffle(labeled_examples)\n\n#   # Separate and return the features and labels.\n#   features = [example for (example, _) in labeled_examples]\n#   labels = [label for (_, label) in labeled_examples]\n#   return (features, labels)\n\n# all_examples , labeled_examples = _get_examples_batch()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DataGenerator(keras.utils.Sequence):\n    'Generates data for Keras'\n    def __init__(self, list_IDs, labels, batch_size=32, \n                 dim=[(settings[\"globals\"]['target_sr'] * 5)//2], n_channels=1,\n                 n_classes=264, shuffle=True):\n        'Initialization'\n        self.dim = dim\n        self.batch_size = batch_size\n        self.labels = labels\n        self.list_IDs = list_IDs\n        self.n_channels = n_channels\n        self.n_classes = n_classes\n        self.shuffle = shuffle\n        self.on_epoch_end()\n        \n    \n    \n    \n    def __len__(self):\n        'Denotes the number of batches per epoch'\n        return int(np.floor(len(self.list_IDs) / self.batch_size))\n\n    def __getitem__(self, index):\n        'Generate one batch of data'\n        # Generate indexes of the batch\n        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n\n        # Find list of IDs\n#         list_IDs_temp = [self.list_IDs[k] for k in indexes]\n\n        # Generate data\n        X, y = self.__data_generation(indexes)\n\n        return X, y\n\n    def on_epoch_end(self):\n        'Updates indexes after each epoch'\n        self.indexes = np.arange(len(self.list_IDs))\n        if self.shuffle == True:\n            np.random.shuffle(self.indexes)\n\n    def __data_generation(self, list_IDs_temp):\n        'Generates data containing batch_size samples' # X : (n_samples, *dim, n_channels)\n        # Initialization\n        X = []\n        y = []\n\n        # Generate data\n        for i, ID in enumerate(list_IDs_temp):\n            # Store sample\n#             print(ID)\n            _tempx = wavfile_to_examples(self.list_IDs[ID])            \n            X.extend(_tempx)\n\n            # Store class\n            labels = np.zeros(len(BIRD_CODE), dtype=\"f\")\n            labels[self.labels[ID]] = 1\n            y.extend([labels] * _tempx.shape[0])\n        labeled_examples = list(zip(X, y))\n        shuffle(labeled_examples)\n        # Separate and return the features and labels.\n        features = [example for (example, _) in labeled_examples]\n        labels = [label for (_, label) in labeled_examples]\n        return features, labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X, Y = train_file_list['file_path'].values.tolist(), train_file_list['ebird_code'].values.tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"training_generator = DataGenerator(X, Y,batch_size=32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# x, y = training_generator[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !curl -O https://storage.googleapis.com/audioset/vggish_model.ckpt\n# !curl -O https://storage.googleapis.com/audioset/vggish_pca_params.npz","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def del_all_flags(FLAGS):\n#     flags_dict = FLAGS._flags()\n#     keys_list = [keys for keys in flags_dict]\n#     for keys in keys_list:\n#     FLAGS.delattr(keys)\n\n# del_all_flags(tf.flags.FLAGS)\n\n\n# flags = tf.app.flags\n\n# flags.DEFINE_integer(\n#     'num_batches', 30,\n#     'Number of batches of examples to feed into the model. Each batch is of '\n#     'variable size and contains shuffled examples of each class of audio.')\n\n# flags.DEFINE_boolean(\n#     'train_vggish', True,\n#     'If True, allow VGGish parameters to change during training, thus '\n#     'fine-tuning VGGish. If False, VGGish parameters are fixed, thus using '\n#     'VGGish as a fixed feature extractor.')\n\n# flags.DEFINE_string(\n#     'checkpoint', 'vggish_model.ckpt',\n#     'Path to the VGGish checkpoint file.')\n\n# FLAGS = flags.FLAGS\n\n_NUM_CLASSES = 264","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tqdm\n_NUM_CLASSES = 264\nwith tf.Graph().as_default(), tf.Session() as sess:\n    # Define VGGish.\n#     embeddings = define_vggish_slim(training=FLAGS.train_vggish)\n    embeddings = define_vggish_slim(training=True)\n\n    # Define a shallow classification model and associated training ops on top\n    # of VGGish.\n    with tf.variable_scope('mymodel'):\n      # Add a fully connected layer with 100 units. Add an activation function\n      # to the embeddings since they are pre-activation.\n      num_units = 100\n      fc = slim.fully_connected(tf.nn.relu(embeddings), num_units)\n\n      # Add a classifier layer at the end, consisting of parallel logistic\n      # classifiers, one per class. This allows for multi-class tasks.\n      logits = slim.fully_connected(\n          fc, _NUM_CLASSES, activation_fn=None, scope='logits')\n      tf.sigmoid(logits, name='prediction')\n      \n      # Add training ops.\n      with tf.variable_scope('train'):\n        global_step = tf.train.create_global_step()\n\n        # Labels are assumed to be fed as a batch multi-hot vectors, with\n        # a 1 in the position of each positive class label, and 0 elsewhere.\n        labels_input = tf.placeholder(\n            tf.float32, shape=(None, _NUM_CLASSES), name='labels')\n\n        # Cross-entropy label loss.\n        xent = tf.nn.sigmoid_cross_entropy_with_logits(\n            logits=logits, labels=labels_input, name='xent')\n        loss = tf.reduce_mean(xent, name='loss_op')\n        tf.summary.scalar('loss', loss)\n\n        # We use the same optimizer and hyperparameters as used to train VGGish.\n        optimizer = tf.train.AdamOptimizer(\n            learning_rate=LEARNING_RATE,\n            epsilon=ADAM_EPSILON)\n        train_op = optimizer.minimize(loss, global_step=global_step)\n\n    # Initialize all variables in the model, and then load the pre-trained\n    # VGGish checkpoint.\n    sess.run(tf.global_variables_initializer())\n#     vggish_slim.load_vggish_slim_checkpoint(sess, FLAGS.checkpoint)\n#     load_vggish_slim_checkpoint(sess, 'vggish_model.ckpt')\n    load_vggish_slim_checkpoint(sess, '../input/tensorflowbirdcallvggish2epoch/saved_vggish1/vggish')\n    \n\n    # The training loop.\n    features_input = sess.graph.get_tensor_by_name(\n        INPUT_TENSOR_NAME)\n    for epoch_id in range(settings['globals']['num_epochs']):\n        print(f\"epoch {epoch_id} started\")\n    #     for _ in range(FLAGS.num_batches):\n        for batch_idx in range(len(training_generator)):\n          (features, labels) = training_generator[batch_idx]\n          [num_steps, loss_value, _] = sess.run(\n              [global_step, loss, train_op],\n              feed_dict={features_input: features, labels_input: labels})\n          print('Step %d: loss %g' % (num_steps, loss_value))\n        saver = tf.train.Saver()\n        directory = f'./saved_vggish{epoch_id}/'\n        filename = 'vggish'\n        if not os.path.exists(directory):\n            os.makedirs(directory)\n        filepath = os.path.join(directory, filename)\n        saver.save(sess, filepath)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# tf.train.latest_checkpoint('../input/tensorflowbirdcallvggish2epoch/saved_vggish1/')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"It will stuck after some data see here\nhttps://github.com/keras-team/keras/issues/10948","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"This will cause deadlock problem\n\nuse_multiprocessing=True,\n\nworkers=2,","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# model = compile_new_model()\n# history = model.fit_generator(generator=training_generator,\n#                     epochs=settings['globals']['num_epochs'],\n# #                     validation_data=validation_generator,\n# #                     use_multiprocessing=True,\n# #                     workers=2,\n#                     callbacks=[earlystopping_cb, mdlcheckpoint_cb])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model = compile_new_model()\n# history = model.fit(\n#     train_ds,\n#     epochs=settings['globals']['num_epochs'],\n#     validation_data=valid_ds,\n#     callbacks=[earlystopping_cb, mdlcheckpoint_cb],\n# )","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}