{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport sys\nimport gc\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn.model_selection import train_test_split\n\nfrom joblib import Parallel, delayed\nfrom tqdm import tqdm_notebook\n\nimport librosa\nimport scipy\nimport glob\nimport cv2\n\n# Machine Learning\nimport tensorflow as tf\nfrom keras import backend as K\nfrom keras.optimizers import Adam\nfrom keras.callbacks import EarlyStopping\nfrom keras.layers import Input, Dense, Lambda, Flatten, Reshape, Activation, Dropout, Add, TimeDistributed, Multiply, Conv1D, Conv2D, MaxPooling1D, AveragePooling1D\nfrom keras.models import Model, Sequential, load_model\nfrom keras import metrics\nfrom keras import optimizers\nfrom keras.callbacks import History, ModelCheckpoint","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Device init\nprint(tf.test.gpu_device_name())\n# See https://www.tensorflow.org/tutorials/using_gpu#allowing_gpu_memory_growth\nconfig = tf.compat.v1.ConfigProto()\nconfig.gpu_options.allow_growth = True","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# The below class can be found at https://github.com/mjpyeon/wavenet-classifier","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class WaveNetClassifier():\n  def __init__(self, input_shape, output_shape, kernel_size = 2, dilation_depth = 9, n_filters = 40, task = 'classification', regression_range = None, load=False, load_dir='./'):\n    \"\"\"\n    Parameters:\n      input_shape: (tuple) tuple of input shape. (e.g. If input is 6s raw waveform with sampling rate = 16kHz, (96000,) is the input_shape)\n      output_shape: (tuple)tuple of output shape. (e.g. If we want classify the signal into 100 classes, (100,) is the output_shape)\n      kernel_size: (integer) kernel size of convolution operations in residual blocks\n      dilation_depth: (integer) type total depth of residual blocks\n      n_filters: (integer) # of filters of convolution operations in residual blocks\n      task: (string) 'classification' or 'regression'\n      regression_range: (list or tuple) target range of regression task\n      load: (bool) load previous WaveNetClassifier or not\n      load_dir: (string) the directory where the previous model exists\n    \"\"\"      \n    # save task info\n    self.task = task\n    if task == 'regression':\n      if regression_range[0] == 0:\n        self.activation = 'sigmoid'\n        self.scale_ratio = regression_range[1]\n      elif regression_range[0] == - regression_range[1]:\n        self.activation = 'tanh'\n        self.scale_ratio = regression_range[1]\n      elif regression_range == None:\n        self.activation = 'linear'\n        self.scale_ratio = 1\n      else:\n        print('ERROR: wrong regression range')\n        sys.exit()  \n    elif task == 'classification':\n      self.activation = 'softmax'\n      self.scale_ratio = 1      \n    else:\n      print('ERROR: wrong task')\n      sys.exit()\n    \n    # save input info\n    if len(input_shape) == 1:\n      self.expand_dims = True\n    elif len(input_shape) == 2:\n      self.expand_dims = False\n    else:\n      print('ERROR: wrong input shape')\n      sys.exit()\n    self.input_shape = input_shape\n    \n    # save output info\n    if len(output_shape) == 1:\n      self.time_distributed = False\n    elif len(output_shape) == 2:\n      self.time_distributed = True\n    else:\n      print('ERROR: wrong output shape')\n      sys.exit()\n    self.output_shape = output_shape\n    \n    # save hyperparameters of WaveNet\n    self.kernel_size = kernel_size\n    self.dilation_depth = dilation_depth\n    self.n_filters = n_filters\n    self.manual_loss = None\n\n    \n    if load is True:\n      self.model = load_model(load_dir+\"saved_wavenet_clasifier.h5\", custom_objects={'tf':tf})\n      self.prev_history = pd.read_csv(load_dir+'wavenet_classifier_training_history.csv')\n      self.start_idx = len(self.prev_history)\n      self.history = None\n    else:\n      self.model = self.construct_model()\n      self.start_idx = 0\n      self.history = None\n      self.prev_history = None\n\n    \n  def residual_block(self, x, i):\n    tanh_out = Conv1D(self.n_filters, \n                      self.kernel_size, \n                      dilation_rate = self.kernel_size**i, \n                      padding='causal', \n                      name='dilated_conv_%d_tanh' % (self.kernel_size ** i), \n                      activation='tanh'\n                      )(x)\n    sigm_out = Conv1D(self.n_filters, \n                      self.kernel_size, \n                      dilation_rate = self.kernel_size**i, \n                      padding='causal', \n                      name='dilated_conv_%d_sigm' % (self.kernel_size ** i), \n                      activation='sigmoid'\n                      )(x)\n    z = Multiply(name='gated_activation_%d' % (i))([tanh_out, sigm_out])\n    skip = Conv1D(self.n_filters, 1, name='skip_%d'%(i))(z)\n    res = Add(name='residual_block_%d' % (i))([skip, x])\n    return res, skip\n  \n  def construct_model(self):    \n    x = Input(shape=self.input_shape, name='original_input')\n    if self.expand_dims == True:\n      x_reshaped = Reshape(self.input_shape + (1,), name='reshaped_input')(x)\n    else:\n      x_reshaped = x\n    skip_connections = []\n    out = Conv1D(self.n_filters, 2, dilation_rate=1, padding='causal', name='dilated_conv_1')(x_reshaped)\n    for i in range(1, self.dilation_depth + 1):\n      out, skip = self.residual_block(out,i)\n      skip_connections.append(skip)\n    out = Add(name='skip_connections')(skip_connections)\n    out = Activation('relu')(out)\n    out = Conv1D(self.n_filters, 80, strides = 1, padding='same', name='conv_5ms', activation = 'relu')(out)\n    out = AveragePooling1D(80, padding='same', name='downsample_to_200Hz')(out)\n    if self.time_distributed:\n      target_kernel_size = (int) (self.input_shape[0] / 80 / self.output_shape[0]) # prev_len / x = target_len => x = prev_len / target_len\n      out = Conv1D(self.n_filters, target_kernel_size, padding='same', name = 'conv_fit_to_target', activation='relu')(out)\n      out = Conv1D(self.output_shape[1], target_kernel_size, padding='same', name='conv_final')(out)\n      out = AveragePooling1D(target_kernel_size, padding='same')(out)\n      out = TimeDistributed(Activation(self.activation))(out)\n    else:\n      out = Conv1D(self.n_filters, 100, padding='same', activation='relu', name='conv_500ms')(out)\n      out = Conv1D(self.output_shape[0], 100, padding='same', activation='relu', name='conv_500ms_target_shape')(out)\n      out = AveragePooling1D(100, padding='same',name = 'downsample_to_2Hz')(out)\n      out = Conv1D(self.output_shape[0], (int) (self.input_shape[0] / 8000), padding='same', name='final_conv')(out)\n      out = AveragePooling1D((int) (self.input_shape[0] / 8000), name='final_pooling')(out)\n      out = Reshape(self.output_shape)(out)\n      out = Activation(self.activation)(out)\n    if self.scale_ratio != 1:\n      out = Lambda(lambda x: x * self.scale_ratio, name='output_reshaped')(out)\n    model = Model(x, out)  \n    model.summary()\n    return model\n    \n  def get_model(self):\n    return self.model\n    \n  def add_loss(self, loss):\n    self.manual_loss = loss\n  \n  def fit(self, X, Y, validation_data = None, epochs = 100, batch_size = 32, optimizer='adam', save=False, save_dir='/kaggle/output/'):\n    # set default losses if not defined\n    if self.manual_loss is not None:\n      loss = self.manual_loss\n      metrics = None\n    else:\n      if self.task == 'classification':\n        loss = 'categorical_crossentropy'\n        metrics = ['accuracy']\n      else:\n        loss = 'mean_squared_error'\n        metrics = None\n        \n    # set callback functions\n    if save:\n      saved = save_dir + \"saved_wavenet_clasifier.h5\"\n      hist = save_dir + 'wavenet_classifier_training_history.csv'\n      if validation_data is None:\n        checkpointer = ModelCheckpoint(filepath=saved, monitor='loss', verbose=1, save_best_only=True)\n      else:\n        checkpointer = ModelCheckpoint(filepath=saved, monitor='val_loss', verbose=1, save_best_only=True)\n      history = History()\n      callbacks = [history, checkpointer]\n    else:\n      callbacks = None\n      \n    # compile the model\n    self.model.compile(optimizer, loss, metrics)\n    try:\n      self.history = self.model.fit(X, Y, shuffle = True, batch_size=batch_size, epochs = epochs, validation_data = validation_data, callbacks=callbacks, initial_epoch=self.start_idx)\n    except:\n      if save:\n      \tdf = pd.DataFrame.from_dict(history.history)\n      \tdf.to_csv(hist, encoding='utf-8', index=False)\n      raise\n      sys.exit()\n    return self.history\n\n\n  def predict(self, x):\n    return self.model.predict(x)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Preprocessing parameters\nduration = 10\n# -----------------------------------------------------------------\ndef read_librosa(path):\n    '''\n    Reads in the audio file and returns\n    an array that we can turn into a melspectogram\n    '''\n    y, sr = librosa.core.load(path)\n    gc.collect()\n    return np.resize(y,(480000,))\n# -----------------------------------------------------------------\ndef read_wav(path):\n    '''\n    Reads in the audio file and returns\n    an array that we can turn into a melspectogram\n    '''\n    sr, y = scipy.io.wavfile.read(path, mmap=False)\n    gc.collect()\n    return np.resize(y,(480000,))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load all available filepaths","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"This miniature wav dataset is at https://www.kaggle.com/eladwar/birdcalls.\nThe source of the files came from https://www.kaggle.com/rhtsingh/cornell-birdcall-superfast-mp3-to-wav.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir('/kaggle/input/birdcalls/train_wav/')\ntrain_wav = []\nfor directory in os.listdir('/kaggle/input/birdcalls/train_wav/'):\n    train_wav.extend(glob.glob('/kaggle/input/birdcalls/train_wav/'+directory+'/*'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_audio = []\nfor directory in glob.glob('/kaggle/input/birdsong-recognition/train_audio/*'):\n    train_audio.extend(glob.glob(directory+'/*'))\ntrain = pd.read_csv('/kaggle/input/birdsong-recognition/train.csv')\ntrain['filename'] = train_audio","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# I know they're .wav files, but why not shave off some time?","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nX = np.array([Parallel(n_jobs=4)(delayed(read_wav)(filename) for filename in train_wav)])[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Preprocess dataset and create validation sets\nseed = 43\ntrain = pd.read_csv('/kaggle/input/birdsong-recognition/train.csv')\nY = pd.get_dummies(train['ebird_code'])[:len(X)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X.shape,Y.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train, x_val, y_train, y_val = train_test_split(X, Y, test_size=0.2, random_state=seed)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Batches need to remain small in order to prevent resoure exhaustion error.**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"wnc = WaveNetClassifier((480000,), (264,), kernel_size = 2, dilation_depth = 9, n_filters = 40, task = 'classification')\nwith tf.device('/gpu:0'):\n    wnc.fit(x_train, y_train, validation_data = (x_val, y_val), epochs = 2, batch_size = 4, optimizer='adam', save=True, save_dir='./')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/birdsong-recognition/test.csv')\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# As a request, I am interested in seeing this wavenet class used with https://github.com/parasj/checkmate.\n# It supposedly trains neural networks in TensorFlow 2.0 with 5x less memory, which could make for much larger batches and better results.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# y_pred = wnc.predict(X_test)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}