{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"#References\n#https://www.groundai.com/project/environment-sound-classification-using-multiple-feature-channels-and-deep-convolutional-neural-networks/1\n#https://keunwoochoi.wordpress.com/2019/09/28/log-melspectrogram-layer-using-tensorflow-keras/\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Import Required Libraries","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os     \nimport seaborn as sns\n\nimport warnings\nwarnings.filterwarnings('ignore')\nimport os\nimport librosa\nimport librosa.display\nimport numpy as np\n\nimport matplotlib.pyplot as plt\nfrom matplotlib.pyplot import specgram\nfrom sklearn.model_selection import KFold\n\nimport tensorflow as tf\nfrom tensorflow import keras\n#!pip install python_speech_features\n%matplotlib inline\nplt.style.use('ggplot')\nimport glob\nimport glob\nimport librosa\nfrom librosa import feature\nimport numpy as np\nfrom pathlib import Path\nimport cv2\nAUTO = tf.data.experimental.AUTOTUNE\nfrom kaggle_datasets import KaggleDatasets\nimport scipy\nimport pickle\nfrom sklearn.model_selection import train_test_split\nimport time\nimport statistics","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Detect Hardware and accordingly set strategy","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ndef get_strategy():\n    TFREC_GCS_DIR = \"\"\n    gpu = \"\"\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())     \n    except ValueError:\n        tpu = None\n        os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n        gpu = tf.config.list_physical_devices(\"GPU\")\n        if len(gpu) == 1:\n            print('Running on GPU ', gpu)\n    if tpu:\n        print(\"Running in TPU\")\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        GCS_PATH = KaggleDatasets().get_gcs_path('birdsong-recognition')\n        TFREC_GCS_DIR = KaggleDatasets().get_gcs_path('birdcall-tfrec-2s')\n        print()\n    elif len(gpu) == 1:\n        strategy = tf.distribute.OneDeviceStrategy(device=\"/gpu:0\")\n        tf.config.optimizer.set_experimental_options({\"auto_mixed_precision\":True})\n        GCS_PATH = \"/kaggle/input/birdsong-recognition/\"\n    else:\n        strategy = tf.distribute.get_strategy()\n        GCS_PATH = \"/kaggle/input/birdsong-recognition/\"\n\n    print(\"REPLICAS: \", strategy.num_replicas_in_sync)\n    base_dir = \"../input/birdsong-recognition/\"\n    print(base_dir)\n    return strategy, GCS_PATH, base_dir, TFREC_GCS_DIR\n\nstrategy,GCS_PATH, base_dir, TFREC_GCS_DIR = get_strategy()\nsns.set_palette(\"pastel\")\npalette = sns.color_palette()\nCACHE = {}\ntfrec_dir = \"../input/birdcall-tfrec-2s/\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_ebird_filename_dic():\n    ebird_code_list = all_train_data[\"ebird_code\"].unique()\n    if os.path.exists(tfrec_dir + \"dic_ebird.pkl\"):\n        with open(tfrec_dir + \"dic_ebird.pkl\",\"rb\") as f:\n            dic_ebird_code = pickle.load(f)\n    else:\n        dic_ebird_code = {k:v for v,k in enumerate(ebird_code_list)}\n    \n    dic_ebird_code_rev = [v for v,k in dic_ebird_code.items()]\n    all_train_data[\"int_ebird_code\"] = all_train_data[\"ebird_code\"].map(dic_ebird_code)\n\n    filename_list = all_train_data[\"filename\"].unique()\n    if os.path.exists(tfrec_dir + \"dic_filename.pkl\"):\n        with open(tfrec_dir + \"dic_filename.pkl\",\"rb\") as f:\n            dic_filename = pickle.load(f)\n    else:\n        dic_filename = {k:v for v,k in enumerate(filename_list)}\n    dic_filename_rev = [v for v,k in dic_filename.items()]\n    all_train_data[\"int_filename\"] = all_train_data[\"filename\"].map(dic_filename)\n\n    with open(\"dic_ebird.pkl\",\"wb\") as f:\n        pickle.dump(dic_ebird_code, f)\n\n    with open(\"dic_filename.pkl\",\"wb\") as f:\n        pickle.dump(dic_filename, f)\n        \n    return dic_ebird_code_rev, dic_filename_rev","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def parse_rec_train(data):           \n    feature_set = {\n        'img': tf.io.FixedLenFeature([], tf.string),\n        'int_ebird_code': tf.io.FixedLenFeature([], tf.int64),\n    }\n    features = tf.io.parse_single_example(data, features= feature_set )\n    img = features[\"img\"]\n    img = tf.image.decode_image(img)\n    img = tf.ensure_shape(img, (img_sz1, img_sz2, 1))\n    y = features['int_ebird_code']\n    return tf.cast(img, tf.float32), tf.one_hot(y, 264)\n\n\ndef parse_waveform_rec_train(data):           \n    feature_set = {\n        'img': tf.io.FixedLenFeature([], tf.string),\n        'int_ebird_code': tf.io.FixedLenFeature([], tf.int64),\n    }\n    features = tf.io.parse_single_example(data, features= feature_set )\n    img = features[\"img\"]\n    img = tf.image.decode_image(img)\n    img = tf.ensure_shape(img, (img_sz1, img_sz2, 1))\n    y = features['int_ebird_code']\n    return tf.cast(img, tf.float32), tf.one_hot(y, 264)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Spectrogram Features","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"all_train_data = pd.read_csv(base_dir + \"train.csv\")\nall_train_data = all_train_data[all_train_data[\"duration\"]>=5]\nall_train_data = all_train_data[all_train_data[\"filename\"]!=\"XC195038.mp3\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_sz1 = 64\nimg_sz2 = 512","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dic_ebird_code_rev, dic_filename_rev = get_ebird_filename_dic()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_lr_callback(batch_size=8):\n    lr_start   = 0.000005\n    lr_max     = 0.000020   * strategy.num_replicas_in_sync\n    lr_min     = 0.000001\n    lr_ramp_ep = 5\n    lr_sus_ep  = 0\n    lr_decay   = 0.8\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.applications.resnet50 import ResNet50\nfrom keras.layers import GlobalMaxPooling2D, Dense, Dropout, Input, Conv2D, BatchNormalization, Activation, MaxPooling2D, AveragePooling2D, GlobalAveragePooling2D\nfrom keras import Model\n\nimport numpy as np\nimport soundfile as sf\n\nimport matplotlib.pyplot as plt\n\n\n\nimport tensorflow as tf\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Input, Dense\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if 1==2:\n    !pip install soundfile\n    !git clone https://github.com/tensorflow/models.git\n    %cd models/research/audioset/yamnet\n\n    # Download YAMNet data\n    !curl -O https://storage.googleapis.com/audioset/yamnet.h5\n\n\n    # Imports.\n    import numpy as np\n    import soundfile as sf\n\n    import matplotlib.pyplot as plt\n\n    import params\n    import yamnet as yamnet_model\n    import tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Params:\n    sample_rate: float = 16000.0\n    stft_window_seconds: float = 0.025\n    stft_hop_seconds: float = 0.010\n    mel_bands: int = 64\n    mel_min_hz: float = 125.0\n    mel_max_hz: float = 7500.0\n    log_offset: float = 0.001\n    patch_window_seconds: float = 0.96\n    patch_hop_seconds: float = 0.48\n\n    @property\n    def patch_frames(self):\n        return int(round(self.patch_window_seconds / self.stft_hop_seconds))\n\n    @property\n    def patch_bands(self):\n        return self.mel_bands\n\n    num_classes: int = 521\n    conv_padding: str = 'same'\n    batchnorm_center: bool = True\n    batchnorm_scale: bool = False\n    batchnorm_epsilon: float = 1e-4\n    classifier_activation: str = 'sigmoid'\n\n    tflite_compatible: bool = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_yamnet_model(params):\n    # Install required packages.\n    \n    yamnet = yamnet_model.yamnet_frames_model(params)\n    yamnet.load_weights('yamnet.h5')\n\n    x = tf.keras.layers.Dense(1024, activation='relu')(yamnet.layers[-3].output)\n    o = tf.keras.layers.Dropout(0.5)(x)\n    o = tf.keras.layers.Dense(264, activation='softmax')(o)\n\n    model = Model(inputs=yamnet.input, outputs=o)\n\n    for layer in model.layers:\n        layer.trainable = True\n\n\n    checkpoint = ModelCheckpoint('model.h5',\n                                 monitor='val_loss', \n                                 verbose=1,\n                                 save_best_only=True, \n                                 mode='auto')\n\n    reducelr = ReduceLROnPlateau(monitor='val_loss', \n                                  factor=0.5, \n                                  patience=3, \n                                  verbose=1)\n\n    opt = tf.keras.optimizers.Adam(learning_rate=0.0001)\n    loss = tf.keras.losses.CategoricalCrossentropy() \n    model.compile(optimizer=opt,loss=loss,metrics=['AUC'])\n    \n#yamnet = get_yamnet_model(Params())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if 1==2:\n    with strategy.scope():\n        if 1==2:\n            base_model = ResNet50(weights='imagenet', include_top=False)\n\n            # add a global spatial average pooling layer\n            x = base_model.output\n            x = GlobalMaxPooling2D()(x)\n            # let's add a fully-connected layer\n            x = Dense(1024, activation='tanh')(x)\n            x = Dropout(0.2)(x)\n            x = Dense(1024, activation='relu')(x)\n            x = Dropout(0.2)(x)\n            # and a logistic layer -- let's say we have 200 classes\n            predictions = Dense(264, activation='sigmoid')(x)\n\n            # this is the model we will train\n            model = Model(inputs=base_model.input, outputs=predictions)\n            model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy',tf.keras.metrics.AUC()])\n            #model.summary()\n            callback = get_lr_callback(BATCH_SIZE)\n            sv = tf.keras.callbacks.ModelCheckpoint(\n                'model.h5', monitor='val_loss', verbose=0, save_best_only=True,\n                save_weights_only=False, mode='min', save_freq='epoch')\n\n            model.fit(train_dataset, epochs=5, verbose=1, callbacks=[sv, callback], steps_per_epoch=200, validation_data = valid_dataset, validation_steps=10)    #steps_per_epoch=steps_per_epoch, \n        else:\n            model = tf.keras.models.load_model('../input/multichannel-spectrogram/model.h5')\n            model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy',tf.keras.metrics.AUC()])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_conv_block(net, filters, kernel_size, activation=\"relu\"):\n    net = Conv2D(filters=RESNET_K*FILTERS[0], kernel_size=KERNEL_SIZES[0], padding='same')(net)\n    net = BatchNormalization()(net)\n    if activation is not None:\n        net = Activation(\"relu\")(net)\n    return net\n\ndef resblock(net_in, filters, kernel_size, stride=1, preactivated=True, block_id=1, name=''):\n    # Show input shape\n    #log.p((\"\\t\\t\" + name + \" IN SHAPE:\", l.get_output_shape(net_in)), new_line=False)\n\n    # Pre-activation\n    if block_id > 1:\n        net_pre = Activation(\"relu\")(net_in)\n    else:\n        net_pre = net_in\n\n    # Pre-activated shortcut?\n    if preactivated:\n        net_in = net_pre\n\n    # Bottleneck Convolution\n    if stride > 1:\n        net_pre = get_conv_block(net_pre, net_pre.shape[1], 1)\n       \n    # First Convolution     \n    net = get_conv_block(net_pre, net_pre.shape[1], kernel_size)\n\n    # Pooling layer\n   \n    if stride > 1:\n        net = MaxPooling2D(pool_size=(stride, stride))(net)\n\n    # Dropout Layer\n    net = Dropout(0.5)(net)       \n\n    # Second Convolution\n    net = get_conv_block(net, filters, kernel_size)\n\n    # Shortcut Layer\n    if not net.shape == net_in.shape:\n        # Average pooling\n        shortcut = AveragePooling2D(pool_size=(stride, stride))(net_in)\n\n        # Shortcut convolution\n        shortcut = get_conv_block(shortcut, filters, 1, None)  \n    else:\n\n        # Shortcut = input\n        shortcut = net_in\n    \n    # Merge Layer\n    out = tf.math.add_n([net, shortcut])\n\n    # Show output shape\n    #log.p((\"OUT SHAPE:\", l.get_output_shape(out), \"LAYER:\", len(l.get_all_layers(out)) - 1))\n\n    return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def classificationBranch(net, kernel_size):\n\n    # Post Convolution\n    branch = get_conv_block(net, int(FILTERS[-1] * RESNET_K), kernel_size)  \n\n    #log.p((\"\\t\\tPOST  CONV SHAPE:\", l.get_output_shape(branch), \"LAYER:\", len(l.get_all_layers(branch)) - 1))\n\n    # Dropout Layer\n    branch = Dropout(0.5)(branch)\n    \n    # Dense Convolution\n    branch = get_conv_block(net, int(FILTERS[-1] * RESNET_K * 2), 1) \n\n    #log.p((\"\\t\\tDENSE CONV SHAPE:\", l.get_output_shape(branch), \"LAYER:\", len(l.get_all_layers(branch)) - 1))\n    \n    # Dropout Layer\n    branch = Dropout(0.5)(branch)\n    \n    # Class Convolution\n    branch = Conv2D(filters=CLASSES,kernel_size=1)(branch)\n    return branch\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import keras.backend as K\ndef f1_loss(y, y_hat):\n    \"\"\"Compute the macro soft F1-score as a cost.\n    Average (1 - soft-F1) across all labels.\n    Use probability values instead of binary predictions.\n    \n    Args:\n        y (int32 Tensor): targets array of shape (BATCH_SIZE, N_LABELS)\n        y_hat (float32 Tensor): probability matrix of shape (BATCH_SIZE, N_LABELS)\n        \n    Returns:\n        cost (scalar Tensor): value of the cost function for the batch\n    \"\"\"\n    \n    y = tf.cast(y, tf.float32)\n    y_hat = tf.cast(y_hat, tf.float32)\n    tp = tf.reduce_sum(y_hat * y, axis=0)\n    fp = tf.reduce_sum(y_hat * (1 - y), axis=0)\n    fn = tf.reduce_sum((1 - y_hat) * y, axis=0)\n    soft_f1 = 2*tp / (2*tp + fn + fp + 1e-16)\n    cost = 1 - soft_f1 # reduce 1 - soft-f1 in order to increase soft-f1\n    macro_cost = tf.reduce_mean(cost) # average on all labels\n    \n    return macro_cost\n\ndef f1_loss(y_true, y_pred):\n    \n    tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)\n    tn = K.sum(K.cast((1-y_true)*(1-y_pred), 'float'), axis=0)\n    fp = K.sum(K.cast((1-y_true)*y_pred, 'float'), axis=0)\n    fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)\n\n    p = tp / (tp + fp + K.epsilon())\n    r = tp / (tp + fn + K.epsilon())\n\n    f1 = 2*p*r / (p+r+K.epsilon())\n    f1 = tf.where(tf.math.is_nan(f1), tf.zeros_like(f1), f1)\n    return 1 - K.mean(f1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CLASSES=264\nFILTERS = [8, 16, 32, 64, 128]\nKERNEL_SIZES = [(5, 5), (3, 3), (3, 3), (3, 3), (3, 3)]\nRESNET_K = 4\nRESNET_N = 3\n\ndef buildNet(fold, model_path=None):\n    \n    inp = Input(shape=(img_sz1, img_sz2, 1))\n    \n    net = get_conv_block(inp, RESNET_K*FILTERS[0], KERNEL_SIZES[0])\n    net = MaxPooling2D(pool_size=(1,2), padding=\"same\")(net)\n    for i in range(1, len(FILTERS)):\n            #log.p((\"\\tRES STACK\", i, ':'))\n            net = resblock(net,\n                           filters=int(FILTERS[i] * RESNET_K),\n                           kernel_size=KERNEL_SIZES[i],\n                           stride=2,\n                           preactivated=True,\n                           block_id=i,\n                           name='BLOCK ' + str(i) + '-1')\n\n            for j in range(1, RESNET_N):\n                net = resblock(net,\n                               filters=int(FILTERS[i] * RESNET_K),\n                               kernel_size=KERNEL_SIZES[i],\n                               preactivated=False,\n                               block_id=i+j,\n                               name='BLOCK ' + str(i) + '-' + str(j + 1))\n    net = BatchNormalization()(net)\n    net = Activation(\"relu\")(net)\n\n    # Classification branch\n    #log.p((\"\\tCLASS BRANCH:\"))\n    net = classificationBranch(net,  (4, 10)) \n    #log.p((\"\\t\\tBRANCH OUT SHAPE:\", l.get_output_shape(net), \"LAYER:\", len(l.get_all_layers(net)) - 1))\n\n    # Pooling\n    net = GlobalAveragePooling2D()(net)\n    #log.p((\"\\tGLOBAL POOLING SHAPE:\", l.get_output_shape(net), \"LAYER:\", len(l.get_all_layers(net)) - 1))\n\n    # Sigmoid output\n    net = Activation(\"softmax\")(net)\n    print(net.shape)\n    model = Model(inputs=inp, outputs=net)\n    \n    optimizer = tf.keras.optimizers.Adam(lr=0.0005)\n    model.compile(optimizer='adam', loss=tf.keras.losses.CategoricalCrossentropy(), metrics=[tf.keras.metrics.Precision(), tf.keras.metrics.Recall()])\n    \n    #model.summary()\n    learning_rate_reduction = keras.callbacks.ReduceLROnPlateau(monitor='val_loss', \n                                            patience=5, \n                                            verbose=1, \n                                            factor=0.5, \n                                            min_lr=0.000001)\n    model_file = \"model_\" + str(fold) + \".h5\"\n    if model_path is None:\n        model_path = \"../input/multichannel-spectrogram/\"\n    if os.path.exists(model_path + model_file):\n        print(\"Found model!\", model_path + model_file)\n        model.load_weights(model_path + model_file)\n        blnTrain = False\n    else:\n        blnTrain = True\n    \n    sv = tf.keras.callbacks.ModelCheckpoint(\n        model_file, monitor='val_loss', verbose=0, save_best_only=True,\n        save_weights_only=True, mode='min', save_freq='epoch')\n    es = tf.keras.callbacks.EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=10)\n    return model, [learning_rate_reduction, sv, es], model_file, True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_train_data.head(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_train_val_ds(train_idx, val_idx, train_val_data):\n    dic_train = get_dic_for_data(train_idx, train_val_data)\n    dic_val = get_dic_for_data(val_idx, train_val_data)\n    \n    train_dataset_raw = tf.data.Dataset.from_generator(\n         DataGenerator, args=[dic_train], \\\n         output_types=(tf.float32, tf.int64),\n         output_shapes=(tf.TensorShape((img_sz1, img_sz2, 1)), tf.TensorShape((264))))\n    \n    valid_dataset_raw = tf.data.Dataset.from_generator(\n         DataGenerator, args=[dic_val],\\\n         output_types=(tf.float32, tf.int64),\n         output_shapes=(tf.TensorShape((img_sz1, img_sz2, 1)), tf.TensorShape((264) )))\n\n\n    \n\n    train_dataset = train_dataset_raw.batch(BATCH_SIZE).prefetch(AUTO)\n    valid_dataset = valid_dataset_raw.batch(BATCH_SIZE).prefetch(AUTO)\n    for X,y in train_dataset.take(1):\n       print(X.shape)\n\n    fix, ax = plt.subplots(2,3, figsize=(18,3))\n    for row in range(2):\n        for col in range(3):\n            ax[row,col].imshow(tf.reshape(X[row*col + col], (img_sz1, img_sz2)))\n            \n    return train_dataset, valid_dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_metrics(num_data, metric_name, color_train, color_valid, loc, min_max_arg_func, min_max_func, train_metric_data, val_metric_data):\n    plt.plot(np.arange(num_data),train_metric_data,'-o',label=metric_name,color=color_train)\n    plt.plot(np.arange(num_data),val_metric_data,'-o',label='Val ' + metric_name ,color=color_valid)\n    x = min_max_arg_func( val_metric_data); y = min_max_func( val_metric_data )\n    xdist = plt.xlim()[1] - plt.xlim()[0]; ydist = plt.ylim()[1] - plt.ylim()[0]\n    plt.scatter(x,y,s=200,color=color_valid); plt.text(x-0.03*xdist,y-0.13*ydist,'min/max %s\\n%.2f'%(metric_name, y),size=14)\n    plt.ylabel(metric_name,size=14); plt.xlabel('Epoch',size=14)\n    plt.legend(loc=loc)\n    \ndef get_f1(history, precision_key, recall_key):\n    train_f1 = []\n    train_precision = history.history[precision_key]\n    train_recall = history.history[recall_key]\n    for i in range(len(train_precision)):\n        f1 = statistics.harmonic_mean([train_precision[i], train_recall[i]])\n        train_f1.append(f1)\n        \n    valid_f1 = []\n    valid_precision = history.history[\"val_\" + precision_key]\n    valid_recall = history.history[\"val_\" + recall_key]\n    for i in range(len(valid_precision)):\n        f1 = statistics.harmonic_mean([valid_precision[i], valid_recall[i]])\n        valid_f1.append(f1)\n        \n    return train_f1, valid_f1\n        \n        \n        \n    \ndef plot_history(history):\n    num_data = len(history.history['loss'])\n    plt.figure(figsize=(15,5))\n    \n    for key in history.history.keys():\n        if 'precision' in key and 'val' not in key:\n            precision_key = key\n            \n    for key in history.history.keys():\n        if 'recall' in key and 'val' not in key:\n            recall_key = key\n            \n    if 1==2:\n        plot_metrics(num_data, 'precision', '#ff7f0e', '#1f77b4', 2, np.argmax, np.max, history.history[precision_key], history.history['val_' + precision_key])\n\n        plt2 = plt.gca().twinx()\n\n        plot_metrics(num_data, 'recall', '#2ca02c', '#d62728', 3, np.argmax, np.max, history.history[recall_key], history.history['val_' + recall_key])\n   \n    plt.show()  \n    \n    train_f1, valid_f1 = get_f1(history, precision_key, recall_key)\n        \n        \n    plot_metrics(num_data, 'F1', '#ff7f0e', '#1f77b4', 2, np.argmax, np.max, train_f1, valid_f1)\n    \n    plt2 = plt.gca().twinx()\n    \n    plot_metrics(num_data, 'loss', '#2ca02c', '#d62728', 3, np.argmin, np.min, history.history[\"loss\"], history.history['val_' + \"loss\"])\n   \n    plt.show();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_file_list():\n    dataset_list = [\"birdcall-tfrec-2s\", \"birdcall-tfrec-2s-28\",\"birdcall-tfrec-2s-34\",\"birdcall-tfrec-2s-40\",\n                   \"birdcall-tfrec-2s-50\",\"birdcall-tfrec-2s-60\"]\n    file_list = []\n    for dataset in dataset_list:\n        gcs_dir = KaggleDatasets().get_gcs_path(dataset)\n        gcs_file = [file for file in tf.io.gfile.glob(TFREC_GCS_DIR + '/train*') if \".pkl\" not in file  if \".csv\" not in file ]\n        file_list = file_list + gcs_file\n\n    tfrec_files_train_all = np.sort(np.array(file_list))\n    return tfrec_files_train_all\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import KFold\nBLN_TRAIN = False\n\nKFOLD_SPLITS = 10\nBATCH_SIZE = 128 * strategy.num_replicas_in_sync\n\nif BLN_TRAIN:\n    kf = KFold(n_splits=KFOLD_SPLITS, shuffle=True,random_state=42)\n    \n    tfrec_files_train_all = get_file_list()\n    dataset_train = tf.data.TFRecordDataset(tfrec_files_train_all).map(parse_rec_train, num_parallel_calls=AUTO)\n    for X,y in dataset_train.skip(14).take(1):\n        plt.imshow(255 * X[:,:,0])\n\n    for fold , (train_idx, val_idx) in enumerate(kf.split(list(range(len(tfrec_files_train_all))))):\n        if fold < 1:\n            train_dataset_raw = tf.data.TFRecordDataset(tfrec_files_train_all[train_idx]).map(parse_rec_train, num_parallel_calls=AUTO).cache()\n            valid_dataset_raw = tf.data.TFRecordDataset(tfrec_files_train_all[val_idx]).map(parse_rec_train, num_parallel_calls=AUTO)\n            train_dataset = train_dataset_raw.shuffle(1024).batch(BATCH_SIZE).prefetch(AUTO)\n            valid_dataset = valid_dataset_raw.cache().batch(BATCH_SIZE).prefetch(AUTO)\n            with strategy.scope():\n                model, callbacks, model_file, bln_train = buildNet(fold)\n            if bln_train:\n                history = model.fit(train_dataset, epochs=25, verbose=1, callbacks=callbacks, validation_data = valid_dataset)    #steps_per_epoch=steps_per_epoch, \n                model.load_weights(model_file)\n                plot_history(history)\n            else:\n                model.save_weights(model_file)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"KFOLD_SPLITS = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nn_fft1 = int(0.0025 * 22050)\nhop_length1 = int(0.001 * 22050)\n\nn_fft2 = int(0.005 * 22050)\nhop_length2 = int(0.0025 * 22050)\n\nn_fft3 = int(0.01 * 22050)\nhop_length3 = int(0.005 * 22050)\nn_mels = 128\nfmin=150\nfmax=15000\nimg_sz1 = 64\nimg_sz2 = 512\n\n\n\n    \ndef buildBandpassFilter(rate, fmin, fmax, order=4):\n\n    global CACHE\n\n    fname = 'bandpass_' + str(rate) + '_' + str(fmin) + '_' + str(fmax)\n    if not fname in CACHE:\n        wn = np.array([fmin, fmax]) / (rate / 2.0)\n        filter_sos = scipy.signal.butter(order, wn, btype='bandpass', output='sos')\n        # Save to cache\n        CACHE[fname] = filter_sos\n\n    return CACHE[fname]\n\ndef applyBandpassFilter(sig, rate, fmin, fmax):\n    # Build filter or load from cache\n    filter_sos = buildBandpassFilter(rate, fmin, fmax)\n\n    return scipy.signal.sosfiltfilt(filter_sos, sig)\n\ndef get_mel_filterbanks(num_banks, fmin, fmax, f_vec, dtype=np.float32):\n    '''\n    An arguably better version of librosa's melfilterbanks wherein issues with \"hard snapping\" are avoided. Works with\n    an existing vector of frequency bins, as returned from signal.spectrogram(), instead of recalculating them and\n    flooring down the bin indices.\n    '''\n\n    global CACHE\n\n    # Filterbank already in cache?\n    fname = 'mel_' + str(num_banks) + '_' + str(fmin) + '_' + str(fmax)\n    if not fname in CACHE:\n        \n        # Break frequency and scaling factor\n        A = 4581.0\n        f_break = 1750.0\n\n        # Convert Hz to mel\n        freq_extents_mel = A * np.log10(1 + np.asarray([fmin, fmax], dtype=dtype) / f_break)\n\n        # Compute points evenly spaced in mels\n        melpoints = np.linspace(freq_extents_mel[0], freq_extents_mel[1], num_banks + 2, dtype=dtype)\n\n        # Convert mels to Hz\n        banks_ends = (f_break * (10 ** (melpoints / A) - 1))\n\n        filterbank = np.zeros([len(f_vec), num_banks], dtype=dtype)\n        for bank_idx in range(1, num_banks+1):\n            # Points in the first half of the triangle\n            mask = np.logical_and(f_vec >= banks_ends[bank_idx - 1], f_vec <= banks_ends[bank_idx])\n            filterbank[mask, bank_idx-1] = (f_vec[mask] - banks_ends[bank_idx - 1]) / \\\n                (banks_ends[bank_idx] - banks_ends[bank_idx - 1])\n\n            # Points in the second half of the triangle\n            mask = np.logical_and(f_vec >= banks_ends[bank_idx], f_vec <= banks_ends[bank_idx+1])\n            filterbank[mask, bank_idx-1] = (banks_ends[bank_idx + 1] - f_vec[mask]) / \\\n                (banks_ends[bank_idx + 1] - banks_ends[bank_idx])\n\n        # Scale and normalize, so that all the triangles do not have same height and the gain gets adjusted appropriately.\n        temp = filterbank.sum(axis=0)\n        non_zero_mask = temp > 0\n        filterbank[:, non_zero_mask] /= np.expand_dims(temp[non_zero_mask], 0)\n\n        # Save to cache\n        CACHE[fname] = (filterbank, banks_ends[1:-1])\n\n    return CACHE[fname][0], CACHE[fname][1]\n\ndef get_spectrogram(sig, rate, shape=(img_sz1, img_sz2), win_len=512, fmin=150, fmax=15000, magnitude_scale='nonlinear', bandpass=True, decompose=False):\n\n    # Compute overlap\n    hop_len = int(len(sig) / (shape[1] - 1)) \n    win_overlap = win_len - hop_len + 2\n    #print 'WIN_LEN:', win_len, 'HOP_LEN:', hop_len, 'OVERLAP:', win_overlap\n\n    \n    n_fft = win_len\n    \n\n    # Bandpass filter?\n    if bandpass:\n        sig = applyBandpassFilter(sig, rate, fmin, fmax)\n\n    # Compute spectrogram\n    f, t, spec = scipy.signal.spectrogram(sig,\n                                          fs=rate,\n                                          window=scipy.signal.windows.hann(win_len),\n                                          nperseg=win_len,\n                                          noverlap=win_overlap,\n                                          nfft=n_fft,\n                                          detrend=False,\n                                          mode='magnitude')\n\n    # Scale frequency?\n   \n\n    # Determine the indices of where to clip the spec\n    valid_f_idx_start = f.searchsorted(fmin, side='left')\n    valid_f_idx_end = f.searchsorted(fmax, side='right') - 1\n\n    # Get mel filter banks\n    mel_filterbank, mel_f = get_mel_filterbanks(shape[0], fmin, fmax, f, dtype=spec.dtype)\n\n    # Clip to non-zero range so that unnecessary multiplications can be avoided\n    mel_filterbank = mel_filterbank[valid_f_idx_start:(valid_f_idx_end + 1), :]\n\n    # Clip the spec representation and apply the mel filterbank.\n    # Due to the nature of np.dot(), the spec needs to be transposed prior, and reverted after\n    spec = np.transpose(spec[valid_f_idx_start:(valid_f_idx_end + 1), :], [1, 0])\n    spec = np.dot(spec, mel_filterbank)\n    spec = np.transpose(spec, [1, 0])        \n\n    # Magnitude transformation\n    if magnitude_scale == 'pcen':\n        \n        # Convert scale using per-channel energy normalization as proposed by Wang et al., 2017\n        # We adjust the parameters for bird voice recognition based on Lostanlen, 2019\n        spec = pcen(spec, rate, hop_len)\n        \n    elif magnitude_scale == 'log':\n        \n        # Convert power spec to dB scale (compute dB relative to peak power)\n        spec = spec ** 2\n        spec = 10.0 * np.log10(np.maximum(1e-10, spec) / np.max(spec))\n        spec = np.maximum(spec, spec.max() - 100) # top_db = 100\n\n    elif magnitude_scale == 'nonlinear':\n\n        # Convert magnitudes using nonlinearity as proposed by Schlüter, 2018\n        a = -1.2 # Higher values yield better noise suppression\n        s = 1.0 / (1.0 + np.exp(-a))\n        spec = spec ** s\n\n    # Flip spectrum vertically (only for better visialization, low freq. at bottom)\n    spec = spec[::-1, ...]\n\n    # Trim to desired shape if too large\n    spec = spec[:shape[0], :shape[1]]\n\n    # Normalize values between 0 and 1\n    spec -= spec.min()\n    if not spec.max() == 0:\n        spec /= spec.max()\n    else:\n        spec = np.clip(spec, 0, 1)\n    spec = (spec * 255).astype(np.int64)\n\n    return spec","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Create predictions\nlast_file_name = \"\" \nlast_data = []\nlast_sr = 0\ndef load_test_clip(path, start_time):\n    global last_file_name, last_data, last_sr\n    try:\n        path = path + \"*.*\"\n        #path = \"../input/birdsong-recognition/train_audio/bkhgro/XC109305.mp3\"\n        path = [file for file in glob.glob(path)][0]\n        if last_file_name != path:\n            data, sr = librosa.load(path, sr=48000, mono=True)\n            last_file_name = path\n            last_data = data\n            last_sr = sr\n        start = int(start_time*last_sr)\n        end = start + int(2*last_sr)\n        data = last_data[start:end]\n        print(data[0:10])\n        return data, last_sr\n    except Exception as e:\n        print(\"Exception:\", e)\n        return None, 0\n    \n\ndic_model = {}\ndef make_prediction(block, sr):\n    test_feature_data = get_spectrogram(block, sr)\n    test_feature_data = test_feature_data.reshape(1,img_sz1, img_sz2, 1)\n    list_pred = []\n    \n    for i in range(KFOLD_SPLITS):\n        if i not in dic_model.keys():\n            model, _, _, _ = buildNet(i)\n            dic_model[i] = model\n        else:\n            model = dic_model[i]\n        list_pred.append(model.predict(test_feature_data))\n    \n        \n    pred = np.stack(list_pred, axis=1).mean(axis=1)\n    return pred[0]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TEST_FOLDER = '../input/birdsong-recognition/test_audio/'\nif not os.path.exists(TEST_FOLDER):\n    TEST_FOLDER = \"../input/birdsong-recognition/example_test_audio/\"\n    test_info = pd.read_csv('../input/birdsong-recognition/example_test_audio_summary.csv').tail(50)\n    test_info[\"site\"] = \"site_1\"\n    test_info.rename(columns={\"filename_seconds\":\"row_id\",\"filename\":\"audio_id\"}, inplace=True)\nelse:\n    test_info = pd.read_csv('../input/birdsong-recognition/test.csv')\ntest_info.head(50)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_clip_pred(path, start_time):\n    sound_clip, sr = load_test_clip(TEST_FOLDER + audio_id , start_time)\n    if sr == 0:\n        pred = np.zeros((264))\n        print(\"Zero:\", pred.shape)\n    else:\n        pred = make_prediction(sound_clip, sr)\n        pred_class = np.argmax(pred)\n        pred_prob = pred[pred_class]\n        pred = pred * 0\n        if pred_prob > 0.5:\n            pred[pred_class] = 1\n    return pred\n        \ndef get_file_pred(path, start_time, duration):\n    if duration is None:\n        duration = get_file_duration(path)\n    list_pred = []\n    start_time = 0\n    while start_time <= duration-2:\n        pred = get_clip_pred(path, start_time)\n        list_pred.append(pred)\n        start_time = start_time + 1.5\n    pred = np.stack(list_pred, axis=1).sum(axis=1)\n    return pred\n\n\ndef get_file_duration(path):\n    #if not os.path.exists(TEST_FOLDER):\n    #    path = base_dir + \"train_audio/aldfly/XC134874.mp3\"\n    data, sr = librosa.load(path, mono=True)\n    duration = len(data)//sr\n    return duration","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#try:\nif 1==1:\n    preds = []\n    for index, row in test_info.iterrows():\n        site = row['site']\n        start_time = row['seconds'] - 5\n        row_id = row['row_id']\n        audio_id = row['audio_id']\n        path = TEST_FOLDER + audio_id + '.mp3'\n       \n        if (site == 'site_1' or site == 'site_2'):\n            pred = get_file_pred(path, start_time, 5)\n        else:\n            pred = get_file_pred(path, 0, None)\n\n        preds.append([row_id, pred])\n\n    preds = pd.DataFrame(preds, columns=['row_id', 'pred'])\n    preds[\"pred2\"] = preds[\"pred\"].map(lambda x: [i for i in range(x.shape[0]) if x[i]>0])\n    preds[\"birds\"] = preds[\"pred2\"].map(lambda x: \" \".join(list(np.sort([dic_ebird_code_rev[i] for i in x]))))\n    preds[\"birds\"] = preds[\"birds\"].map(lambda x: \"nocall\" if x==\"\" else x)\n\n    preds[[\"row_id\",\"birds\"]].to_csv('submission.csv', index=False)\n#except Exception as e:\nif 1==2:\n    e = None\n    print(\"exception\",e)\n    preds = pd.read_csv('../input/birdsong-recognition/sample_submission.csv')\n    preds[[\"row_id\",\"birds\"]].head(1).to_csv('submission.csv', index=False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds.head(50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds.tail(50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}