{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"#References\n#https://www.groundai.com/project/environment-sound-classification-using-multiple-feature-channels-and-deep-convolutional-neural-networks/1\n#https://keunwoochoi.wordpress.com/2019/09/28/log-melspectrogram-layer-using-tensorflow-keras/\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Import Required Libraries","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os     \nimport seaborn as sns\n\nimport warnings\nwarnings.filterwarnings('ignore')\nimport os\nimport librosa\nimport librosa.display\nimport numpy as np\n\nimport matplotlib.pyplot as plt\nfrom matplotlib.pyplot import specgram\nfrom sklearn.model_selection import KFold\n\nimport tensorflow as tf\nfrom tensorflow import keras\n#!pip install python_speech_features\n%matplotlib inline\nplt.style.use('ggplot')\nimport glob\nimport glob\nimport librosa\nfrom librosa import feature\nimport numpy as np\nfrom pathlib import Path\nimport cv2\nAUTO = tf.data.experimental.AUTOTUNE\nfrom kaggle_datasets import KaggleDatasets\nimport scipy\nimport pickle\nfrom sklearn.model_selection import train_test_split\nimport time","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Detect Hardware and accordingly set strategy","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ndef get_strategy():\n    gpu = \"\"\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())     \n    except ValueError:\n        tpu = None\n        os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n        gpu = tf.config.list_physical_devices(\"GPU\")\n        if len(gpu) == 1:\n            print('Running on GPU ', gpu)\n    if tpu:\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        GCS_PATH = KaggleDatasets().get_gcs_path('birdsong-recognition')\n    elif len(gpu) == 1:\n        strategy = tf.distribute.OneDeviceStrategy(device=\"/gpu:0\")\n        tf.config.optimizer.set_experimental_options({\"auto_mixed_precision\":True})\n        GCS_PATH = \"/kaggle/input/birdsong-recognition/\"\n    else:\n        strategy = tf.distribute.get_strategy()\n        GCS_PATH = \"/kaggle/input/birdsong-recognition/\"\n\n    print(\"REPLICAS: \", strategy.num_replicas_in_sync)\n    base_dir = \"../input/birdsong-recognition/\"\n    print(base_dir)\n    return strategy, GCS_PATH, base_dir\n\nstrategy,GCS_PATH, base_dir = get_strategy()\nsns.set_palette(\"pastel\")\npalette = sns.color_palette()\nCACHE = {}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Spectrogram Features","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"num_train_data_per_class = 1\nn_fft1 = int(0.0025 * 22050)\nhop_length1 = int(0.001 * 22050)\n\nn_fft2 = int(0.005 * 22050)\nhop_length2 = int(0.0025 * 22050)\n\nn_fft3 = int(0.01 * 22050)\nhop_length3 = int(0.005 * 22050)\nn_mels = 128\nfmin=150\nfmax=15000\nimg_sz1 = 64\nimg_sz2 = 512\n\ndef load_test_clip(path, start_time, duration=5):\n    try:\n        data, sr = librosa.load(path, offset=start_time, duration=duration, sr=48000, mono=True)\n        return data, sr\n    except Exception as e:\n        print(\"Exception:\", e)\n        return None, 0\n\n    \ndef buildBandpassFilter(rate, fmin, fmax, order=4):\n\n    global CACHE\n\n    fname = 'bandpass_' + str(rate) + '_' + str(fmin) + '_' + str(fmax)\n    if not fname in CACHE:\n        wn = np.array([fmin, fmax]) / (rate / 2.0)\n        filter_sos = scipy.signal.butter(order, wn, btype='bandpass', output='sos')\n        # Save to cache\n        CACHE[fname] = filter_sos\n\n    return CACHE[fname]\n\ndef applyBandpassFilter(sig, rate, fmin, fmax):\n    # Build filter or load from cache\n    filter_sos = buildBandpassFilter(rate, fmin, fmax)\n\n    return scipy.signal.sosfiltfilt(filter_sos, sig)\n\ndef get_mel_filterbanks(num_banks, fmin, fmax, f_vec, dtype=np.float32):\n    '''\n    An arguably better version of librosa's melfilterbanks wherein issues with \"hard snapping\" are avoided. Works with\n    an existing vector of frequency bins, as returned from signal.spectrogram(), instead of recalculating them and\n    flooring down the bin indices.\n    '''\n\n    global CACHE\n\n    # Filterbank already in cache?\n    fname = 'mel_' + str(num_banks) + '_' + str(fmin) + '_' + str(fmax)\n    if not fname in CACHE:\n        \n        # Break frequency and scaling factor\n        A = 4581.0\n        f_break = 1750.0\n\n        # Convert Hz to mel\n        freq_extents_mel = A * np.log10(1 + np.asarray([fmin, fmax], dtype=dtype) / f_break)\n\n        # Compute points evenly spaced in mels\n        melpoints = np.linspace(freq_extents_mel[0], freq_extents_mel[1], num_banks + 2, dtype=dtype)\n\n        # Convert mels to Hz\n        banks_ends = (f_break * (10 ** (melpoints / A) - 1))\n\n        filterbank = np.zeros([len(f_vec), num_banks], dtype=dtype)\n        for bank_idx in range(1, num_banks+1):\n            # Points in the first half of the triangle\n            mask = np.logical_and(f_vec >= banks_ends[bank_idx - 1], f_vec <= banks_ends[bank_idx])\n            filterbank[mask, bank_idx-1] = (f_vec[mask] - banks_ends[bank_idx - 1]) / \\\n                (banks_ends[bank_idx] - banks_ends[bank_idx - 1])\n\n            # Points in the second half of the triangle\n            mask = np.logical_and(f_vec >= banks_ends[bank_idx], f_vec <= banks_ends[bank_idx+1])\n            filterbank[mask, bank_idx-1] = (banks_ends[bank_idx + 1] - f_vec[mask]) / \\\n                (banks_ends[bank_idx + 1] - banks_ends[bank_idx])\n\n        # Scale and normalize, so that all the triangles do not have same height and the gain gets adjusted appropriately.\n        temp = filterbank.sum(axis=0)\n        non_zero_mask = temp > 0\n        filterbank[:, non_zero_mask] /= np.expand_dims(temp[non_zero_mask], 0)\n\n        # Save to cache\n        CACHE[fname] = (filterbank, banks_ends[1:-1])\n\n    return CACHE[fname][0], CACHE[fname][1]\n\ndef get_spectrogram(sig, rate, shape=(img_sz1, img_sz2), win_len=512, fmin=150, fmax=15000, magnitude_scale='nonlinear', bandpass=True, decompose=False):\n\n    # Compute overlap\n    hop_len = int(len(sig) / (shape[1] - 1)) \n    win_overlap = win_len - hop_len + 2\n    #print 'WIN_LEN:', win_len, 'HOP_LEN:', hop_len, 'OVERLAP:', win_overlap\n\n    \n    n_fft = win_len\n    \n\n    # Bandpass filter?\n    if bandpass:\n        sig = applyBandpassFilter(sig, rate, fmin, fmax)\n\n    # Compute spectrogram\n    f, t, spec = scipy.signal.spectrogram(sig,\n                                          fs=rate,\n                                          window=scipy.signal.windows.hann(win_len),\n                                          nperseg=win_len,\n                                          noverlap=win_overlap,\n                                          nfft=n_fft,\n                                          detrend=False,\n                                          mode='magnitude')\n\n    # Scale frequency?\n   \n\n    # Determine the indices of where to clip the spec\n    valid_f_idx_start = f.searchsorted(fmin, side='left')\n    valid_f_idx_end = f.searchsorted(fmax, side='right') - 1\n\n    # Get mel filter banks\n    mel_filterbank, mel_f = get_mel_filterbanks(shape[0], fmin, fmax, f, dtype=spec.dtype)\n\n    # Clip to non-zero range so that unnecessary multiplications can be avoided\n    mel_filterbank = mel_filterbank[valid_f_idx_start:(valid_f_idx_end + 1), :]\n\n    # Clip the spec representation and apply the mel filterbank.\n    # Due to the nature of np.dot(), the spec needs to be transposed prior, and reverted after\n    spec = np.transpose(spec[valid_f_idx_start:(valid_f_idx_end + 1), :], [1, 0])\n    spec = np.dot(spec, mel_filterbank)\n    spec = np.transpose(spec, [1, 0])        \n\n    # Magnitude transformation\n    if magnitude_scale == 'pcen':\n        \n        # Convert scale using per-channel energy normalization as proposed by Wang et al., 2017\n        # We adjust the parameters for bird voice recognition based on Lostanlen, 2019\n        spec = pcen(spec, rate, hop_len)\n        \n    elif magnitude_scale == 'log':\n        \n        # Convert power spec to dB scale (compute dB relative to peak power)\n        spec = spec ** 2\n        spec = 10.0 * np.log10(np.maximum(1e-10, spec) / np.max(spec))\n        spec = np.maximum(spec, spec.max() - 100) # top_db = 100\n\n    elif magnitude_scale == 'nonlinear':\n\n        # Convert magnitudes using nonlinearity as proposed by Schlüter, 2018\n        a = -1.2 # Higher values yield better noise suppression\n        s = 1.0 / (1.0 + np.exp(-a))\n        spec = spec ** s\n\n    # Flip spectrum vertically (only for better visialization, low freq. at bottom)\n    spec = spec[::-1, ...]\n\n    # Trim to desired shape if too large\n    spec = spec[:shape[0], :shape[1]]\n\n    # Normalize values between 0 and 1\n    spec -= spec.min()\n    if not spec.max() == 0:\n        spec /= spec.max()\n    else:\n        spec = np.clip(spec, 0, 1)\n    spec = (spec * 255).astype(np.int64)\n\n    return spec\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_ebird_filename_dic():\n    ebird_code_list = all_train_data[\"ebird_code\"].unique()\n    if os.path.exists(\"../input/birdcall-spectrogram-tfrecords/dic_ebird.pkl\"):\n        with open(\"../input/birdcall-spectrogram-tfrecords/dic_ebird.pkl\",\"rb\") as f:\n            dic_ebird_code = pickle.load(f)\n    else:\n        dic_ebird_code = {k:v for v,k in enumerate(ebird_code_list)}\n    dic_ebird_code_rev = [v for v,k in dic_ebird_code.items()]\n    all_train_data[\"int_ebird_code\"] = all_train_data[\"ebird_code\"].map(dic_ebird_code)\n\n    filename_list = all_train_data[\"filename\"].unique()\n    if os.path.exists(\"../input/birdcall-spectrogram-tfrecords/dic_filename.pkl\"):\n        with open(\"../input/birdcall-spectrogram-tfrecords/dic_filename.pkl\",\"rb\") as f:\n            dic_filename = pickle.load(f)\n    else:\n        dic_filename = {k:v for v,k in enumerate(filename_list)}\n    dic_filename_rev = [v for v,k in dic_filename.items()]\n    all_train_data[\"int_filename\"] = all_train_data[\"filename\"].map(dic_filename)\n\n    with open(\"dic_ebird.pkl\",\"wb\") as f:\n        pickle.dump(dic_ebird_code, f)\n\n    with open(\"dic_filename.pkl\",\"wb\") as f:\n        pickle.dump(dic_filename, f)\n        \n    return dic_ebird_code_rev, dic_filename_rev","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_img_file():\n    list_img_data = []\n    max_duration = all_train_data[\"duration\"].max()\n    img_data = all_train_data[[\"int_ebird_code\",\"int_filename\",\"duration\"]]\n    duration = 0\n    while duration < max_duration:\n        img_data = img_data[img_data[\"duration\"] >= duration+5]\n        img_data[\"start\"] = duration\n        list_img_data.append(img_data)\n        duration = duration + 5\n\n    img_data = pd.concat(list_img_data)\n    print(img_data.shape)\n    print(all_train_data.shape)\n    return img_data\n\n\ndef split_train_data():\n    pickle_filename = \"../input/birdcall-spectrogram-tfrecords/train_dic.pkl\"\n    if not os.path.exists(pickle_filename):\n        print(\"Pickle file not found!\")\n        pickle_filename = \"train_dic.pkl\"\n    if not os.path.exists(pickle_filename):\n        print(\"Pickle file not found!\")\n        img_data = get_img_file()\n        size = img_data.shape[0]//64\n        arr_data = []\n        split_data = img_data\n        for i in range(63):\n            split_data, test_data = train_test_split(split_data, test_size=size, stratify= split_data[\"int_ebird_code\"])\n            arr_data.append(test_data)\n        arr_data.append(split_data)\n        for df in arr_data[-3:]:\n            print(df.shape)\n\n        i  = 0\n        arr_dic = []\n        for df in arr_data:\n            tfrec_path = \"train_\" + str(i)\n            arr_dic.append({\"tfrec_path\":tfrec_path, \"df\":df})\n            i = i + 1\n        with open(pickle_filename, 'wb') as file:\n            pickle.dump(arr_dic, file)\n        \n    else:\n        print(\"Pickle file found!\")\n        with open(pickle_filename, 'rb') as file:\n            arr_dic = pickle.load(file)\n        pickle_filename = \"train_dic.pkl\"\n        with open(pickle_filename, 'wb') as file:\n            pickle.dump(arr_dic, file)\n    return arr_dic","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _bytestring_feature(list_of_bytestrings):\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=list_of_bytestrings))\n\ndef _int_feature(list_of_ints): # int64\n  return tf.train.Feature(int64_list=tf.train.Int64List(value=list_of_ints))\n\ndef _float_feature(list_of_floats): # float32\n  return tf.train.Feature(float_list=tf.train.FloatList(value=list_of_floats))\n\ndef write_all_tfrec(tfrec_path, train_data, bln_crop=False):\n    try:\n        t0 = time.process_time()\n        tfrec_full_path = \"../input/birdcall-spectrogram-tfrecords/\" + tfrec_path\n        is_valid_file = False\n        if not os.path.exists(tfrec_full_path):\n            print(tfrec_path, \": Valid file not found!\")\n            with tf.io.TFRecordWriter(tfrec_path) as out_file:\n                for idx,row in train_data.iterrows():\n                    int_ebird_code = row[\"int_ebird_code\"]\n                    int_filename = row[\"int_filename\"]\n                    filepath = base_dir + \"train_audio/\" + dic_ebird_code_rev[int_ebird_code] + \"/\" + dic_filename_rev[int_filename]\n                    start_time=row[\"start\"]\n                    clip, sr = load_test_clip(filepath, start_time)\n                    if sr > 0:\n                        img = get_spectrogram(clip, sr)\n                        #img = mel_spec.reshape(mel_spec.shape[0], mel_spec.shape[1])\n\n                    img = cv2.imencode('.jpg', img, (cv2.IMWRITE_JPEG_QUALITY, 94))[1].tostring()\n                    feature = {\n                        \"img\": _bytestring_feature([img]),\n                    }\n                    if \"train\" in tfrec_path:\n                        feature[\"int_ebird_code\"] = _int_feature([row[\"int_ebird_code\"]])\n                    tf_record = tf.train.Example(features=tf.train.Features(feature=feature))\n                    out_file.write(tf_record.SerializeToString())\n        else:\n            print(\"File exists!\")\n            os.popen('cp ' + tfrec_full_path + ' ' + tfrec_path)\n        t1 = time.process_time()\n        print(\"Process time:\", t1-t0)\n    except Exception as e:\n        print(\"Error:\", e, filepath)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_train_data = pd.read_csv(base_dir + \"train.csv\")\nall_train_data = all_train_data[all_train_data[\"duration\"]>=5]\nall_train_data = all_train_data[all_train_data[\"filename\"]!=\"XC195038.mp3\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dic_ebird_code_rev, dic_filename_rev = get_ebird_filename_dic()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"arr_dic = split_train_data()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for param in arr_dic:\n    write_all_tfrec(param[\"tfrec_path\"], param[\"df\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}