{"cells":[{"metadata":{},"cell_type":"markdown","source":"I think It's interesting to be able to solve the problem of audio recognition, beyond just voice. For the demo-day there's potential to do a live-demo recongizing sounds in real-time. (keys, coughing, traffic...)\nFreesound has a fairly low number of teams still, fewer than 500 so I think it's more winnable that say the 'two-sigma'. I have been looking at the 2018 freesound competition, and essentially the main difference is that: there was no curated set. In the 2019 competition there's a curated set with highly accurate  labels, and another much larger set with not-so-accurate labels. This is a very  interesting hurdle which you recall Girish also mentioned. This kind of curated/uncurated sets are common in the industry. He also recommended this technique calleld \"transfer learning\" which I plan to incorporate in my solution. Another technique I heard about from Bruce Sharpe, the Vancouver Kaggle meetup organizer is what's called 'Batch Normalization'. There was an academic paper released in 2015 on Batch normalization, that showed significant improvements to the image-net classification dataset. It trained 14x faster. I'll be looking for ideas like these to improve my solution bit-by-bit throughout the remaining 6 weeks until the final submission deadline of June 10th. Another thing I thought about was training time. If anyone knows how to use AWS to train models, that would be useful info. Thank you, -keagan"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nfrom os.path import isdir, join\nfrom pathlib import Path\nimport pandas as pd\nimport wave\n\n# Math\nimport numpy as np\nfrom scipy.fftpack import fft\nfrom scipy import signal\nfrom scipy.io import wavfile\nimport librosa\nimport cv2\n\nfrom sklearn.decomposition import PCA\n\n# Visualization\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport IPython.display as ipd\nimport librosa.display\n\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\nimport plotly.tools as tls\nimport pandas as pd\n\n%matplotlib inline\n\nimport warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def log_specgram(audio, sample_rate, window_size=20,\n                 step_size=10, eps=1e-10):\n    nperseg = int(round(window_size * sample_rate / 1e3))\n    noverlap = int(round(step_size * sample_rate / 1e3))\n    freqs, times, spec = signal.spectrogram(audio,\n                                    fs=sample_rate,\n                                    window='hann',\n                                    nperseg=nperseg,\n                                    noverlap=noverlap,\n                                    detrend=False)\n    return freqs, times, np.log(spec.T.astype(np.float32) + eps)\n\ndef plot_log_specgram(audio, sample_rate, window_size=20, step_size=10, eps=1e-10):\n    \n    fig = plt.figure(figsize=(14, 3))\n    freqs, times, spectrogram = log_specgram(audio, sample_rate)\n    plt.imshow(spectrogram.T, aspect='auto', origin='lower', \n               extent=[times.min(), times.max(), freqs.min(), freqs.max()])\n    plt.yticks(freqs[::16])\n    plt.xticks(times[::16])\n    plt.title('Spectrogram')\n    plt.ylabel('Freqs in Hz')\n    plt.xlabel('Seconds')\n    plt.show()\n    \nclass EasyDict(dict):\n    def __init__(self, d=None, **kwargs):\n        if d is None:\n            d = {}\n        if kwargs:\n            d.update(**kwargs)\n        for k, v in d.items():\n            setattr(self, k, v)\n        # Class attributes\n        for k in self.__class__.__dict__.keys():\n            if not (k.startswith('__') and k.endswith('__')) and not k in ('update', 'pop'):\n                setattr(self, k, getattr(self, k))\n\n    def __setattr__(self, name, value):\n        if isinstance(value, (list, tuple)):\n            value = [self.__class__(x)\n                     if isinstance(x, dict) else x for x in value]\n        elif isinstance(value, dict) and not isinstance(value, self.__class__):\n            value = self.__class__(value)\n        super(EasyDict, self).__setattr__(name, value)\n        super(EasyDict, self).__setitem__(name, value)\n\n    __setitem__ = __setattr__\n\n    def update(self, e=None, **f):\n        d = e or dict()\n        d.update(f)\n        for k in d:\n            setattr(self, k, d[k])\n\n    def pop(self, k, d=None):\n        delattr(self, k)\n        return super(EasyDict, self).pop(k, d)\n    \ndef custom_fft(y, fs):\n    T = 1.0 / fs\n    N = y.shape[0]\n    yf = fft(y)\n    xf = np.linspace(0.0, 1.0/(2.0*T), N//2)\n    vals = 2.0/N * np.abs(yf[0:N//2])  # FFT is simmetrical, so we take just the first half\n    # FFT is also complex, to we take just the real part (abs)\n    return xf, vals\n\ndef plot_custom_fft(samples, sample_rate):\n    xf, vals = custom_fft(samples, sample_rate)\n    plt.figure(figsize=(12, 4))\n    plt.title('FFT of recording sampled with ' + str(sample_rate) + ' Hz')\n    plt.plot(xf, vals)\n    plt.xlabel('Frequency')\n    plt.grid()\n    plt.show()\n    \ndef plot_raw_wave(samples):\n    plt.figure(figsize=(14, 3))\n    plt.title('Raw wave')\n    plt.ylabel('Amplitude')\n    # ax1.plot(np.linspace(0, sample_rate/len(samples1), sample_rate), samples1)\n    plt.plot(samples)\n    plt.show()\n    \n\ntrain_df = pd.read_csv('../input/train_curated.csv')\nsample_submission = pd.read_csv('../input/sample_submission.csv')\n# print('train: {}'.format(train_df.shape))\n# print('test: {}'.format(sample_submission.shape))\n\nROOT = '../input/'\ntest_root = os.path.join(ROOT, 'test/')\ntrain_root = os.path.join(ROOT, 'train_curated/')\n\n\nCONFIG = EasyDict()\nCONFIG.hop_length = 347 # to make time steps 128\nCONFIG.fmin = 20\nCONFIG.fmax = 44100 / 2\nCONFIG.n_fft = 480\n\nN_SAMPLES = 48\nSAMPLE_DIM = 256\n\nTRAINING_CONFIG = {\n    'sample_dim': (N_SAMPLES, SAMPLE_DIM),\n    'padding_mode': cv2.BORDER_REFLECT,\n}\n\n# print(CONFIG)\n# print(TRAINING_CONFIG)\n\ntrain_df.head()\n    \nclass DataProcessor(object):\n    \n    def __init__(self, debug=False):\n        self.debug = debug\n        \n        # Placeholders for global statistics\n        self.mel_mean = None\n        self.mel_std = None\n        self.mel_max = None\n        self.mfcc_max = None\n        \n    def createMel(self, filename, params, normalize=False):\n        \"\"\"\n        Create Mel Spectrogram sample out of raw wavfile\n        \"\"\"\n        y, sr = librosa.load(filename, sr=None)\n        mel = librosa.feature.melspectrogram(y, sr, n_mels=N_SAMPLES, **params)\n        mel = librosa.power_to_db(mel)\n        if normalize:\n            if self.mel_mean is not None and self.mel_std is not None:\n                mel = (mel - self.mel_mean) / self.mel_std\n            else:\n                sample_mean = np.mean(mel)\n                sample_std = np.std(mel)\n                mel = (mel - sample_mean) / sample_std\n            if self.mel_max is not None:\n                mel = mel / self.mel_max\n            else:\n                mel = mel / np.max(np.abs(mel))\n        return mel\n    \n    def createMfcc(self, filename, params, normalize=False):\n        \"\"\"\n        Create MFCC sample out of raw wavfile\n        \"\"\"\n        y, sr = librosa.load(filename, sr=None)\n        nonzero_idx = [y > 0]\n        y[nonzero_idx] = np.log(y[nonzero_idx])\n        mfcc = librosa.feature.mfcc(y, sr, n_mfcc=N_SAMPLES, **params)\n        if normalize:\n            if self.mfcc_max is not None:\n                mfcc = mfcc / self.mfcc_max\n            else:\n                mfcc = mfcc / np.max(np.abs(mfcc))\n        return mfcc\n    \n    def prepareSample(self, root, row, \n                      preprocFunc, \n                      preprocParams, trainingParams, \n                      test_mode=False, normalize=False, \n                      proc_mode='split'):\n        \"\"\"\n        Prepare sample for model training.\n        Function takes row of DataFrame, extracts filename and labels and processes them.\n        \n        If proc_mode is 'split':\n        Outputs sets of arrays of constant shape padded to TRAINING_CONFIG shape\n        with selected padding mode, also specified in TRAINING_CONFIG.\n        This approach prevents loss of information caused by trimming the audio sample,\n        instead it splits it into equally-sized parts and pads them.\n        To account for creation of multiple samples, number of labels are multiplied to a number\n        equal to number of created samples.\n        \n        If proc_mode is 'resize':\n        Resizes the original processed sample to (SAMPLE_DIM, N_SAMPLES) shape.\n        \"\"\"\n        \n        assert proc_mode in ['split', 'resize'], 'proc_must be one of split or resize'\n        \n        filename = os.path.join(root, row['fname'])\n        if not test_mode:\n            labels = row['labels']\n            \n        sample = preprocFunc(filename, preprocParams, normalize=normalize)\n        # print(sample.min(), sample.max())\n        \n        if proc_mode == 'split':\n            sample_split = np.array_split(\n                sample, np.ceil(sample.shape[1] / SAMPLE_DIM), axis=1)\n            samples_pad = []\n            for i in sample_split:\n                padding_dim = SAMPLE_DIM - i.shape[1]\n                sample_pad = cv2.copyMakeBorder(i, 0, 0, 0, padding_dim, trainingParams['padding_mode'])\n                samples_pad.append(sample_pad)\n            samples_pad = np.asarray(samples_pad)\n            if not test_mode:\n                labels = [labels] * len(samples_pad)\n                labels = np.asarray(labels)\n                return samples_pad, labels\n            return samples_pad\n        elif proc_mode == 'resize':\n            sample_pad = cv2.resize(sample, (SAMPLE_DIM, N_SAMPLES), interpolation=cv2.INTER_NEAREST)\n            sample_pad = np.expand_dims(sample_pad, axis=0)\n            if not test_mode:\n                labels = np.asarray(labels)\n                return sample_pad, labels\n            return sample_pad\n        \n    \nprocessor = DataProcessor()\n\ndef spec_augment(spec: np.ndarray, num_mask=2, \n                 freq_masking_max_percentage=0.15, time_masking_max_percentage=0.3):\n\n    spec = spec.copy()\n    for i in range(num_mask):\n        all_frames_num, all_freqs_num = spec.shape\n        freq_percentage = random.uniform(0.0, freq_masking_max_percentage)\n        \n        num_freqs_to_mask = int(freq_percentage * all_freqs_num)\n        f0 = np.random.uniform(low=0.0, high=all_freqs_num - num_freqs_to_mask)\n        f0 = int(f0)\n        spec[:, f0:f0 + num_freqs_to_mask] = 0\n\n        time_percentage = random.uniform(0.0, time_masking_max_percentage)\n        \n        num_frames_to_mask = int(time_percentage * all_frames_num)\n        t0 = np.random.uniform(low=0.0, high=all_frames_num - num_frames_to_mask)\n        t0 = int(t0)\n        spec[t0:t0 + num_frames_to_mask, :] = 0\n    \n    return spec\n    \naudio_path = os.path.join(\"../input/train_curated/d7d25898.wav\")\nsr, audio = wavfile.read(audio_path)\n\nx = librosa.feature.melspectrogram(y=audio.astype(float), sr=sr, S=None, n_fft=512, hop_length=256, n_mels=40).T\nx = librosa.power_to_db(x, ref=np.max)\n\ntrain = pd.read_csv(\"../input/train_curated.csv\")\ntrain_new = train.sort_values('labels').reset_index()\ntrain_new['nframes'] = train_new['fname'].apply(lambda f: wave.open('../input/train_curated/' + f).getnframes())\n\ntrain_fname = train_new.head(1000)\npath = \"../input/train_curated/\"\n\nFILENAME  = train_root + train_df.fname[5]\nNORMALIZE = True\n\nsample_mel = processor.createMel(FILENAME, CONFIG, normalize=NORMALIZE)\nsample_mfcc = processor.createMfcc(FILENAME, CONFIG, normalize=NORMALIZE)\n# print(sample_mel.shape)\n# print(sample_mfcc.shape)\n\nsample_idxs = np.random.choice(np.arange(0, len(train_df)), 5)\n\ntrain_audio_path = '../input/train_curated/' \n# 8a8110c2 c2aff189 d7d25898 0a2895b8 6459fc05 54940c5c 024e0fbe c6f8f09e f46cc65b  \n# 1acaf122 a0a85eae da3a5cd5 412c28dd 0f301184 2ce5262c\nsample_rate, samples1 = wavfile.read(os.path.join(train_audio_path, '98b0df76.wav'))\nsample_rate, samples2 = wavfile.read(os.path.join(train_audio_path, 'd7d25898.wav'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ipd.Audio(samples1, rate=sample_rate)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_raw_wave(samples1)\nplot_raw_wave(samples2)\nipd.Audio(samples2, rate=sample_rate)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(figsize=(16,5))\ntrain_new.nframes.hist(bins=100)\nplt.suptitle('Frame Length Distribution in Train Curated', ha='center', fontsize='large');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in sample_idxs:\n    sample_prep, labels = processor.prepareSample(\n        train_root, \n        train_df.iloc[i, :], \n        processor.createMel,  \n        CONFIG, TRAINING_CONFIG,  \n        test_mode=False,\n        proc_mode='split',  \n    )  \n    NCOLS = 2\n    NROWS = int(np.ceil(sample_prep.shape[0] / NCOLS))\n    fig, ax = plt.subplots(NCOLS, NROWS, figsize=(20, 5))\n    fig.suptitle('Sample: {}'.format(i))\n    idx = 0\n    for c in range(NCOLS):\n        if NROWS > 1:\n            for r in range(NROWS):\n                if idx < sample_prep.shape[0]:\n                    ax[c, r].imshow(sample_prep[idx], cmap='Spectral')\n                    ax[c, r].set_title('class: {}'.format(labels[idx]))\n                    idx += 1\n        else:\n            if idx < sample_prep.shape[0]:\n                ax[c].imshow(sample_prep[idx], cmap='Spectral')\n                ax[c].set_title('class: {}'.format(labels[idx]))\n                idx += 1\n    plt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"S = librosa.feature.melspectrogram(samples1.astype(float), sr=sample_rate, n_mels=128)\n\n# Convert to log scale (dB). We'll use the peak power (max) as reference.\nlog_S = librosa.power_to_db(S, ref=np.max)\n\nplt.figure(figsize=(12, 4))\nlibrosa.display.specshow(log_S, sr=sample_rate, x_axis='time', y_axis='mel')\nplt.title('Mel power spectrogram ')\nplt.colorbar(format='%+02.0f dB')\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"freqs, times, spectrogram = log_specgram(samples2, sample_rate)\ndata = [go.Surface(z=spectrogram.T)]\nlayout = go.Layout(\n    title='Specgtogram 3d',\n    \n#     scene = dict(\n#     yaxis = dict(title='Frequencies', range=freqs),\n#     xaxis = dict(title='Time', range=times),\n#     zaxis = dict(title='Log amplitude'),\n#     )\n)\n\nfig = go.Figure(data=data, layout=layout)\npy.iplot(fig)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}