{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import copy\nimport gc\nimport glob\nimport os\nimport time\n\nimport cv2\nimport IPython\nimport IPython.display\nimport librosa\nimport librosa.display\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport tensorflow.keras as keras\nfrom joblib import Parallel, delayed\nfrom sklearn.model_selection import KFold, StratifiedKFold, train_test_split\nfrom tensorflow.keras.applications.resnet50 import ResNet50\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.models import Model\nfrom tqdm import tqdm\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.options.display.max_columns = 128\npd.options.display.max_rows = 128\nplt.rcParams['figure.figsize'] = (15, 8)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Configuration and global parameters:"},{"metadata":{"trusted":true},"cell_type":"code","source":"class EasyDict(dict):\n    def __init__(self, d=None, **kwargs):\n        if d is None:\n            d = {}\n        if kwargs:\n            d.update(**kwargs)\n        for k, v in d.items():\n            setattr(self, k, v)\n        # Class attributes\n        for k in self.__class__.__dict__.keys():\n            if not (k.startswith('__') and k.endswith('__')) and not k in ('update', 'pop'):\n                setattr(self, k, getattr(self, k))\n\n    def __setattr__(self, name, value):\n        if isinstance(value, (list, tuple)):\n            value = [self.__class__(x)\n                     if isinstance(x, dict) else x for x in value]\n        elif isinstance(value, dict) and not isinstance(value, self.__class__):\n            value = self.__class__(value)\n        super(EasyDict, self).__setattr__(name, value)\n        super(EasyDict, self).__setitem__(name, value)\n\n    __setitem__ = __setattr__\n\n    def update(self, e=None, **f):\n        d = e or dict()\n        d.update(f)\n        for k in d:\n            setattr(self, k, d[k])\n\n    def pop(self, k, d=None):\n        delattr(self, k)\n        return super(EasyDict, self).pop(k, d)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/train_curated.csv')\nsample_submission = pd.read_csv('../input/sample_submission.csv')\nprint('train: {}'.format(train_df.shape))\nprint('test: {}'.format(sample_submission.shape))\n\nROOT = '../input/'\ntest_root = os.path.join(ROOT, 'test/')\ntrain_root = os.path.join(ROOT, 'train_curated/')\n\n\nCONFIG = EasyDict()\nCONFIG.hop_length = 347 # to make time steps 128\nCONFIG.fmin = 20\nCONFIG.fmax = 44100 / 2\nCONFIG.n_fft = 480\n\nN_SAMPLES = 48\nSAMPLE_DIM = 256\n\nTRAINING_CONFIG = {\n    'sample_dim': (N_SAMPLES, SAMPLE_DIM),\n    'padding_mode': cv2.BORDER_REFLECT,\n}\n\nprint(CONFIG)\nprint(TRAINING_CONFIG)\n\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Processing class:"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Preprocessing functions inspired by:\n# https://github.com/xiaozhouwang/tensorflow_speech_recognition_solution/blob/master/data.py\nclass DataProcessor(object):\n    \n    def __init__(self, debug=False):\n        self.debug = debug\n        \n        # Placeholders for global statistics\n        self.mel_mean = None\n        self.mel_std = None\n        self.mel_max = None\n        self.mfcc_max = None\n        \n    def createMel(self, filename, params, normalize=False):\n        \"\"\"\n        Create Mel Spectrogram sample out of raw wavfile\n        \"\"\"\n        y, sr = librosa.load(filename, sr=None)\n        mel = librosa.feature.melspectrogram(y, sr, n_mels=N_SAMPLES, **params)\n        mel = librosa.power_to_db(mel)\n        if normalize:\n            if self.mel_mean is not None and self.mel_std is not None:\n                mel = (mel - self.mel_mean) / self.mel_std\n            else:\n                sample_mean = np.mean(mel)\n                sample_std = np.std(mel)\n                mel = (mel - sample_mean) / sample_std\n            if self.mel_max is not None:\n                mel = mel / self.mel_max\n            else:\n                mel = mel / np.max(np.abs(mel))\n        return mel\n    \n    def createMfcc(self, filename, params, normalize=False):\n        \"\"\"\n        Create MFCC sample out of raw wavfile\n        \"\"\"\n        y, sr = librosa.load(filename, sr=None)\n        nonzero_idx = [y > 0]\n        y[nonzero_idx] = np.log(y[nonzero_idx])\n        mfcc = librosa.feature.mfcc(y, sr, n_mfcc=N_SAMPLES, **params)\n        if normalize:\n            if self.mfcc_max is not None:\n                mfcc = mfcc / self.mfcc_max\n            else:\n                mfcc = mfcc / np.max(np.abs(mfcc))\n        return mfcc\n    \n    def prepareSample(self, root, row, \n                      preprocFunc, \n                      preprocParams, trainingParams, \n                      test_mode=False, normalize=False, \n                      proc_mode='split'):\n        \"\"\"\n        Prepare sample for model training.\n        Function takes row of DataFrame, extracts filename and labels and processes them.\n        \n        If proc_mode is 'split':\n        Outputs sets of arrays of constant shape padded to TRAINING_CONFIG shape\n        with selected padding mode, also specified in TRAINING_CONFIG.\n        This approach prevents loss of information caused by trimming the audio sample,\n        instead it splits it into equally-sized parts and pads them.\n        To account for creation of multiple samples, number of labels are multiplied to a number\n        equal to number of created samples.\n        \n        If proc_mode is 'resize':\n        Resizes the original processed sample to (SAMPLE_DIM, N_SAMPLES) shape.\n        \"\"\"\n        \n        assert proc_mode in ['split', 'resize'], 'proc_must be one of split or resize'\n        \n        filename = os.path.join(root, row['fname'])\n        if not test_mode:\n            labels = row['labels']\n            \n        sample = preprocFunc(filename, preprocParams, normalize=normalize)\n        # print(sample.min(), sample.max())\n        \n        if proc_mode == 'split':\n            sample_split = np.array_split(\n                sample, np.ceil(sample.shape[1] / SAMPLE_DIM), axis=1)\n            samples_pad = []\n            for i in sample_split:\n                padding_dim = SAMPLE_DIM - i.shape[1]\n                sample_pad = cv2.copyMakeBorder(i, 0, 0, 0, padding_dim, trainingParams['padding_mode'])\n                samples_pad.append(sample_pad)\n            samples_pad = np.asarray(samples_pad)\n            if not test_mode:\n                labels = [labels] * len(samples_pad)\n                labels = np.asarray(labels)\n                return samples_pad, labels\n            return samples_pad\n        elif proc_mode == 'resize':\n            sample_pad = cv2.resize(sample, (SAMPLE_DIM, N_SAMPLES), interpolation=cv2.INTER_NEAREST)\n            sample_pad = np.expand_dims(sample_pad, axis=0)\n            if not test_mode:\n                labels = np.asarray(labels)\n                return sample_pad, labels\n            return sample_pad\n        \n    \nprocessor = DataProcessor()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Inspection of created samples, without normalization:"},{"metadata":{"trusted":true},"cell_type":"code","source":"FILENAME = train_root + train_df.fname[5]\n\n\nsample_mel = processor.createMel(FILENAME, CONFIG)\nsample_mfcc = processor.createMfcc(FILENAME, CONFIG)\nprint(sample_mel.shape)\nprint(sample_mfcc.shape)\n\nidx_cut = 400\nplt.imshow(sample_mel[:, :idx_cut], cmap='Spectral')\nplt.title('Mel Spectrogram:')\nplt.show()\nplt.imshow(sample_mfcc[:, :idx_cut], cmap='Spectral')\nplt.title('MFCC:')\nplt.show()\n\n# Mel sample range:\nprint(np.min(sample_mel), np.max(sample_mel))\n# MFCC sample range:\nprint(np.min(sample_mfcc), np.max(sample_mfcc))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Inspection of created samples, with normalization:"},{"metadata":{"trusted":true},"cell_type":"code","source":"FILENAME  = train_root + train_df.fname[5]\nNORMALIZE = True\n\nsample_mel = processor.createMel(FILENAME, CONFIG, normalize=NORMALIZE)\nsample_mfcc = processor.createMfcc(FILENAME, CONFIG, normalize=NORMALIZE)\nprint(sample_mel.shape)\nprint(sample_mfcc.shape)\n\nidx_cut = 400\nplt.imshow(sample_mel[:, :idx_cut], cmap='Spectral')\nplt.title('Mel Spectrogram:')\nplt.show()\nplt.imshow(sample_mfcc[:, :idx_cut], cmap='Spectral')\nplt.title('MFCC:')\nplt.show()\n\n# Mel sample range:\nprint(np.min(sample_mel), np.max(sample_mel))\n# MFCC sample range:\nprint(np.min(sample_mfcc), np.max(sample_mfcc))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Visualize processed samples, split:"},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_idxs = np.random.choice(np.arange(0, len(train_df)), 5)\n\n\nfor i in sample_idxs:\n    sample_prep, labels = processor.prepareSample(\n        train_root,  # training set directory\n        train_df.iloc[i, :],  # sample index from train_df \n        processor.createMel,  # function for data processing\n        CONFIG, TRAINING_CONFIG,  # parameters for processing and training\n        test_mode=False,\n        proc_mode='split',  # indicate split into N sub-arrays\n    )  # whether to labels are available\n    print(sample_prep.max(), sample_prep.min())\n    print(sample_prep.shape, labels.shape)\n    NCOLS = 2\n    NROWS = int(np.ceil(sample_prep.shape[0] / NCOLS))\n    fig, ax = plt.subplots(NCOLS, NROWS, figsize=(20, 5))\n    fig.suptitle('Sample: {}'.format(i))\n    idx = 0\n    for c in range(NCOLS):\n        if NROWS > 1:\n            for r in range(NROWS):\n                if idx < sample_prep.shape[0]:\n                    ax[c, r].imshow(sample_prep[idx], cmap='Spectral')\n                    ax[c, r].set_title('class: {}'.format(labels[idx]))\n                    idx += 1\n        else:\n            if idx < sample_prep.shape[0]:\n                ax[c].imshow(sample_prep[idx], cmap='Spectral')\n                ax[c].set_title('class: {}'.format(labels[idx]))\n                idx += 1\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Visualize processed samples, resize:"},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_idxs = np.random.choice(np.arange(0, len(train_df)), 5)\n\n\nfor i in sample_idxs:\n    sample_prep, labels = processor.prepareSample(\n        train_root,  # training set directory\n        train_df.iloc[i, :],  # sample index from train_df \n        processor.createMel,  # function for data processing\n        CONFIG, TRAINING_CONFIG,  # parameters for processing and training\n        test_mode=False,\n        proc_mode='resize'  # resize mode\n    )  # whether to labels are available\n    print(sample_prep.max(), sample_prep.min())\n    print(sample_prep.shape, labels.shape)\n    NCOLS = 1\n    NROWS = 1\n    fig, ax = plt.subplots(NCOLS, NROWS, figsize=(10, 3))\n    fig.suptitle('Sample: {}, class: {}'.format(i, labels))\n    ax.imshow(sample_prep[0], cmap='Spectral')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Visualize test processed samples, resize:"},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_idxs = np.random.choice(np.arange(0, len(sample_submission)), 5)\n\n\nfor i in sample_idxs:\n    sample_prep = processor.prepareSample(\n        test_root,  # test set directory\n        sample_submission.iloc[i, :],  # sample index from sample_submission \n        processor.createMel,  # function for data processing\n        CONFIG, TRAINING_CONFIG,  # parameters for processing and training\n        test_mode=True,  # indicate test mode\n        proc_mode='resize'\n    )  # whether to labels are available\n    print(sample_prep.max(), sample_prep.min())\n    print(sample_prep.shape, labels.shape)\n    NCOLS = 1\n    NROWS = int(np.ceil(sample_prep.shape[0] / NCOLS))\n    fig, ax = plt.subplots(NCOLS, NROWS, figsize=(10, 3))\n    fig.suptitle('Test Sample: {}'.format(i))\n    ax.imshow(sample_prep[0], cmap='Spectral')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Process train data in parallel:"},{"metadata":{"trusted":true},"cell_type":"code","source":"output = Parallel(n_jobs=-3, verbose=1)(\n    delayed(processor.prepareSample)(\n        train_root, \n        train_df.iloc[f, :],\n        processor.createMel,\n        CONFIG,\n        TRAINING_CONFIG,\n        test_mode=False,\n        proc_mode='resize',\n    ) for f in range(100))  # change to number of samples in train data for full processing\n\n\nX_train = np.array([x[0] for x in output])\ny_train = np.array([x[1] for x in output])\ny_train = pd.Series(y_train).str.get_dummies(sep=',')\nprint(X_train.shape, y_train.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Process test data in parallel:"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test = Parallel(n_jobs=-3, verbose=1)(\n    delayed(processor.prepareSample)(\n        test_root, \n        sample_submission.iloc[f, :],\n        processor.createMel,\n        CONFIG,\n        TRAINING_CONFIG,\n        test_mode=True,\n        proc_mode='resize',\n    ) for f in range(100))  # change to number of samples in test data for full processing\n\n\nX_test = np.array(X_test)\nprint(X_test.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Data ready!\n\nNow the data can be fed to a 2D convolutional neural network, for example one of `keras.applications` networks.\nIt can also be used to train recurrent network such as LSTM."}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.7"}},"nbformat":4,"nbformat_minor":1}