{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Imports\nimport pickle\nimport keras\nimport random\nimport scipy\n\nfrom pathlib import Path\nfrom subprocess import check_output\n\nfrom scipy import signal\nfrom scipy.io import wavfile\nimport matplotlib.pyplot as plt\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e352708b7d24e77f30b403193b89b979fec4d339"},"cell_type":"code","source":"train_audio_path = '../input/tensorflow-speech-recognition-challenge/train/audio'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2379a7b4c6abf41c4efee0796cf8899c64739caa"},"cell_type":"code","source":"with open('../input/speech-recognition-data-processing/SavedPartition.pickle', 'rb') as handle:\n    partition = pickle.load(handle)\nwith open('../input/speech-recognition-data-processing/SavedLabels.pickle', 'rb') as handle:\n    labels = pickle.load(handle)\nprint(\"loaded data.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c26b3153ef2ec859880454d54a9f7fa6d7a29da"},"cell_type":"code","source":"# Data Augmenting\ndef bandpass(sample_rate, samples):\n    \n    fs = sample_rate  # Sample frequency (Hz)\n    fl = 180.0  # Human voices range from 85 Hz to 255 Hz\n    fh = 240.0\n    Q = 1.0  # Quality factor\n    w0 = fl/(fs/2)  # Normalized Frequency\n    w1 = fh/(fs/2)\n    # Design notch filter\n    b, a = scipy.signal.butter(3, [w0, w1], btype='bandpass', analog=True)\n    samples = scipy.signal.lfilter(b,a,samples)*30\n\n    return sample_rate, samples","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"03fa26857166dfd714811effe51f9022957cf78d"},"cell_type":"code","source":"def spectrogram(sample_rate, samples):\n    eps=1e-10\n    frequencies, times, spectrogram = signal.spectrogram(samples, sample_rate)\n    \n    # silence can end up being empty files, in this case we can just return one second of zeros\n    if len(spectrogram.shape) < 2:\n        return np.zeros((71,129))\n    else:\n        return np.log(np.abs(spectrogram).T+eps)\n\ndef stft(sample_rate, samples):\n\n    eps=1e-10\n\n    frequencies, times, Zxx = signal.stft(samples, sample_rate, nperseg = sample_rate/50, noverlap = sample_rate/75)\n    \n    # silence can end up being empty files, in this case we can just return one second of zeros\n    if len(Zxx.shape) < 2:\n        return np.zeros((151,161))\n    else:\n        return np.log(np.abs(Zxx).T+eps)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bd5626be0e738f2241986cfec76133a53a3776d5"},"cell_type":"code","source":"class DataGenerator(keras.utils.Sequence):\n    # Generates data for Keras\n    def __init__(self, list_IDs, labels, batch_size=32, dim=(151,161), x1dim=(71,129), x2dim=(151,161), n_channels=1,\n                 n_classes=11, shuffle=True, input_type='wav', testing=False, augment=True):\n        # Initialization\n        self.dim = dim\n        self.x1dim = x1dim\n        self.x2dim = x2dim\n        self.batch_size = batch_size\n        self.labels = labels\n        self.list_IDs = list_IDs\n        self.n_channels = n_channels\n        self.n_classes = n_classes\n        self.shuffle = shuffle\n        self.input_type = input_type\n        self.testing = testing\n        self.augment = augment\n        self.on_epoch_end()\n\n    def __len__(self):\n        # Denotes the number of batches per epoch\n        return int(np.floor(len(self.list_IDs) / self.batch_size))\n\n    def __getitem__(self, index):\n        # Generate one batch of data\n        # Generate indexes of the batch\n        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n\n        # Find list of IDs\n        list_IDs_temp = [self.list_IDs[k] for k in indexes]\n\n        # Generate data\n        if self.input_type == 'all':\n            X1, X2, y = self.__data_generation(list_IDs_temp)\n            if self.testing:\n                return [X1,X2]\n            else:\n                return [X1,X2], [y]\n        else:\n            X, y = self.__data_generation(list_IDs_temp)\n            \n            return X, y\n\n    def on_epoch_end(self):\n        # Updates indexes after each epoch\n        self.indexes = np.arange(len(self.list_IDs))\n        if self.shuffle == True:\n            np.random.shuffle(self.indexes)\n\n    def __data_generation(self, list_IDs_temp):\n        # Generates data containing batch_size samples # X : (n_samples, *dim, n_channels)\n        # Initialization\n\n        if self.input_type == 'all':\n            X1 = np.empty((self.batch_size, *self.x1dim, self.n_channels))\n            X2 = np.empty((self.batch_size, *self.x2dim, self.n_channels))\n        else:\n            X = np.empty((self.batch_size, *self.dim, self.n_channels))\n        y = np.empty((self.batch_size), dtype=int)\n\n        # Generate data  \n        if self.input_type == 'spectrogram':\n            for i, ID in enumerate(list_IDs_temp):\n                if self.labels[ID] == 11:\n                    path = '../input/'\n                else:\n                    path = train_audio_path + '/'\n                sample_rate, samples = wavfile.read(path + ID)\n\n                if self.augment:\n                    if random.randint(1,101) < 51:\n                        sample_rate, samples = bandpass(sample_rate, samples)\n                        \n                trans = spectrogram(sample_rate, samples)\n                #last = ID, self.dim, spect.shape\n\n                padded = np.zeros((self.x1dim))\n                padded[:trans.shape[0], :trans.shape[1]] = trans\n                X[i,] = padded[:, :, np.newaxis]\n                y[i] = self.labels[ID]\n                \n            return X, keras.utils.to_categorical(y, num_classes=self.n_classes)\n\n        elif self.input_type == 'stft':\n            for i, ID in enumerate(list_IDs_temp):\n                if self.labels[ID] == 11:\n                    path = '../input/'\n                else:\n                    path = train_audio_path + '/'\n                sample_rate, samples = wavfile.read(path + ID)\n\n                if self.augment:\n                    if random.randint(1,101) < 51:\n                        sample_rate, samples = bandpass(sample_rate, samples)\n                        \n                trans = stft(sample_rate, samples)\n                #last = ID, self.dim, spect.shape\n\n                padded = np.zeros((self.x2dim))\n                padded[:trans.shape[0], :trans.shape[1]] = trans\n                X[i,] = padded[:, :, np.newaxis]\n                y[i] = self.labels[ID]\n            \n            return X, keras.utils.to_categorical(y, num_classes=self.n_classes)\n        \n        elif self.input_type == 'all':\n        \n            for i, ID in enumerate(list_IDs_temp):\n                if self.labels[ID] == 11:\n                    path = '../input/'\n                else:\n                    path = train_audio_path + '/'\n                sample_rate, samples = wavfile.read(path + ID)\n\n                if self.augment:\n                    if random.randint(1,101) < 51:\n                        sample_rate, samples = bandpass(sample_rate, samples)\n                        \n                trans = spectrogram(sample_rate, samples)\n                #last = ID, self.dim, spect.shape\n\n                padded = np.zeros((self.x1dim))\n                padded[:trans.shape[0], :trans.shape[1]] = trans\n                X1[i,] = padded[:, :, np.newaxis]\n                y[i] = self.labels[ID]\n            for i, ID in enumerate(list_IDs_temp):\n                if self.labels[ID] == 11:\n                    path = '../input/'\n                else:\n                    path = train_audio_path + '/'\n                sample_rate, samples = wavfile.read(path + ID)\n\n                if self.augment:\n                    if random.randint(1,101) < 51:\n                        sample_rate, samples = bandpass(sample_rate, samples)\n                        \n                trans = stft(sample_rate, samples)\n                #last = ID, self.dim, spect.shape\n\n                padded = np.zeros((self.x2dim))\n                padded[:trans.shape[0], :trans.shape[1]] = trans\n                X2[i,] = padded[:, :, np.newaxis]\n                \n            return X1, X2, keras.utils.to_categorical(y, num_classes=self.n_classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c086d0cbaa825e141b19998e964d280efbd4c2da"},"cell_type":"code","source":"from keras.models import load_model\n\nspect_model = load_model('../input/spect-model/spect_model.h5')\nstft_model = load_model('../input/stft-model/stft_model.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d4ce9f5249a19bb1de86380fa154758df2550156"},"cell_type":"code","source":"spect_model.trainable = False\nspect_model.layers.pop()\nspect_model.compile\n\nfor layer in spect_model.layers:\n    layer.name = \"spect_\" + layer.name\n    \nstft_model.trainable = False\nstft_model.layers.pop()\nstft_model.compile\n\nfor layer in stft_model.layers:\n    layer.name = \"stft_\" + layer.name","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8b4a1b805ff4ec91f902422ef1bed48b01c7bfac"},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.callbacks import LearningRateScheduler\nfrom keras.layers import concatenate, Dense, Dropout, Flatten, Conv2D, MaxPool2D, BatchNormalization\nfrom keras.models import Model\nfrom keras.optimizers import Adam\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.utils.np_utils import to_categorical\n\n# Parameters\nparams = {'x1dim': (71,129),\n          'x2dim': (151,161),\n          'batch_size': 64,\n          'n_classes':12,\n          'n_channels': 1,\n          'shuffle': True,\n          'input_type': 'all',\n          'augment': True\n         }\n\n# Generators\ntraining_generator = DataGenerator(partition['train'], labels, **params)\nvalidation_generator = DataGenerator(partition['validation'], labels, **params)\n\nspect_model_output = spect_model.get_layer('spect_dropout_3').output\nstft_model_output = stft_model.get_layer('stft_dropout_3').output\n\nconcatenated = concatenate([spect_model_output, stft_model_output])\nprediction = Dense(12, activation='softmax', name='prediction')(concatenated)\n        \nensemble_model = Model([spect_model.input, stft_model.input], prediction)\n\nfor layer in ensemble_model.layers:\n    if not(layer.name) == 'prediction':\n        layer.trainable = False\n\nensemble_model.compile(optimizer = 'adam',\n                       loss='categorical_crossentropy',\n                       metrics=[\"accuracy\"])\n\nensemble_model.summary()\n\nannealer = LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x)\n\n# Train model on dataset\nhistory = ensemble_model.fit_generator(generator=training_generator,\n                                       validation_data=validation_generator,\n                                       steps_per_epoch=30,\n                                       epochs=20,\n                                       verbose=2,\n                                       callbacks=[annealer]\n                                      )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d6d25a440b54efe20f16389eee541c07b0721608"},"cell_type":"code","source":"acc = history.history['acc']\nval_acc = history.history['val_acc']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs = range(1, len(acc) + 1)\n\nplt.plot(epochs, acc, 'bo', label='Training acc')\nplt.plot(epochs, val_acc, 'b', label='Validation acc')\nplt.title('CNN Accuracy')\nplt.legend()\n\nplt.figure()\n\nplt.plot(epochs, loss, 'bo', label='Training loss')\nplt.plot(epochs, val_loss, 'b', label='Validation loss')\nplt.title('CNN Loss')\nplt.legend()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f1c53e72063dd4005a5a7764f1924521fab82245"},"cell_type":"code","source":"ensemble_model.save('ensemble_model.h5')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}