{"cells":[{"metadata":{"_uuid":"82ccc722c0b467c3bbdceaed2c37dc4b1abe56e1","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom matplotlib import pyplot as plt\nfrom keras.models import Sequential, Input\nfrom keras.layers.core import Dense, Dropout, Activation, Flatten\nfrom keras.layers.convolutional import Conv2D, Convolution2D, MaxPooling2D\nfrom keras.utils import np_utils\nimport numpy as np\nfrom keras import backend as K\nfrom keras.preprocessing.image import Iterator\nfrom keras.preprocessing.image import img_to_array\n\nimport librosa\nimport os\nimport multiprocessing.pool\nfrom functools import partial\n\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\ntrain_path = '../input/train/audio/'\ntest_path = '../input/test.7z'\n\n#\n# The classes correspond to directory names under ../train/audio\nclassnames = ['yes', 'no', 'up', 'down', 'left', 'right', 'on', 'off', 'stop', 'go']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"27f30f06493e549b8c6e17fee0d3f4e49c1a6c1a"},"cell_type":"code","source":"def spect_loader(path, window_size, window_stride, window, normalize, max_len=101, logit=True):\n    y, sr = librosa.load(path, sr=None)\n    # n_fft = 4096\n    n_fft = int(sr * window_size)\n    win_length = n_fft\n    hop_length = int(sr * window_stride)\n\n    # STFT\n    D = librosa.stft(y, n_fft=n_fft, hop_length=hop_length,\n                     win_length=win_length, window=window)\n    spect, phase = librosa.magphase(D)\n\n    if logit==True:\n        #S = log(S+1)\n        spect = np.log1p(spect)\n    # make all spects with the same dims\n    # TODO: change that in the future\n    if spect.shape[1] < max_len:\n        pad = np.zeros((spect.shape[0], max_len - spect.shape[1]))\n        spect = np.hstack((spect, pad))\n    elif spect.shape[1] > max_len:\n        spect = spect[:, :max_len]\n    spect = np.resize(spect, (1, spect.shape[0], spect.shape[1]))\n    #spect = torch.FloatTensor(spect)\n\n    # z-score normalization\n    if normalize:\n        mean = np.mean(np.ravel(spect))\n        std = np.std(np.ravel(spect))\n        if std != 0:\n            spect = spect -mean\n            spect = spect / std\n\n    return spect\n\ndef _count_valid_files_in_directory(directory, white_list_formats, follow_links):\n    \"\"\"Count files with extension in `white_list_formats` contained in a directory.\n    # Arguments\n        directory: absolute path to the directory containing files to be counted\n        white_list_formats: set of strings containing allowed extensions for\n            the files to be counted.\n    # Returns\n        the count of files with extension in `white_list_formats` contained in\n        the directory.\n    \"\"\"\n    def _recursive_list(subpath):\n        return sorted(os.walk(subpath, followlinks=follow_links), key=lambda tpl: tpl[0])\n\n    samples = 0\n    for root, _, files in _recursive_list(directory):\n        for fname in files:\n            is_valid = False\n            for extension in white_list_formats:\n                if fname.lower().endswith('.' + extension):\n                    is_valid = True\n                    break\n            if is_valid:\n                samples += 1\n    return samples\n\ndef _list_valid_filenames_in_directory(directory, white_list_formats,\n                                       class_indices, follow_links):\n    \"\"\"List paths of files in `subdir` relative from `directory` whose extensions are in `white_list_formats`.\n    # Arguments\n        directory: absolute path to a directory containing the files to list.\n            The directory name is used as class label and must be a key of `class_indices`.\n        white_list_formats: set of strings containing allowed extensions for\n            the files to be counted.\n        class_indices: dictionary mapping a class name to its index.\n    # Returns\n        classes: a list of class indices\n        filenames: the path of valid files in `directory`, relative from\n            `directory`'s parent (e.g., if `directory` is \"dataset/class1\",\n            the filenames will be [\"class1/file1.jpg\", \"class1/file2.jpg\", ...]).\n    \"\"\"\n    def _recursive_list(subpath):\n        return sorted(os.walk(subpath, followlinks=follow_links), key=lambda tpl: tpl[0])\n\n    classes = []\n    filenames = []\n    subdir = os.path.basename(directory)\n    basedir = os.path.dirname(directory)\n    for root, _, files in _recursive_list(directory):\n        for fname in sorted(files):\n            is_valid = False\n            for extension in white_list_formats:\n                if fname.lower().endswith('.' + extension):\n                    is_valid = True\n                    break\n            if is_valid:\n                classes.append(class_indices[subdir])\n                # add filename relative to directory\n                absolute_path = os.path.join(root, fname)\n                filenames.append(os.path.relpath(absolute_path, basedir))\n    return classes, filenames\n\nclass SpeechDirectoryIterator(Iterator):\n    \"\"\"Iterator capable of reading images from a directory on disk.\n    # Arguments\n       \n    \"\"\"\n\n    def __init__(self, directory, window_size, window_stride, \n                 window_type, normalize, max_len=101, logit=True,\n                 target_size=(256, 256), color_mode='grayscale',\n                 classes=None, class_mode='categorical',\n                 batch_size=32, shuffle=True, seed=None,\n                 data_format=None, save_to_dir=None,\n                 save_prefix='', save_format='png',\n                 follow_links=False, interpolation='nearest'):\n        if data_format is None:\n            data_format = K.image_data_format()\n        self.window_size = window_size\n        self.window_stride = window_stride\n        self.window_type = window_type\n        self.normalize = normalize\n        self.max_len = max_len\n        self.directory = directory\n        self.logit = logit\n#        self.image_data_generator = image_data_generator\n        self.target_size = tuple(target_size)\n        if color_mode not in {'rgb', 'grayscale'}:\n            raise ValueError('Invalid color mode:', color_mode,\n                             '; expected \"rgb\" or \"grayscale\".')\n        self.color_mode = color_mode\n        self.data_format = data_format\n        if self.color_mode == 'rgb':\n            if self.data_format == 'channels_last':\n                self.image_shape = self.target_size + (3,)\n            else:\n                self.image_shape = (3,) + self.target_size\n        else:\n            if self.data_format == 'channels_last':\n                self.image_shape = self.target_size + (1,)\n            else:\n                self.image_shape = (1,) + self.target_size\n        self.classes = classes\n        if class_mode not in {'categorical', 'binary', 'sparse',\n                              'input', None}:\n            raise ValueError('Invalid class_mode:', class_mode,\n                             '; expected one of \"categorical\", '\n                             '\"binary\", \"sparse\", \"input\"'\n                             ' or None.')\n        self.class_mode = class_mode\n        self.save_to_dir = save_to_dir\n        self.save_prefix = save_prefix\n        self.save_format = save_format\n        self.interpolation = interpolation\n\n        white_list_formats = {'png', 'jpg', 'jpeg', 'bmp', 'ppm', 'wav'}\n\n        # first, count the number of samples and classes\n        self.samples = 0\n\n        if not classes:\n            classes = []\n            for subdir in sorted(os.listdir(directory)):\n                if os.path.isdir(os.path.join(directory, subdir)):\n                    classes.append(subdir)\n        self.num_classes = len(classes)\n        self.class_indices = dict(zip(classes, range(len(classes))))\n\n        pool = multiprocessing.pool.ThreadPool()\n        function_partial = partial(_count_valid_files_in_directory,\n                                   white_list_formats=white_list_formats,\n                                   follow_links=follow_links)\n        self.samples = sum(pool.map(function_partial,\n                                    (os.path.join(directory, subdir)\n                                     for subdir in classes)))\n\n        print('Found %d images belonging to %d classes.' % (self.samples, self.num_classes))\n\n        # second, build an index of the images in the different class subfolders\n        results = []\n\n        self.filenames = []\n        self.classes = np.zeros((self.samples,), dtype='int32')\n        i = 0\n        for dirpath in (os.path.join(directory, subdir) for subdir in classes):\n            results.append(pool.apply_async(_list_valid_filenames_in_directory,\n                                            (dirpath, white_list_formats,\n                                             self.class_indices, follow_links)))\n            \n        \n        for res in results:\n            classes, filenames = res.get()\n            self.classes[i:i + len(classes)] = classes\n            self.filenames += filenames\n            if i==0:\n                img = spect_loader(os.path.join(self.directory, filenames[0]), \n                               self.window_size, \n                               self.window_stride, \n                               self.window_type, \n                               self.normalize, \n                               self.max_len, \n                               self.logit) \n                img=np.swapaxes(img, 0, 2)\n                self.target_size = tuple((img.shape[0], img.shape[1]))\n                print(self.target_size)\n                if self.color_mode == 'rgb':\n                    if self.data_format == 'channels_last':\n                        self.image_shape = self.target_size + (3,)\n                    else:\n                        self.image_shape = (3,) + self.target_size\n                else:\n                    if self.data_format == 'channels_last':\n                        self.image_shape = self.target_size + (1,)\n                    else:\n                        self.image_shape = (1,) + self.target_size\n                        \n            i += len(classes)\n        pool.close()\n        pool.join()\n        super(SpeechDirectoryIterator, self).__init__(self.samples, batch_size, shuffle, seed)\n    \n\n    \n    \n    def _get_batches_of_transformed_samples(self, index_array):\n        batch_x = np.zeros((len(index_array),) + self.image_shape, dtype=K.floatx())\n        batch_f = []\n        grayscale = self.color_mode == 'grayscale'\n        # build batch of image data\n        #print(index_array)\n        for i, j in enumerate(index_array):\n            #print(i, j, self.filenames[j])\n            fname = self.filenames[j]\n            #img = load_img(os.path.join(self.directory, fname),\n            #               grayscale=grayscale,\n            #               target_size=self.target_size,\n            #               interpolation=self.interpolation)\n            img = spect_loader(os.path.join(self.directory, fname), \n                               self.window_size, \n                               self.window_stride, \n                               self.window_type, \n                               self.normalize, \n                               self.max_len)\n            img=np.swapaxes(img, 0, 2)\n            \n            x = img_to_array(img, data_format=self.data_format)\n            #x = self.image_data_generator.random_transform(x)\n            #x = self.image_data_generator.standardize(x)\n            batch_x[i] = x\n            batch_f.append(fname)\n        # optionally save augmented images to disk for debugging purposes\n        if self.save_to_dir:\n            for i, j in enumerate(index_array):\n                img = array_to_img(batch_x[i], self.data_format, scale=True)\n                fname = '{prefix}_{index}_{hash}.{format}'.format(prefix=self.save_prefix,\n                                                                  index=j,\n                                                                  hash=np.random.randint(1e7),\n                                                                  format=self.save_format)\n                img.save(os.path.join(self.save_to_dir, fname))\n        # build batch of labels\n        if self.class_mode == 'input':\n            batch_y = batch_x.copy()\n        elif self.class_mode == 'sparse':\n            batch_y = self.classes[index_array]\n        elif self.class_mode == 'binary':\n            batch_y = self.classes[index_array].astype(K.floatx())\n        elif self.class_mode == 'categorical':\n            batch_y = np.zeros((len(batch_x), self.num_classes), dtype=K.floatx())\n            for i, label in enumerate(self.classes[index_array]):\n                batch_y[i, label] = 1.\n        else:\n            return batch_x\n        return batch_x, batch_y\n\n    def next(self):\n        \"\"\"For python 2.x.\n        # Returns\n            The next batch.\n        \"\"\"\n        with self.lock:\n            index_array = next(self.index_generator)[0]\n        # The transformation of images is not under thread lock\n        # so it can be done in parallel\n        return self._get_batches_of_transformed_samples(index_array)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6df58245ff611b25112e55549a984b04b872a235"},"cell_type":"code","source":"window_size=.02\nwindow_stride=.01\nwindow_type='hamming'\nnormalize=True\nmax_len=101\nbatch_size = 64\ntrain_iterator = SpeechDirectoryIterator(directory=train_path, \n                                   batch_size=batch_size, \n                                   window_size=window_size, \n                                   window_stride=window_stride, \n                                   window_type=window_type,\n                                   normalize=normalize, \n                                   max_len=max_len, \n                                   classes=classnames, \n                                   shuffle=True, \n                                   seed=123)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"27e05b909fc60acb7577ab3190ab89e16297ae72"},"cell_type":"code","source":"train_iterator.reset()\nX, y = next(train_iterator)\nprint(X.shape)\nf, axarr = plt.subplots(3, 3)\nf.set_figheight(8)\nf.set_figwidth(15)\nfor i in range(9):\n    axarr[int(i/3), i%3].imshow(X[i, ..., 0], cmap='gray')\n    axarr[int(i/3), i%3].set_title(classnames[np.argmax(y[i])])\nplt.show()\n#plt.imshow(X[0, ..., 0], cmap='gray')\n#plt.title(classnames[np.argmax(y[0])])\n#plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"909e6e73d361608926be763ca5bd34d369ae08b0"},"cell_type":"code","source":"train_iterator_2 = SpeechDirectoryIterator(directory=train_path, \n                                   batch_size=batch_size, \n                                   window_size=window_size, \n                                   window_stride=window_stride, \n                                   window_type=window_type,\n                                   normalize=normalize, \n                                   logit = False, \n                                   max_len=61, \n                                   classes=classnames, \n                                   shuffle=True, \n                                   seed=123)\ntrain_iterator_2.reset()\nX, y = next(train_iterator_2)\nprint(X.shape)\nf, axarr = plt.subplots(3, 3)\nf.set_figheight(8)\nf.set_figwidth(15)\nfor i in range(9):\n    axarr[int(i/3), i%3].imshow(X[i, ..., 0], cmap='gray')\n    axarr[int(i/3), i%3].set_title(classnames[np.argmax(y[i])])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"894040e00c25c842ff59f62785348b9fb5d77f4f"},"cell_type":"code","source":"model = Sequential()\nmodel.add(Conv2D(12, (5, 5), activation = 'relu', input_shape=train_iterator.image_shape))\n\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel.add(Conv2D(25, (5, 5), activation = 'relu'))\n\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel.add(Flatten())\nmodel.add(Dense(180, activation = 'relu'))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(100, activation = 'relu'))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(len(classnames), activation = 'softmax')) #Last layer with one output per class\n\nmodel.compile(loss='categorical_crossentropy', optimizer='Adam', metrics=[\"accuracy\"])\nmodel.summary()\n# Make the model learn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"887ba0ec598d6e09ee1188ee6da4e489aa5ff185"},"cell_type":"code","source":"model.fit_generator(train_iterator,\n        steps_per_epoch=np.ceil(train_iterator.n / batch_size),\n        epochs=3,\n        verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3562e99817d787f94ec2c2bffa866d0e5182d77"},"cell_type":"code","source":"predict_iterator = SpeechDirectoryIterator(directory=test_path, \n                                   batch_size=batch_size, \n                                   window_size=window_size, \n                                   window_stride=window_stride, \n                                   window_type=window_type,\n                                   normalize=normalize, \n                                   max_len=max_len, \n                                   classes=None,\n                                   shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"9a39b733ecd91ae498c39d498b13aa4bcb031728"},"cell_type":"code","source":"preds = model.predict_generator(generator=predict_iterator, \n                        steps=int(np.ceil(predict_iterator.n)/batch_size), \n                        verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"905c0e3f4b6db207373a0ee2c0ad596ae4b32d52"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}