{"nbformat_minor": 1, "cells": [{"execution_count": null, "outputs": [], "metadata": {"_uuid": "cd6bb6e55d735d00849fcf1ae522f78b358c5b6a", "_cell_guid": "5a4cf075-58d8-4c14-a6f4-6fe73d8a64ff", "_kg_hide-input": false, "_kg_hide-output": false}, "cell_type": "code", "source": ["%matplotlib inline\n", "from subprocess import check_output\n", "#print(check_output([\"ls\", \"../working\"]).decode(\"utf8\"))\n", " \n"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "5026fb497564faeca2475fa16e3d74f7b6e07d85", "_cell_guid": "8f6fdbff-3ae0-4c25-be6b-79d41b6c1ae9", "collapsed": true}, "cell_type": "code", "source": ["#load libraries\n", "\n", "#import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "#from pathlib import Path\n", "#import IPython.display as ipd\n", "\n", "\n", "#import seaborn as sns\n", "\n"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "0df13f3b0ed2d27c16b16ab2039f08b08edd7d50", "_cell_guid": "360df6a2-d3f6-4114-82c0-57d8b6047c1a", "collapsed": true}, "cell_type": "code", "source": ["from scipy.io import wavfile\n", "train_audio_path = \"../input/train/audio/\"\n", "filename = \"/yes/012c8314_nohash_0.wav\"\n", "sample_rate, samples = wavfile.read(str(train_audio_path)+filename)"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "4a90ca3df5bc1ce2f154921597a9451240b8309f", "_cell_guid": "34480efd-e356-42f9-8887-1cd251f005c5", "collapsed": true}, "cell_type": "code", "source": ["from scipy import signal\n", "import numpy as np # linear algebra\n", "def visual_spectogram(audio, sample_rate, window_size=20, step_size=10, eps=1e-10):\n", "    n_per_scnd = int(round(window_size * sample_rate / 1e3))\n", "    n_overlaps = int(round(step_size * sample_rate / 1e3))\n", "    _, _, spec = signal.spectrogram(audio, fs=sample_rate, window='hann',\n", "                                  nperseg = n_per_scnd,\n", "                                  noverlap = n_overlaps,\n", "                                  detrend=False)\n", "    return np.log(spec.T.astype(np.float32) + eps)"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "3c6b4da41f14ddb7647ac9a94e97cedc8b7e69d8", "_cell_guid": "067df237-4cff-4b47-927e-946ce03939a7"}, "cell_type": "code", "source": ["import matplotlib.pyplot as plt\n", "spectgrm = visual_spectogram(samples, sample_rate)\n", "figure = plt.figure(figsize=(10,10))\n", "ax1 = figure.add_subplot(211)\n", "ax1.set_title(\"Wave form for yes\")\n", "ax1.set_ylabel(\"Amplitude\")\n", "ax1.plot(samples)\n", "ax2 = figure.add_subplot(212)\n", "ax2.set_title(\"Spectogram for yes\")\n", "ax2.set_ylabel(\"Features (from 0 to 8000)\")\n", "ax2.set_xlabel(\"Samples\")\n", "ax2.imshow(spectgrm.T, aspect=\"auto\",origin=\"lower\")\n"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "e2ddf17f133f57632e5091d88a025e540ed166b2", "_cell_guid": "c927bd98-3a83-4ff5-a68e-e867e0e1b8c0"}, "cell_type": "code", "source": ["print(spectgrm)"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "b673ddadbcaf42972c4063062af96a32453f2d72", "_cell_guid": "6e10aea8-ac57-45ef-9381-7ed939514d5b"}, "cell_type": "code", "source": ["#normalize values\n", "mean = np.mean(spectgrm, axis = 0)\n", "std = np.std(spectgrm, axis = 0)\n", "spectgrm = (spectgrm - mean) /std\n", "print (spectgrm)"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "dd2a344f35719ac1c0c0e3c8c999b85aaea6e018", "_cell_guid": "0d5dbe80-e625-47d1-8ccc-c0cd22ca558c"}, "cell_type": "code", "source": ["import os\n", "from os.path import isdir, join\n", "directories = [f for f in os.listdir(train_audio_path) if \n", "              isdir(join(train_audio_path, f))]\n", "directories.sort() # 31 labels\n", "number_of_recs = []\n", "for directory in directories:\n", "    waves = [f for f in os.listdir(join(train_audio_path, directory))]\n", "    number_of_recs.append(len(waves))\n", "plt.figure(figsize=(10,10))\n", "plt.bar(directories, number_of_recs)\n", "plt.title(\"Number of recs by label\")\n", "plt.xticks(rotation=\"vertical\")\n", "plt.ylabel(\"Y\");plt.xlabel(\"X\")\n", "plt.show()\n"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "5a3e46e86c0218671d610172e0175dda0437b8ef", "_cell_guid": "427c5834-ee21-4cd0-b0b4-78bcfabe31e7", "collapsed": true}, "cell_type": "code", "source": ["def fastfouriertransform (y, fs):\n", "    T = 1.0 / fs\n", "    N = y.shape[0]\n", "    yf = fft(y)\n", "    xf = np.linspace(0.0, 1.0/(2.0*T), N//2)\n", "    vals = 2.0/N * np.abs(yf[0:N//2])\n", "    return xf, vals\n", "    "]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "f209ade0ef2982e89e0bd72a703516c07bef2076", "_cell_guid": "6b2814dd-71bd-4d21-ae8e-09485f29b60d"}, "cell_type": "code", "source": ["words = 'yes no up down left right on off stop go silence unknown'.split()\n", "print(directories)"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "25c8565eab69f2d92613764e03d418372d7ed1ff", "_cell_guid": "67f588e4-e586-47bf-bc07-3547b475dfb0"}, "cell_type": "code", "source": ["from scipy.fftpack import fft\n", "for directory in directories:\n", "    if directory in words:\n", "        vals_all = []\n", "        spec_all = []\n", "        waves = [f for f in os.listdir(join(train_audio_path, directory))]\n", "        for wav in waves:\n", "            sample_rate,samples = wavfile.read(train_audio_path \n", "                                               + directory +\"/\"+wav)\n", "            if samples.shape[0] != 16000:\n", "                continue\n", "            xf, vals = fastfouriertransform(samples, 16000)\n", "            vals_all.append(vals)\n", "            spec_all.append(visual_spectogram(samples, 16000))\n", "            \n", "        plt.figure(figsize=(10,8))\n", "        plt.subplot(121)\n", "        plt.title(\"Mean foutransf of \"+ directory)\n", "        plt.plot(np.mean(np.array(vals_all), axis=0))\n", "        plt.grid()\n", "        plt.subplot(122)\n", "        plt.title(\"Mean spectgram of \"+ directory)\n", "        plt.imshow(np.mean(np.array(spec_all), axis = 0).T, \n", "                   aspect='auto', origin='lower')\n", "        plt.show()\n", "        "]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "5da1adfac5f06d0ae363f1d3e1f80db448798549", "_cell_guid": "18a4e4de-ce8f-4583-968a-072eef1aa288"}, "cell_type": "code", "source": ["print(waves)"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "687a47ecb7c74e8e2f5aaf44db6638d88a06e3aa", "_cell_guid": "33a2311b-084c-4b08-9509-4c9149d5d272"}, "cell_type": "code", "source": ["import os\n", "import re\n", "from glob import glob\n", "\n", "labels = 'yes no up down left right on off stop go silence unknown'.split()\n", "id2name = {i: name for i, name in enumerate(labels)}\n", "name2id = {name: i for i, name in id2name.items()}\n", "pattern = re.compile(\"([^_]+)_([^_]+)_.+wav\")\n", "data_dir = \"../input\"\n", "def load_data(data_dir):\n", "    \"\"\" \n", "    Returns 2 lists of tuples: [(class_id, user_id, path), ...] \n", "    \"\"\"\n", "    # prefix, label, user_id\n", "    pattern = re.compile(\"(.+\\/)?(\\w+)\\/([^_]+)_.+wav\")\n", "    all_files = glob(os.path.join(data_dir, 'train/audio/*/*wav'))\n", "    with open(os.path.join(data_dir, 'train/validation_list.txt'), 'r') as fin:\n", "        validation_files = fin.readlines()\n", "    validation_set = set()\n", "    for entry in validation_files:\n", "        r = re.match(pattern, entry)\n", "        if r:\n", "            validation_set.add(r.group(3))\n", "    \n", "    possible = set(labels)\n", "    train, val = [], []\n", "    for entry in all_files:\n", "        r = re.match(pattern, entry)\n", "        if r:\n", "            label, uid = r.group(2), r.group(3)\n", "            if label == '_background_noise_':\n", "                label = 'silence'\n", "            if label not in possible:\n", "                label = 'unknown'\n", "\n", "            label_id = name2id[label]\n", "\n", "            sample = (label_id, uid, entry)\n", "            if uid in validation_set:\n", "                val.append(sample)\n", "            else:\n", "                train.append(sample)\n", "\n", "    print('There are {} train and {} val samples'.format(len(train), len(val)))\n", "    return train, val\n", "\n", "trainset, valset = load_data(data_dir)\n", "        \n"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "33e71bd75d3f30d5a5fae218913590adb56fedff", "_cell_guid": "97c88318-0316-4ce8-b650-ce9e7600249a", "collapsed": true}, "cell_type": "code", "source": ["import numpy as np\n", "from scipy.io import wavfile\n", "def generate_data(data, params, mode='train'):\n", "    def generator():\n", "        if mode == 'train':\n", "            np.random.shuffle(data)\n", "        for (label_id, uid, fname) in data:\n", "            try:\n", "                _, wav = wavfile.read(fname)\n", "                wav = wav.astype(np.float32) / np.iinfo(np.int16).max\n", "                L = 16000\n", "                if len(wav) < L:\n", "                    continue\n", "                samples_per_file = 1 if label_id != name2id['silence'] else 20\n", "                for _ in range(samples_per_file):\n", "                    if len(wav) > L:\n", "                        beg = np.random.randint(0, len(wav) -L)\n", "                    else:\n", "                        beg = 0\n", "                    yield dict(target=np.int32(label_id),\n", "                              wav = wav[beg: beg + L])\n", "            except Exception as err:\n", "                print(err, label_id, uid, frame)\n", "    return generator"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "303b9e61a40ca9026c95d7af112ebcfb653fd546", "_cell_guid": "a0bda891-c2d8-407e-a1ec-4643f0f36b69"}, "cell_type": "code", "source": ["import tensorflow as tf\n", "from tensorflow.contrib import layers\n", "\n", "def baseline(x, params, is_training):\n", "    x = layers.batch_norm(x, is_training = is_training)\n", "    for i in range(4):\n", "        x = layers.conv2d(x, 16*(2**i),3,1,activation_fn=tf.nn.elu,\n", "                          normalizer_fn=layers.batch_norm if params.use_batch_norm else None,\n", "                          normalizer_params={'is_training': is_training})\n", "        x = layers.max_pool2d(x,2,2)\n", "    mpool = tf.reduce_max(x, axis=[1, 2], keep_dims=True)\n", "    apool = tf.reduce_mean(x, axis=[1, 2], keep_dims=True)\n", "    \n", "    x = 0.5 * (mpool + apool)\n", "    x = layers.conv2d(x, 128, 1, 1, activation_fn=tf.nn.elu)\n", "    x = tf.nn.dropout(x, keep_prob=params.keep_prob if is_training else 1.0)\n", "    logits = layers.conv2d(x, params.num_classes, 1, 1, activation_fn=None)\n", "    return tf.squeeze(logits, [1,2])\n", "            "]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "6b53cbe5c6caeff9e96a559b078763ab8c91ba1d", "_cell_guid": "f21c4f44-332c-43e4-baf6-cff79756b835", "collapsed": true}, "cell_type": "code", "source": ["from tensorflow.contrib import signal\n", "\n", "def model_helper(features, labels, mode, params, config):\n", "    extractor = tf.make_template('extractor', baseline, create_scope_now_=True)\n", "    wav = features['wav']\n", "    specgram = signal.stft(wav, 400, 160)\n", "    phase = tf.angle(specgram) / np.pi\n", "    amp = tf.log1p(tf.abs(specgram))\n", "    x = tf.stack([amp, phase], axis=3)\n", "    x = tf.to_float(x)\n", "    logits = extractor(x, params, mode == tf.estimator.ModeKeys.TRAIN)\n", "    if mode == tf.estimator.ModeKeys.TRAIN:\n", "        loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=labels,\n", "                                                                            logits=logits))\n", "        def learning_rate_decay_fn(learning_rate, global_step):\n", "            return tf.train.exponential_decay(learning_rate, global_step, \n", "                                              decay_steps= 10000, decay_rate=0.99)\n", "        \n", "        train_op = tf.contrib.layers.optimize_loss(loss=loss,\n", "                            global_step=tf.contrib.framework.get_global_step(),\n", "                            learning_rate=params.learning_rate,\n", "                            optimizer=lambda lr: tf.train.MomentumOptimizer(lr, \n", "                                                  0.9, use_nesterov=True),\n", "                            learning_rate_decay_fn =learning_rate_decay_fn,\n", "                            clip_gradients=params.clip_gradients,\n", "                            variables=tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES))\n", "        specs = dict(mode=mode, loss=loss, train_op=train_op)\n", "    if mode == tf.estimator.ModeKeys.EVAL:\n", "        prediction = tf.argmax(logits, axis=-1)\n", "        acc, acc_op = tf.metrics.mean_per_class_accuracy(labels, prediction,\n", "                                                        params.num_classes)\n", "        loss=tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(\n", "                            labels=labels, logits=logits))\n", "        specs = dict(mode=mode, loss=loss, eval_metric_ops=dict(acc=(acc,acc_op)))\n", "        if mode == tf.estimator.ModeKeys.PREDICT:\n", "            predictions = {\n", "                'label': tf.argmax(logits, axis=-1),\n", "                'sample': features['sample']\n", "            }\n", "            specs = dict(mode=mode, predictions=predictions)\n", "    if mode == tf.estimator.ModeKeys.PREDICT:\n", "        predictions = {\n", "            'label': tf.argmax(logits, axis=-1),  # for probability just take tf.nn.softmax()\n", "            'sample': features['sample'], # it's a hack for simplicity\n", "        }\n", "        specs = dict(\n", "            mode=mode,\n", "            predictions=predictions,\n", "        )\n", "    return tf.estimator.EstimatorSpec(**specs)\n", "\n", "def new_model(config=None, hparams=None):\n", "    return tf.estimator.Estimator(model_fn=model_helper, config=config, params=hdparams)"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "2785e4e6a64dd553500917e25a845e4ad461ce26", "_cell_guid": "7a4aa7c0-3c8b-420a-b855-5afd68e9135d"}, "cell_type": "code", "source": ["params = dict(seed=2018, batch_size=64, keep_prob=0.5, learning_rate=1e-3,\n", "             clip_gradients=15.0, use_batch_norm=True, num_classes=len(words))\n", "hparams = tf.contrib.training.HParams(**params)\n", "os.makedirs(os.path.join(\"../working\", 'eval'), exist_ok=True)\n", "model_dir = \"../working\"\n", "run_config = tf.contrib.learn.RunConfig(model_dir=model_dir)"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "c1a7c308314540a1905c5ed6fd98eb3c86312e16", "_cell_guid": "1b1cedf6-6b4b-42cf-94b6-356d10c9e9ba"}, "cell_type": "code", "source": ["from tensorflow.contrib.learn.python.learn.learn_io.generator_io import generator_input_fn\n", "\n", "train_input_fn = generator_input_fn(x=generate_data(trainset, hparams,\n", "                                                    'train'),\n", "                                    target_key='target',\n", "                                   batch_size=hparams.batch_size,\n", "                                   shuffle=True, num_epochs=None,\n", "                                   queue_capacity=3*hparams.batch_size\n", "                                   + 10, num_threads=1)\n", "val_input_fn = generator_input_fn(x=generate_data(valset,hparams,\n", "                                                  'val'),\n", "                                 target_key='target',\n", "                                 batch_size=hparams.batch_size, \n", "                                 shuffle=True, num_epochs=None,\n", "                                 queue_capacity=3*hparams.batch_size\n", "                                 + 10, num_threads=1)\n", "\n", "def jFlowTest(run_config, hparams):\n", "    exp = tf.contrib.learn.Experiment(\n", "            estimator=new_model(config=run_config, hparams=hparams),\n", "            train_input_fn=train_input_fn,\n", "            eval_input_fn=val_input_fn,\n", "            train_steps=1000,\n", "            eval_steps=20,\n", "            train_steps_per_iteration=100)\n", "    return exp\n", "\n", "tf.contrib.learn.learn_runner.run(experiment_fn=jFlowTest,\n", "                                 run_config=run_config,\n", "                                 schedule='continuous_train_and_eval',\n", "                                 hparams=hparams)"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "4b34ef21ec8e6e2c556a660a4d4a32435ddaf6c5", "_cell_guid": "11c5391a-5dfa-412f-9af7-b9513dfba1b4", "collapsed": true}, "cell_type": "code", "source": []}], "metadata": {"language_info": {"nbconvert_exporter": "python", "codemirror_mode": {"version": 3, "name": "ipython"}, "pygments_lexer": "ipython3", "version": "3.6.3", "file_extension": ".py", "name": "python", "mimetype": "text/x-python"}, "kernelspec": {"language": "python", "display_name": "Python 3", "name": "python3"}}, "nbformat": 4}