{"nbformat_minor":1,"metadata":{"language_info":{"name":"python","file_extension":".py","mimetype":"text/x-python","pygments_lexer":"ipython3","codemirror_mode":{"name":"ipython","version":3},"version":"3.6.3","nbconvert_exporter":"python"},"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"}},"cells":[{"cell_type":"markdown","metadata":{"_cell_guid":"fd4e020e-eb9e-4923-b3b2-a57d250cfce9","_uuid":"e6f828d9451b8467c31df464be29ae1207e768f0"},"source":"# This is a TF Estimator end-to-end baseline solution\n\n**For local run**\n\nTested with\n\n```\nnumpy==1.13.3\nscipy==0.19.1\ntensorflow-gpu==1.4.0\ntqdm\n```\n\n\nI want to show usage of Estimators with custom python datagenerators.\n\n\nDetailed documentation you can find at https://www.tensorflow.org/api_docs/python/tf/estimator/Estimator\n\nI also recommend to read source code  https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/estimator/estimator.py"},{"cell_type":"markdown","metadata":{"_cell_guid":"c259db77-8b94-4358-a857-a9b2af2a7bff","_uuid":"016cccacbb91b615408f50665f4c581e4d9d33b7"},"source":"Suppose we have following project structure:\n```\n.\n├── data\n│   ├── test            # extracted\n│   │   └── audio          # all test\n│   ├── test.7z         # downloaded\n│   ├── train           # extracted\n│   │   ├── audio          # folder with all train command/file.wav\n│   │   ├── LICENSE\n│   │   ├── README.md\n│   │   ├── testing_list.txt\n│   │   └── validation_list.txt\n│   └── train.7z         # downloaded\n├── kernel.ipynb      # this ipynb  \n└── model-k           # folder for model, checkpoints, logs and submission.csv\n```"},{"execution_count":null,"cell_type":"code","metadata":{"collapsed":true,"_uuid":"0cec4d047a2690ce89d616727a99d63c599932df","_cell_guid":"c9b84c36-51d1-420d-bcb4-e3ba21e329ae"},"source":"DATADIR = './data' # unzipped train and test data\nOUTDIR = './model-k' # just a random name\n# Data Loading\nimport os\nimport re\nfrom glob import glob\n\n\nPOSSIBLE_LABELS = 'yes no up down left right on off stop go silence unknown'.split()\nid2name = {i: name for i, name in enumerate(POSSIBLE_LABELS)}\nname2id = {name: i for i, name in id2name.items()}\n\n\ndef load_data(data_dir):\n    \"\"\" Return 2 lists of tuples:\n    [(class_id, user_id, path), ...] for train\n    [(class_id, user_id, path), ...] for validation\n    \"\"\"\n    # Just a simple regexp for paths with three groups:\n    # prefix, label, user_id\n    pattern = re.compile(\"(.+\\/)?(\\w+)\\/([^_]+)_.+wav\")\n    all_files = glob(os.path.join(data_dir, 'train/audio/*/*wav'))\n\n    with open(os.path.join(data_dir, 'train/validation_list.txt'), 'r') as fin:\n        validation_files = fin.readlines()\n    valset = set()\n    for entry in validation_files:\n        r = re.match(pattern, entry)\n        if r:\n            valset.add(r.group(3))\n\n    possible = set(POSSIBLE_LABELS)\n    train, val = [], []\n    for entry in all_files:\n        r = re.match(pattern, entry)\n        if r:\n            label, uid = r.group(2), r.group(3)\n            if label == '_background_noise_':\n                label = 'silence'\n            if label not in possible:\n                label = 'unknown'\n\n            label_id = name2id[label]\n\n            sample = (label_id, uid, entry)\n            if uid in valset:\n                val.append(sample)\n            else:\n                train.append(sample)\n\n    print('There are {} train and {} val samples'.format(len(train), len(val)))\n    return train, val\n\ntrainset, valset = load_data(DATADIR)","outputs":[]},{"cell_type":"markdown","metadata":{"_cell_guid":"60f9c087-cd45-4ac6-a059-e8509eae9c6b","_uuid":"917945bb153c92c34e3fbe6f7596ac087649bfc9"},"source":"Let me introduce pythonic datagenerator.\nIt is just a python/numpy/... function **without tf** that yields dicts such that\n```\n{\n  'x': np.array(...),\n  'str_key': np.string_(...),\n  'label': np.int32(...),\n}\n```\n\nBe sure, every value in this dict has `.dtype` method."},{"execution_count":null,"cell_type":"code","metadata":{"collapsed":true,"_uuid":"c0949a3d233e1fbd2f6635a25e418129bdcbff8b","_cell_guid":"dc61746e-a954-488a-bc79-57769997c81c"},"source":"import numpy as np\nfrom scipy.io import wavfile\n\ndef data_generator(data, params, mode='train'):\n    def generator():\n        if mode == 'train':\n            np.random.shuffle(data)\n        # Feel free to add any augmentation\n        for (label_id, uid, fname) in data:\n            try:\n                _, wav = wavfile.read(fname)\n                wav = wav.astype(np.float32) / np.iinfo(np.int16).max\n\n                L = 16000  # be aware, some files are shorter than 1 sec!\n                if len(wav) < L:\n                    continue\n                # let's generate more silence!\n                samples_per_file = 1 if label_id != name2id['silence'] else 20\n                for _ in range(samples_per_file):\n                    if len(wav) > L:\n                        beg = np.random.randint(0, len(wav) - L)\n                    else:\n                        beg = 0\n                    yield dict(\n                        target=np.int32(label_id),\n                        wav=wav[beg: beg + L],\n                    )\n            except Exception as err:\n                print(err, label_id, uid, fname)\n\n    return generator","outputs":[]},{"cell_type":"markdown","metadata":{"_cell_guid":"549de038-fe87-4885-ab03-421a02bfaef0","_uuid":"b11d20405b10ffb5fd8b53f6b09407bcc39844fd"},"source":"\nSuppose, we have spectrograms and want to write feature extractor that produces logits.\n\n\nLet's write some simple net, treat sound as a picture.\n\n\n**Spectrograms** (input x) have shape `(batch_size, time_frames, freq_bins, 2)`.\n\n**Logits** is a tensor with shape `(batch_size, num_classes)`."},{"execution_count":null,"cell_type":"code","metadata":{"collapsed":true,"_uuid":"3311fa56f0c8477d58f43e446f1ba546ce484b18","_cell_guid":"b2dcec9e-ee86-4dfd-9ee1-8247763eac4f"},"source":"import tensorflow as tf\nfrom tensorflow.contrib import layers\n\ndef baseline(x, params, is_training):\n    x = layers.batch_norm(x, is_training=is_training)\n    for i in range(4):\n        x = layers.conv2d(\n            x, 16 * (2 ** i), 3, 1,\n            activation_fn=tf.nn.elu,\n            normalizer_fn=layers.batch_norm if params.use_batch_norm else None,\n            normalizer_params={'is_training': is_training}\n        )\n        x = layers.max_pool2d(x, 2, 2)\n\n    # just take two kind of pooling and then mix them, why not :)\n    mpool = tf.reduce_max(x, axis=[1, 2], keep_dims=True)\n    apool = tf.reduce_mean(x, axis=[1, 2], keep_dims=True)\n\n    x = 0.5 * (mpool + apool)\n    # we can use conv2d 1x1 instead of dense\n    x = layers.conv2d(x, 128, 1, 1, activation_fn=tf.nn.elu)\n    x = tf.nn.dropout(x, keep_prob=params.keep_prob if is_training else 1.0)\n    \n    # again conv2d 1x1 instead of dense layer\n    logits = layers.conv2d(x, params.num_classes, 1, 1, activation_fn=None)\n    return tf.squeeze(logits, [1, 2])","outputs":[]},{"cell_type":"markdown","metadata":{"_cell_guid":"3a481a98-6665-4057-bdb2-37939c637ebb","_uuid":"cd8a966e82c30c0acd2c0248194edb7beb1361f5"},"source":"We need to write a model handler for three regimes:\n- train\n- eval\n- predict\n\nLoss function, train_op, additional metrics and summaries should be defined.\n\nAlso, we need to convert sound waveform into spectrograms (we could do it with numpy/scipy/librosa in data generator, but TF has new signal processing API)"},{"execution_count":null,"cell_type":"code","metadata":{"collapsed":true,"_uuid":"73722beb5345d73b7157230a0b837933cab87c53","_cell_guid":"4b4a2dd4-7207-418f-bdb7-2ed8f3926f7b"},"source":"from tensorflow.contrib import signal\n\n# features is a dict with keys: tensors from our datagenerator\n# labels also were in features, but excluded in generator_input_fn by target_key\n\ndef model_handler(features, labels, mode, params, config):\n    # Im really like to use make_template instead of variable_scopes and re-usage\n    extractor = tf.make_template(\n        'extractor', baseline,\n        create_scope_now_=True,\n    )\n    # wav is a waveform signal with shape (16000, )\n    wav = features['wav']\n    # we want to compute spectograms by means of short time fourier transform:\n    specgram = signal.stft(\n        wav,\n        400,  # 16000 [samples per second] * 0.025 [s] -- default stft window frame\n        160,  # 16000 * 0.010 -- default stride\n    )\n    # specgram is a complex tensor, so split it into abs and phase parts:\n    phase = tf.angle(specgram) / np.pi\n    # log(1 + abs) is a default transformation for energy units\n    amp = tf.log1p(tf.abs(specgram))\n    \n    x = tf.stack([amp, phase], axis=3) # shape is [bs, time, freq_bins, 2]\n    x = tf.to_float(x)  # we want to have float32, not float64\n\n    logits = extractor(x, params, mode == tf.estimator.ModeKeys.TRAIN)\n\n    if mode == tf.estimator.ModeKeys.TRAIN:\n        loss = tf.reduce_mean(\n            tf.nn.sparse_softmax_cross_entropy_with_logits(labels=labels, logits=logits))\n        # some lr tuner, you could use move interesting functions\n        def learning_rate_decay_fn(learning_rate, global_step):\n            return tf.train.exponential_decay(\n                learning_rate, global_step, decay_steps=10000, decay_rate=0.99)\n\n        train_op = tf.contrib.layers.optimize_loss(\n            loss=loss,\n            global_step=tf.contrib.framework.get_global_step(),\n            learning_rate=params.learning_rate,\n            optimizer=lambda lr: tf.train.MomentumOptimizer(lr, 0.9, use_nesterov=True),\n            learning_rate_decay_fn=learning_rate_decay_fn,\n            clip_gradients=params.clip_gradients,\n            variables=tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES))\n\n        specs = dict(\n            mode=mode,\n            loss=loss,\n            train_op=train_op,\n        )\n\n    if mode == tf.estimator.ModeKeys.EVAL:\n        prediction = tf.argmax(logits, axis=-1)\n        acc, acc_op = tf.metrics.mean_per_class_accuracy(\n            labels, prediction, params.num_classes)\n        loss = tf.reduce_mean(\n            tf.nn.sparse_softmax_cross_entropy_with_logits(labels=labels, logits=logits))\n        specs = dict(\n            mode=mode,\n            loss=loss,\n            eval_metric_ops=dict(\n                acc=(acc, acc_op),\n            )\n        )\n\n    if mode == tf.estimator.ModeKeys.PREDICT:\n        predictions = {\n            'label': tf.argmax(logits, axis=-1),  # for probability just take tf.nn.softmax()\n            'sample': features['sample'], # it's a hack for simplicity\n        }\n        specs = dict(\n            mode=mode,\n            predictions=predictions,\n        )\n    return tf.estimator.EstimatorSpec(**specs)\n\n\ndef create_model(config=None, hparams=None):\n    return tf.estimator.Estimator(\n        model_fn=model_handler,\n        config=config,\n        params=hparams,\n    )","outputs":[]},{"cell_type":"markdown","metadata":{"_cell_guid":"a140c8a4-616b-419f-871e-fc3c509d5bc4","_uuid":"3cc6ffde960ae468479a3f273d891e10030007a2"},"source":"Define some params. Move model hyperparams (optimizer, extractor, num of layers, activation fn, ...) here"},{"execution_count":null,"cell_type":"code","metadata":{"collapsed":true,"_uuid":"cdbf3ad7f4142c1d34d43b89492b4108e1cb99b7","_cell_guid":"b7d3f860-0367-413f-81bc-87c8cdf32c33"},"source":"params=dict(\n    seed=2018,\n    batch_size=64,\n    keep_prob=0.5,\n    learning_rate=1e-3,\n    clip_gradients=15.0,\n    use_batch_norm=True,\n    num_classes=len(POSSIBLE_LABELS),\n)\n\nhparams = tf.contrib.training.HParams(**params)\nos.makedirs(os.path.join(OUTDIR, 'eval'), exist_ok=True)\nmodel_dir = OUTDIR\n\nrun_config = tf.contrib.learn.RunConfig(model_dir=model_dir)","outputs":[]},{"cell_type":"markdown","metadata":{"_cell_guid":"66ed9e2f-7c80-44fb-8963-352f43944983","_uuid":"47777cb17d5e456a11149e6e2aa2d23598ae334a"},"source":"**Let's run training!**"},{"execution_count":null,"cell_type":"code","metadata":{"collapsed":true,"_uuid":"989bb42acc28e698fe5a7267e35d93476c8959a0","_cell_guid":"72eefb5a-06d4-4a13-b4e5-2d4f81541595"},"source":"# it's a magic function :)\nfrom tensorflow.contrib.learn.python.learn.learn_io.generator_io import generator_input_fn\n            \ntrain_input_fn = generator_input_fn(\n    x=data_generator(trainset, hparams, 'train'),\n    target_key='target',  # you could leave target_key in features, so labels in model_handler will be empty\n    batch_size=hparams.batch_size, shuffle=True, num_epochs=None,\n    queue_capacity=3 * hparams.batch_size + 10, num_threads=1,\n)\n\nval_input_fn = generator_input_fn(\n    x=data_generator(valset, hparams, 'val'),\n    target_key='target',\n    batch_size=hparams.batch_size, shuffle=True, num_epochs=None,\n    queue_capacity=3 * hparams.batch_size + 10, num_threads=1,\n)\n            \n\ndef _create_my_experiment(run_config, hparams):\n    exp = tf.contrib.learn.Experiment(\n        estimator=create_model(config=run_config, hparams=hparams),\n        train_input_fn=train_input_fn,\n        eval_input_fn=val_input_fn,\n        train_steps=10000, # just randomly selected params\n        eval_steps=200,  # read source code for steps-epochs ariphmetics\n        train_steps_per_iteration=1000,\n    )\n    return exp\n\ntf.contrib.learn.learn_runner.run(\n    experiment_fn=_create_my_experiment,\n    run_config=run_config,\n    schedule=\"continuous_train_and_eval\",\n    hparams=hparams)","outputs":[]},{"cell_type":"markdown","metadata":{"_cell_guid":"58b2914f-8c14-452b-88d8-3a9ae3267810","_uuid":"ef9d2527d72ce27e8c09ccff20408349cb79fb19"},"source":"\nWhile it trains (~10-20min on i5 + 1080), you could start tensorboard on model_dir and see live chart like this\n\n![Tensorboard](https://pp.userapi.com/c841329/v841329524/3db60/fdNDyRMJHMQ.jpg)\n\n\nNow we want to predict testset and make submission file.\n\n1. Create datagenerator and input_function\n2. Load model\n3. Iterate over predictions and store results"},{"execution_count":null,"cell_type":"code","metadata":{"collapsed":true,"_uuid":"47a197904a9722e45f5dce0951f850efe7c230d7","_cell_guid":"cb4c90a2-38d6-4340-b45c-de417c47d94e"},"source":"from tqdm import tqdm\n# now we want to predict!\npaths = glob(os.path.join(DATADIR, 'test/audio/*wav'))\n\ndef test_data_generator(data):\n    def generator():\n        for path in data:\n            _, wav = wavfile.read(path)\n            wav = wav.astype(np.float32) / np.iinfo(np.int16).max\n            fname = os.path.basename(path)\n            yield dict(\n                sample=np.string_(fname),\n                wav=wav,\n            )\n\n    return generator\n\ntest_input_fn = generator_input_fn(\n    x=test_data_generator(paths),\n    batch_size=hparams.batch_size, \n    shuffle=False, \n    num_epochs=1,\n    queue_capacity= 10 * hparams.batch_size, \n    num_threads=1,\n)\n\nmodel = create_model(config=run_config, hparams=hparams)\nit = model.predict(input_fn=test_input_fn)\n\n\n# last batch will contain padding, so remove duplicates\nsubmission = dict()\nfor t in tqdm(it):\n    fname, label = t['sample'].decode(), id2name[t['label']]\n    submission[fname] = label\n\nwith open(os.path.join(model_dir, 'submission.csv'), 'w') as fout:\n    fout.write('fname,label\\n')\n    for fname, label in submission.items():\n        fout.write('{},{}\\n'.format(fname, label))","outputs":[]},{"cell_type":"markdown","metadata":{"_cell_guid":"e45e3957-0fd1-49bf-a66a-949610ecdfcb","_uuid":"6855a21623da16a326132598b63173551605432b"},"source":"## About tf.Estimators\n\n**Pros**:\n- no need to control Session\n- datagenerator feeds model via queues without explicit queue coding :)\n- you could naturaly export models into production\n    \n**Cons**:\n- it's very hard to debug computational graph (use `tf.add_check_numerics()` and `tf.Print` in case of problems)\n- boilerplate code\n- need to read source code for making interesting things\n\n\n**Conclusion**:\nEstimator is a nice abstraction with some boilerplate code :)\n\n\n## About Speech Recognition Challenge:\n\nYou could start from this end-to-end ipynb, improving several functions for much better results.\n\n\n\nMay the gradient flow be with you. "},{"execution_count":null,"cell_type":"code","metadata":{"collapsed":true,"_uuid":"519da838f8c86adf79ec66228f27119d61d8edbb","_cell_guid":"ca1689c9-d61f-4111-b18b-31892580ad82"},"source":"","outputs":[]}],"nbformat":4}