{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# The goal of this notebook is to\n## 1. Build an efficient dataset for training time series models\n## 2. Demonstrate how to define a time series model","metadata":{}},{"cell_type":"code","source":"!pip install git+https://github.com/tensorflow/docs","metadata":{"execution":{"iopub.status.busy":"2023-02-28T10:34:41.707621Z","iopub.execute_input":"2023-02-28T10:34:41.708596Z","iopub.status.idle":"2023-02-28T10:35:02.811145Z","shell.execute_reply.started":"2023-02-28T10:34:41.708541Z","shell.execute_reply":"2023-02-28T10:35:02.809544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%load_ext tensorboard\nimport tensorflow as tf\nimport tensorflow_io as tfio\nimport tensorflow_docs as tfdocs\nimport tensorflow_docs.modeling\nfrom tensorboard.plugins.hparams import api as hp\nimport numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot as plt\nfrom matplotlib import animation\nfrom IPython.display import HTML\nimport pathlib, shutil, tempfile, json\n\n\nNSIGNS = 250\nSEQLEN = 64\n\nLOGDIR = pathlib.Path(tempfile.mkdtemp())/\"tensorboard_logs\"\nshutil.rmtree(LOGDIR, ignore_errors=True)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-28T10:35:54.998697Z","iopub.execute_input":"2023-02-28T10:35:54.999123Z","iopub.status.idle":"2023-02-28T10:35:55.011974Z","shell.execute_reply.started":"2023-02-28T10:35:54.999086Z","shell.execute_reply":"2023-02-28T10:35:55.010461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('/kaggle/input/asl-signs/sign_to_prediction_index_map.json') as f:\n    sign_ids = json.load(f)\nsign_ids","metadata":{"execution":{"iopub.status.busy":"2023-02-28T10:35:55.536600Z","iopub.execute_input":"2023-02-28T10:35:55.537028Z","iopub.status.idle":"2023-02-28T10:35:55.560441Z","shell.execute_reply.started":"2023-02-28T10:35:55.536990Z","shell.execute_reply":"2023-02-28T10:35:55.558889Z"},"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define a function to visualize a sign","metadata":{}},{"cell_type":"code","source":"basepath = '/kaggle/input/asl-signs/'\nmetadata = pd.read_csv(basepath + 'train.csv')\n\ndef animate_parquet_sign(filename):\n    if basepath in filename:\n        filename = filename[len(basepath):]\n    signdata = pd.read_parquet(basepath + filename)\n    label = metadata.sign[metadata.path == filename].values[0]\n    frame_iterator = iter(signdata.groupby('frame'))\n    fig = plt.figure()\n    axis = plt.axes(xlim=(0,1),ylim=(0,2))\n    dots, = axis.plot([],[],'.', markersize=1)\n    def get_frame(framedata):\n        _, data = framedata\n        x = data.x\n        y = -data.y + 2\n        dots.set_data((x,y))\n        return dots,\n\n    a = animation.FuncAnimation(fig, get_frame, frames = frame_iterator)\n    plt.gca().set_aspect('equal')\n    plt.close(fig)\n    return label, HTML(a.to_html5_video())","metadata":{"execution":{"iopub.status.busy":"2023-02-28T10:36:00.090858Z","iopub.execute_input":"2023-02-28T10:36:00.091249Z","iopub.status.idle":"2023-02-28T10:36:00.390589Z","shell.execute_reply.started":"2023-02-28T10:36:00.091217Z","shell.execute_reply":"2023-02-28T10:36:00.389165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_file = '/kaggle/input/asl-signs/train_landmark_files/30680/1004525587.parquet'\nlabel, vid = animate_parquet_sign(example_file)\nprint(f'Showing animation for the sign \"{label}\"')\nvid","metadata":{"execution":{"iopub.status.busy":"2023-02-28T10:36:00.392807Z","iopub.execute_input":"2023-02-28T10:36:00.394176Z","iopub.status.idle":"2023-02-28T10:36:01.573019Z","shell.execute_reply.started":"2023-02-28T10:36:00.394137Z","shell.execute_reply":"2023-02-28T10:36:01.570958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Let's define some features\n\n* Input to the sign recognition model is a tensor\n* Each sample has the shape (nframes, npoints, 2), corresponding to landmark x and y coordinates\n* There are 468 face landmarks, 21 left_hand landmarks, 33 pose landmarks and 21 right_hand landmarks.","metadata":{}},{"cell_type":"code","source":"def get_features(filename):\n    filename = basepath + filename\n    df = pd.read_parquet(filename)\n    d_array = tf.keras.utils.pad_sequences(\n        [np.nan_to_num(\n            df[['x','y']].values.reshape((-1,1086)),\n        )],\n        maxlen = SEQLEN,\n        dtype = 'float32',\n    )\n    return np.squeeze(d_array)","metadata":{"execution":{"iopub.status.busy":"2023-02-28T10:36:04.992788Z","iopub.execute_input":"2023-02-28T10:36:04.993197Z","iopub.status.idle":"2023-02-28T10:36:05.002291Z","shell.execute_reply.started":"2023-02-28T10:36:04.993161Z","shell.execute_reply":"2023-02-28T10:36:05.000693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#max_frames = 2**10\n#def get_tf_features(filename):\n#    abs_name = basepath + filename\n#    spec = tf.TensorSpec(shape=[None], dtype=tf.float32)\n#    ds = tfio.IODataset.from_parquet(abs_name, {'x': spec, 'y': spec})\n#    return ds.batch(543, drop_remainder=True).batch(max_frames)\ndef tf_get_features(ftensor):\n    def feat_wrapper(ftensor):\n        return get_features(ftensor.numpy().decode('utf-8'))\n    return tf.py_function(\n        feat_wrapper,\n        [ftensor],\n        Tout=tf.float32\n    )\n    ","metadata":{"execution":{"iopub.status.busy":"2023-02-28T10:36:05.679529Z","iopub.execute_input":"2023-02-28T10:36:05.679956Z","iopub.status.idle":"2023-02-28T10:36:05.688172Z","shell.execute_reply.started":"2023-02-28T10:36:05.679921Z","shell.execute_reply":"2023-02-28T10:36:05.686562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Now construct the dataset","metadata":{}},{"cell_type":"code","source":"x_ds = tf.data.Dataset.from_tensor_slices(metadata['path'])\nsigns = metadata['sign'].map(sign_ids)\ny_ds = tf.data.Dataset.from_tensor_slices(signs)\ny_ds = y_ds.map(lambda y: tf.reshape(y, (1,)))","metadata":{"execution":{"iopub.status.busy":"2023-02-28T10:36:07.400019Z","iopub.execute_input":"2023-02-28T10:36:07.400414Z","iopub.status.idle":"2023-02-28T10:36:07.582025Z","shell.execute_reply.started":"2023-02-28T10:36:07.400379Z","shell.execute_reply":"2023-02-28T10:36:07.581085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"signs","metadata":{"execution":{"iopub.status.busy":"2023-02-28T10:36:07.697306Z","iopub.execute_input":"2023-02-28T10:36:07.698688Z","iopub.status.idle":"2023-02-28T10:36:07.709314Z","shell.execute_reply.started":"2023-02-28T10:36:07.698644Z","shell.execute_reply":"2023-02-28T10:36:07.707649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's look at what elements of this preliminary dataset are like","metadata":{}},{"cell_type":"code","source":"x_ds = x_ds.map(tf_get_features, num_parallel_calls = 4)","metadata":{"execution":{"iopub.status.busy":"2023-02-28T10:36:11.911789Z","iopub.execute_input":"2023-02-28T10:36:11.912199Z","iopub.status.idle":"2023-02-28T10:36:12.017238Z","shell.execute_reply.started":"2023-02-28T10:36:11.912166Z","shell.execute_reply":"2023-02-28T10:36:12.015732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir /kaggle/tmp","metadata":{"execution":{"iopub.status.busy":"2023-02-28T10:36:12.467981Z","iopub.execute_input":"2023-02-28T10:36:12.468386Z","iopub.status.idle":"2023-02-28T10:36:13.572987Z","shell.execute_reply.started":"2023-02-28T10:36:12.468352Z","shell.execute_reply":"2023-02-28T10:36:13.571509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds = tf.data.Dataset.zip((x_ds, y_ds))","metadata":{"execution":{"iopub.status.busy":"2023-02-28T10:36:14.793071Z","iopub.execute_input":"2023-02-28T10:36:14.793964Z","iopub.status.idle":"2023-02-28T10:36:14.807025Z","shell.execute_reply.started":"2023-02-28T10:36:14.793905Z","shell.execute_reply":"2023-02-28T10:36:14.805590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Check the approximate throughput of the data pipeline","metadata":{}},{"cell_type":"markdown","source":"This speed is very slow when loading every parquet file on kaggle provided disk space and cpus, which is why I will soon publish a dataset that includes cached, batched versions of the training and validation datasets","metadata":{}},{"cell_type":"code","source":"from tqdm import tqdm\nfor elem in tqdm(ds.take(1000)):\n    pass","metadata":{"execution":{"iopub.status.busy":"2023-02-28T10:36:17.792926Z","iopub.execute_input":"2023-02-28T10:36:17.793452Z","iopub.status.idle":"2023-02-28T10:36:34.274831Z","shell.execute_reply.started":"2023-02-28T10:36:17.793412Z","shell.execute_reply":"2023-02-28T10:36:34.273422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Now we get to define the model","metadata":{}},{"cell_type":"markdown","source":"* Each frame is first encoded by a fully connected DNN\n* Sequences are padded to a constant length for batching\n    * Empty padding frames are masked with tf.keras.layers.Masking\n* The encoded frames are fed to an LSTM","metadata":{}},{"cell_type":"code","source":"def get_model(hparams):\n    channels = 543*2\n    inputs = tf.keras.Input(shape=(SEQLEN,channels))\n    r = hparams['encoder_dropout']\n    norm = hparams['encoder_normalization']\n    n = len(list(filter(lambda x: x, hparams['encoder_units'])))\n    \n    mask = tf.keras.layers.Masking()\n    get_instance_normalization = lambda ch: tf.keras.layers.GroupNormalization(groups=ch)\n    norm_layers = {\n        'none': lambda _: None,\n        'batch': lambda _: tf.keras.layers.BatchNormalization(),\n        'layer': lambda _: tf.keras.layers.LayerNormalization(),\n        'instance': get_instance_normalization,\n    }\n    \n    enc_connections = [tf.keras.layers.Dense(units) for units in hparams['encoder_units'] if n]\n    enc_dropouts    = [tf.keras.layers.Dropout(r)           if r > 0 else None   for _ in range(n)]\n    enc_norms  = [norm_layers[norm](units) if units > 0 else None   for units in hparams['encoder_units']]\n    enc_activations = [tf.keras.layers.Activation(hparams['encoder_activation']) for _ in range(n)]\n    enc_layers = [\n        l for ls in zip(enc_connections, enc_dropouts, enc_norms, enc_activations)\n        for l in ls if l is not None\n    ]\n    frame_encoder = tf.keras.Sequential(enc_layers)\n\n    classifier_layers = [\n        tf.keras.layers.LSTM(hparams['lstm_units']),\n        tf.keras.layers.Dense(NSIGNS, activation='softmax')\n    ]\n    sign_model    = tf.keras.Sequential(classifier_layers)\n\n    outputs = sign_model(frame_encoder(mask(inputs)))\n    return tf.keras.Model(inputs, outputs)","metadata":{"execution":{"iopub.status.busy":"2023-02-28T10:37:02.081362Z","iopub.execute_input":"2023-02-28T10:37:02.081925Z","iopub.status.idle":"2023-02-28T10:37:02.097290Z","shell.execute_reply.started":"2023-02-28T10:37:02.081881Z","shell.execute_reply":"2023-02-28T10:37:02.095742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_callbacks(name, hparams):\n    return [\n        tfdocs.modeling.EpochDots(),\n        tf.keras.callbacks.EarlyStopping(\n            monitor='val_accuracy',\n            patience=hparams['earlystop_patience'],\n            restore_best_weights=True\n        ),\n        tf.keras.callbacks.ReduceLROnPlateau(\n            monitor = 'val_accuracy',\n            factor = hparams['lr_factor'],\n            patience = hparams['lr_patience']\n        ),\n        tf.keras.callbacks.TensorBoard(LOGDIR/name),\n        hp.KerasCallback(f'{LOGDIR}/{name}', hparams)\n    ]","metadata":{"execution":{"iopub.status.busy":"2023-02-28T10:37:04.373125Z","iopub.execute_input":"2023-02-28T10:37:04.374671Z","iopub.status.idle":"2023-02-28T10:37:04.383530Z","shell.execute_reply.started":"2023-02-28T10:37:04.374600Z","shell.execute_reply":"2023-02-28T10:37:04.381872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"HP_NUM_UNITS_0 = hp.HParam('num_units_0', hp.Discrete([256, 512,1024,2048]))\nHP_UNITS_1_FACTOR = hp.HParam('units_1_f', hp.Discrete([1/2, 1.,]))\nHP_UNITS_2_FACTOR = hp.HParam('units_2_f', hp.Discrete([0., 1/2, 1.]))\nHP_UNITS_3_FACTOR = hp.HParam('units_3_f', hp.Discrete([0., 1/2, 1.]))\n\nHP_LSTM_UNITS = hp.HParam('lstm_units', hp.Discrete([256, 512, 1024]))\n\nHP_DROPOUT = hp.HParam('encoder_dropout', hp.Discrete([0.0, 0.1]))\nHP_NORM = hp.HParam('encoder_normalization', hp.Discrete(['none', 'layer', 'instance'])) # batch norm is bad here\nHP_ACTIVATION = hp.HParam('encoder_activation', hp.Discrete(['relu', 'swish', 'sigmoid']))\nHP_BATCHSIZE = hp.HParam('batch_size', hp.Discrete([512]))\n\nHP_LOG_LR = hp.HParam('log_lr', hp.RealInterval(-10., -7.))\nHP_LR_PATIENCE = hp.HParam('lr_patience', hp.Discrete([5]))\nHP_LR_FACTOR = hp.HParam('lr_factor', hp.RealInterval(0.1, 0.7))\nHP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete(['adam']))\nHP_EARLYSTOP = hp.HParam('earlystop_patience', hp.Discrete([2]))\n\nhparams = [HP_NUM_UNITS_0,\n           HP_UNITS_1_FACTOR,\n           HP_UNITS_2_FACTOR,\n           HP_UNITS_3_FACTOR,\n           HP_LSTM_UNITS,\n           HP_DROPOUT,\n           HP_NORM,\n           HP_ACTIVATION,\n           HP_BATCHSIZE,\n           HP_LOG_LR,\n           HP_LR_PATIENCE,\n           HP_LR_FACTOR,\n           HP_OPTIMIZER,\n           HP_EARLYSTOP,\n          ]\n\nmetrics = [hp.Metric('epoch_loss', group = 'validation', display_name='loss (val)'),\n           hp.Metric('epoch_accuracy', group = 'validation', display_name='acc (val)'),\n          ]\n\nwith tf.summary.create_file_writer('logs/hparam_tuning/').as_default():\n  hp.hparams_config(hparams = hparams, metrics = metrics)\n\ndef sample_hparams():\n    sample =  {h.name: h.domain.sample_uniform() for h in hparams}\n    units_0 = sample['num_units_0']\n    units_1 = int(units_0 * sample['units_1_f'])\n    units_2 = int(units_1 * sample['units_2_f'])\n    units_3 = int(units_2 * sample['units_3_f'])\n    sample['encoder_units'] = [\n        units_0,\n        units_1,\n        units_2,\n        units_3,\n    ]\n    return sample","metadata":{"execution":{"iopub.status.busy":"2023-02-28T10:37:04.746326Z","iopub.execute_input":"2023-02-28T10:37:04.746756Z","iopub.status.idle":"2023-02-28T10:37:04.789177Z","shell.execute_reply.started":"2023-02-28T10:37:04.746719Z","shell.execute_reply":"2023-02-28T10:37:04.787686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run_trial(name, hparams, epochs, return_model=False):\n    batched_ds = ds.batch(hparams['batch_size'])\n    val_size = int(100000 / hparams['batch_size'] * 0.1)\n    val_ds = batched_ds.take(val_size).cache('/tmp/cached_val_dataset')\n    train_ds = batched_ds.skip(val_size).cache('/tmp/cached_dataset').prefetch(tf.data.AUTOTUNE)\n    \n    m = get_model(hparams)\n    print(m.summary(expand_nested=True))\n    \n    lr = np.exp(hparams['log_lr'])\n    optimizer = tf.keras.optimizers.get(\n        {'class_name': hparams['optimizer'],\n         'config': {\n             'lr':lr,\n         },\n        }\n    )\n    m.compile(optimizer=optimizer, loss='sparse_categorical_crossentropy', metrics = 'accuracy')\n    hparams.pop('encoder_units')\n    m.fit(train_ds, validation_data=val_ds, callbacks = get_callbacks(name, hparams), epochs=epochs)\n    if return_model: return m.get_metrics_result, m\n    return m.get_metrics_result()","metadata":{"execution":{"iopub.status.busy":"2023-02-28T10:37:07.115812Z","iopub.execute_input":"2023-02-28T10:37:07.116839Z","iopub.status.idle":"2023-02-28T10:37:07.130300Z","shell.execute_reply.started":"2023-02-28T10:37:07.116791Z","shell.execute_reply":"2023-02-28T10:37:07.128640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%tensorboard --logdir {LOGDIR} --bind_all","metadata":{"execution":{"iopub.status.busy":"2023-02-28T10:37:07.939843Z","iopub.execute_input":"2023-02-28T10:37:07.940261Z","iopub.status.idle":"2023-02-28T10:37:15.993204Z","shell.execute_reply.started":"2023-02-28T10:37:07.940224Z","shell.execute_reply":"2023-02-28T10:37:15.991426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"default_hparams = {\n    'encoder_units': [1024,512,256,0],\n    'num_units_0': 1024,\n    'units_1_f': 1/2,\n    'units_2_f': 1/2,\n    'lstm_units': 512,\n    'encoder_dropout': 0,\n    'encoder_normalization': 'none',\n    'encoder_activation': 'relu',\n    'batch_size': 512,\n    'log_lr': -8,\n    'lr_factor':0.5,\n    'lr_patience': 3,\n    'optimizer': 'adam',\n    'earlystop_patience': 2,\n}","metadata":{"execution":{"iopub.status.busy":"2023-02-28T10:37:15.996123Z","iopub.execute_input":"2023-02-28T10:37:15.996570Z","iopub.status.idle":"2023-02-28T10:37:16.006655Z","shell.execute_reply.started":"2023-02-28T10:37:15.996524Z","shell.execute_reply":"2023-02-28T10:37:16.002988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Example of how we can run a training trial","metadata":{}},{"cell_type":"code","source":"run_trial('test',default_hparams,1)","metadata":{"execution":{"iopub.status.busy":"2023-02-28T10:37:33.187935Z","iopub.execute_input":"2023-02-28T10:37:33.188544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}