{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<a id=\"contents\"></a>\n# Contents\n1. [Load a predefined dataset](#section-one)\n2. [Define the model](#section-two)\n3. [Training](#section-three)\n4. [Conversion to TFLite](#section-four)\n5. [Submission](#section-five)\n6. [Sample predictions](#section-six)","metadata":{"papermill":{"duration":0.006821,"end_time":"2023-03-06T23:39:54.939891","exception":false,"start_time":"2023-03-06T23:39:54.93307","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# As always, we need to do our imports first\n%matplotlib inline\nimport tensorflow as tf\nfrom tensorflow.keras import layers, optimizers\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport mpl_toolkits.mplot3d.axes3d as p3\nfrom matplotlib import animation\nfrom IPython.display import HTML\nimport json\nimport os\n\nBASE_PATH = \"/kaggle/input/asl-signs\"\nDATASET_PATH = \"/kaggle/input/saved-tfdataset-of-google-isl-recognition-data\"\nBATCH_SIZE = 512\nSAMPLES = 94477\n\n# and define some constants\n# We will drop most of the face landmarks and the leg pose (or optionally all of them)\n# to reduce the dimensionality of the data\nUSE_Z = True\nUSE_POSE = False\nUSE_FACE = True\nRAGGED = False\n\n# THE ORDER LEFT HAND -> FACE -> POSE -> RIGHT HAND IS HEREBY SACRED AND UNCHANGEABLE\nL_LANDMARKS = list(range(468,489))\nFACE_LANDMARKS = [0,1,7,11,13,14,17,73,118,180,185,199,223,249,287,303,347,404,409,443] if USE_FACE else []\nPOSE_LANDMARKS = list(range(489,522))[11:23] if USE_POSE else []\nR_LANDMARKS = list(range(522,543))\nDATA_PATH = DATASET_PATH + \"/GoogleISLDatasetBatched\"\n\nDS_CARDINALITY = SAMPLES // BATCH_SIZE + 1\n\nVAL_SIZE  = DS_CARDINALITY // 10\nN_SIGNS = 250\nROWS_PER_FRAME = 543\n\nchannels = 3 if USE_Z else 2","metadata":{"papermill":{"duration":7.10775,"end_time":"2023-03-06T23:40:02.054688","exception":false,"start_time":"2023-03-06T23:39:54.946938","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-08T21:34:24.137261Z","iopub.execute_input":"2023-03-08T21:34:24.137592Z","iopub.status.idle":"2023-03-08T21:34:31.802286Z","shell.execute_reply.started":"2023-03-08T21:34:24.137563Z","shell.execute_reply":"2023-03-08T21:34:31.80144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata = pd.read_csv(BASE_PATH + \"/train.csv\")\nwith open(BASE_PATH + \"/sign_to_prediction_index_map.json\") as f:\n    sign_map = json.load(f)\nsign_list = list(sign_map.keys())","metadata":{"papermill":{"duration":0.216585,"end_time":"2023-03-06T23:40:02.278579","exception":false,"start_time":"2023-03-06T23:40:02.061994","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-08T21:35:40.77654Z","iopub.execute_input":"2023-03-08T21:35:40.77692Z","iopub.status.idle":"2023-03-08T21:35:40.951264Z","shell.execute_reply.started":"2023-03-08T21:35:40.776887Z","shell.execute_reply":"2023-03-08T21:35:40.950321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"section-one\"></a>\n# 1. Loading the data\n\nWe use a saved tf.Dataset containing ragged batches of the competition data, generated in the same way that the submission data is loaded.","metadata":{"papermill":{"duration":0.006838,"end_time":"2023-03-06T23:40:02.292707","exception":false,"start_time":"2023-03-06T23:40:02.285869","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def my_mean(x):\n    nonzero = tf.ragged.boolean_mask(x, x[...,0] != 0.)\n    return tf.reduce_mean(nonzero, axis=(1,2), keepdims = True)\n    \ndef my_std(x):\n    nonzero = tf.ragged.boolean_mask(x, x[...,0] != 0)\n    return tf.math.reduce_std(nonzero, axis=(1,2), keepdims = True)\n\ndef flatten_features(x, n_features):\n    return tf.concat([x[...,i,:] for i in range(n_features)], -1)\n\ndef preprocess(X, Y):\n    \n    X = X[...,:channels]\n    \n    X = tf.where(tf.math.is_nan(X), tf.zeros_like(X), X)\n    \n    l_hand = tf.gather(X, L_LANDMARKS, axis=2)\n    r_hand = tf.gather(X, R_LANDMARKS, axis=2)\n    d_hand = l_hand + r_hand # only one hand is ever labeled in a single frame\n\n    l_visible = l_hand[...,0,1] != 0.\n    r_visible = r_hand[...,0,1] != 0.\n    \n    l_pct = tf.reduce_mean(tf.cast(l_visible, tf.float32), axis=-1, keepdims=True)\n    r_pct = tf.reduce_mean(tf.cast(r_visible, tf.float32), axis=-1, keepdims=True)\n    l_dominant = l_pct > r_pct\n    \n    cond = (l_dominant & l_visible) | (tf.math.logical_not(l_dominant) & r_visible)\n    X = tf.ragged.boolean_mask(X, cond)\n    d_hand = tf.ragged.boolean_mask(d_hand, cond)\n    \n    l_x = X[...,0]\n    zero_x = tf.zeros_like(l_x)\n    l_x = tf.where(tf.expand_dims(l_dominant,-1), l_x, zero_x) \n    X = X - 2 * tf.stack([l_x, zero_x, zero_x], -1)\n    \n    sections = [d_hand]\n    lengths = [len(L_LANDMARKS)]\n    if USE_POSE:\n        pose = tf.gather(X, POSE_LANDMARKS, axis=2)\n        sections.append(pose)\n        lengths.append(len(POSE_LANDMARKS))\n    if USE_FACE:\n        face = tf.gather(X, FACE_LANDMARKS, axis=2)\n        sections.append(face)\n        lengths.append(len(FACE_LANDMARKS))\n\n    # concatenating ragged tensors on something else than the last dimension loses shape information (why??)\n    X = tf.concat([flatten_features(xyz, l) for xyz, l in zip(sections, lengths)], -1)\n    X = tf.stack([X[...,i::channels] for i in range(channels)], -1)\n    \n    mu = my_mean(X)\n    sigma = my_std(X)\n    \n    X = (X - mu) / sigma\n    X = flatten_features(X, sum(lengths))\n    \n    return X, Y","metadata":{"papermill":{"duration":0.02644,"end_time":"2023-03-06T23:40:02.326166","exception":false,"start_time":"2023-03-06T23:40:02.299726","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-08T21:33:20.213759Z","iopub.status.idle":"2023-03-08T21:33:20.214113Z","shell.execute_reply.started":"2023-03-08T21:33:20.213944Z","shell.execute_reply":"2023-03-08T21:33:20.213961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# optional data augmentation\ndef augment(ragged_batch, labels):\n    \n    # additive gaussian noise\n    add_noise = lambda x: x + tf.random.normal(tf.shape(ragged_batch.flat_values), 0, 0.1)\n    ragged_batch = tf.ragged.map_flat_values(add_noise, ragged_batch)\n    \n    return ragged_batch, labels","metadata":{"papermill":{"duration":0.015718,"end_time":"2023-03-06T23:40:02.395222","exception":false,"start_time":"2023-03-06T23:40:02.379504","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-08T21:33:20.215272Z","iopub.status.idle":"2023-03-08T21:33:20.215558Z","shell.execute_reply.started":"2023-03-08T21:33:20.215411Z","shell.execute_reply":"2023-03-08T21:33:20.215426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pad_ragged(x,y):\n    return x.to_tensor(), y","metadata":{"papermill":{"duration":0.014869,"end_time":"2023-03-06T23:40:02.416957","exception":false,"start_time":"2023-03-06T23:40:02.402088","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-08T21:33:20.216649Z","iopub.status.idle":"2023-03-08T21:33:20.21695Z","shell.execute_reply.started":"2023-03-08T21:33:20.216788Z","shell.execute_reply":"2023-03-08T21:33:20.216803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the data and split to train, validation\nraw_dataset = tf.data.Dataset.load(DATA_PATH)\ndataset = raw_dataset.map(preprocess)\nval_ds = dataset.take(VAL_SIZE).cache()\ntrain_ds = dataset.skip(VAL_SIZE).cache()\n\nif not RAGGED:\n    train_ds = train_ds.map(pad_ragged)\n    val_ds = val_ds.map(pad_ragged)\n    \ntrain_ds = train_ds.prefetch(tf.data.AUTOTUNE)\nval_ds = val_ds.prefetch(tf.data.AUTOTUNE)","metadata":{"papermill":{"duration":9.75792,"end_time":"2023-03-06T23:40:12.182463","exception":false,"start_time":"2023-03-06T23:40:02.424543","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-08T21:33:20.21807Z","iopub.status.idle":"2023-03-08T21:33:20.218356Z","shell.execute_reply.started":"2023-03-08T21:33:20.21821Z","shell.execute_reply":"2023-03-08T21:33:20.218225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(dataset.element_spec)","metadata":{"papermill":{"duration":0.018201,"end_time":"2023-03-06T23:40:12.207934","exception":false,"start_time":"2023-03-06T23:40:12.189733","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-08T21:33:20.219617Z","iopub.status.idle":"2023-03-08T21:33:20.219928Z","shell.execute_reply.started":"2023-03-08T21:33:20.219761Z","shell.execute_reply":"2023-03-08T21:33:20.219777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_sample, y_sample = next(iter(val_ds))","metadata":{"papermill":{"duration":12.475807,"end_time":"2023-03-06T23:40:24.690876","exception":false,"start_time":"2023-03-06T23:40:12.215069","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-08T21:33:20.220585Z","iopub.status.idle":"2023-03-08T21:33:20.220869Z","shell.execute_reply.started":"2023-03-08T21:33:20.220713Z","shell.execute_reply":"2023-03-08T21:33:20.220727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# visualize\nn = 2\n\nproj = \"3d\" if USE_Z else None\nfig = plt.figure()\nax = fig.add_subplot(projection=proj)\ndots = ax.plot(*([[]] * channels),'.', markersize=1)[0]\ndef get_frame(data):\n    x = data[::channels]\n    y = -data[1::channels] + 2\n    if USE_Z:\n        z = data[2::channels]\n        dots.set_data((x,y))\n        dots.set_3d_properties(z)\n    dots.set_data((x,y))\n    return dots,\n\nframes = x_sample[n]\nlabel = sign_list[y_sample[n].numpy()[0]]\n\nif USE_Z:\n    ax.set(xlim3d=(-5, 5), xlabel='X')\n    ax.set(ylim3d=(-5, 5), ylabel='Y')\n    ax.set(zlim3d=(-5, 5), zlabel='Z')\nelse: \n    ax.set(xlim=(-2, 2), xlabel='X')\n    ax.set(ylim=(-2, 2), ylabel='Y')\n    ax.set_aspect(\"equal\")\n\nprint(f\"animating input features for sample of sign: {label}\")\na = animation.FuncAnimation(fig, get_frame, frames = frames.numpy())\n#plt.show()\nplt.close(fig)\nHTML(a.to_html5_video())","metadata":{"papermill":{"duration":14.087323,"end_time":"2023-03-06T23:40:39.076485","exception":false,"start_time":"2023-03-06T23:40:24.989162","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-08T21:33:20.22161Z","iopub.status.idle":"2023-03-08T21:33:20.221934Z","shell.execute_reply.started":"2023-03-08T21:33:20.221751Z","shell.execute_reply":"2023-03-08T21:33:20.221765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"section-two\"></a>\n# 2. Defining the classifier model","metadata":{"papermill":{"duration":0.008747,"end_time":"2023-03-06T23:40:39.094767","exception":false,"start_time":"2023-03-06T23:40:39.08602","status":"completed"},"tags":[]}},{"cell_type":"code","source":"ENCODER_UNITS = [1024,512]\nENCODER_DROPOUT = [0.6,0.4]\n\nHIDDEN_LSTM_UNITS = 256\nLSTM_UNITS = 512\nBILSTM = False\n\nCONV_FILTERS = LSTM_UNITS\n\nCLASSIFIER_DROPOUT = 0.6\n\nEPOCHS = 150\nSTOP_PATIENCE = 10\nINIT_LR = 5e-4\nLR_PATIENCE = 3\nLR_FACTOR = 0.8","metadata":{"papermill":{"duration":0.018773,"end_time":"2023-03-06T23:40:39.122486","exception":false,"start_time":"2023-03-06T23:40:39.103713","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-08T21:33:20.222699Z","iopub.status.idle":"2023-03-08T21:33:20.222984Z","shell.execute_reply.started":"2023-03-08T21:33:20.222843Z","shell.execute_reply":"2023-03-08T21:33:20.222858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def dense_block(units, dropout, name):\n    fc = layers.Dense(units, name=name)\n    norm = layers.BatchNormalization()\n    act = layers.Activation(\"swish\")\n    dropout = layers.Dropout(dropout)\n    if dropout: return lambda x: dropout(act(norm(fc(x))))\n    return lambda x: act(norm(fc(x)))","metadata":{"papermill":{"duration":0.018577,"end_time":"2023-03-06T23:40:39.149845","exception":false,"start_time":"2023-03-06T23:40:39.131268","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-08T21:33:20.223549Z","iopub.status.idle":"2023-03-08T21:33:20.22383Z","shell.execute_reply.started":"2023-03-08T21:33:20.223679Z","shell.execute_reply":"2023-03-08T21:33:20.223694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def classifier(lstm_units, dropout, bidirectional=False, hidden_units = 512):\n    if not bidirectional:\n        lstm = layers.LSTM(lstm_units)\n    else:\n        def lstm(x):\n            fw = layers.LSTM(hidden_units, return_sequences=True)(x)\n            bw = layers.LSTM(hidden_units, return_sequences=True, go_backwards=True)(x)\n            bidirectional = layers.Concatenate(axis=-1)([fw, bw])\n            return layers.LSTM(lstm_units)(bidirectional)\n        \n    dropout = layers.Dropout(dropout)\n    out = layers.Dense(N_SIGNS, activation=\"softmax\")\n    return lambda x: out(dropout(lstm(x)))","metadata":{"papermill":{"duration":0.019533,"end_time":"2023-03-06T23:40:39.178087","exception":false,"start_time":"2023-03-06T23:40:39.158554","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-08T21:33:20.225006Z","iopub.status.idle":"2023-03-08T21:33:20.225277Z","shell.execute_reply.started":"2023-03-08T21:33:20.22514Z","shell.execute_reply":"2023-03-08T21:33:20.225154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# placeholder, Conv1D doesn't support ragged inputs so we would need padding\ndef conv_classifier(filters, dropout, kernel_size=3):\n    conv = layers.Conv1D(filters, kernel_size, padding=\"same\")\n    pool = layers.GlobalAveragePooling1D()\n    dropout = layers.Dropout(dropout)\n    out = layers.Dense(N_SIGNS, activation=\"softmax\")\n    return lambda x: out(dropout(pool(conv(x))))    ","metadata":{"papermill":{"duration":0.017224,"end_time":"2023-03-06T23:40:39.204315","exception":false,"start_time":"2023-03-06T23:40:39.187091","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-08T21:33:20.228515Z","iopub.status.idle":"2023-03-08T21:33:20.229041Z","shell.execute_reply.started":"2023-03-08T21:33:20.228859Z","shell.execute_reply":"2023-03-08T21:33:20.228877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#define the inputs (ragged batches of time series of landmark coordinates)\ninput_channels = channels * (len(L_LANDMARKS) + len(POSE_LANDMARKS) + len(FACE_LANDMARKS))\ninputs = tf.keras.Input(shape=(None,input_channels), ragged=RAGGED)\nx = layers.Masking()(inputs)\n\nfor i, (units, dropout) in enumerate(zip(ENCODER_UNITS, ENCODER_DROPOUT)):\n    x = dense_block(units, dropout, f\"encoder_{i}\")(x)\n\n# classifier model\nout = classifier(LSTM_UNITS, CLASSIFIER_DROPOUT, bidirectional=BILSTM, hidden_units=HIDDEN_LSTM_UNITS )(x)\n# out = conv_classifier(conv_filters, classifier_dropout)(x)\n\nmodel = tf.keras.Model(inputs=inputs, outputs=out)\n\n#steps_per_epoch = DS_CARDINALITY - VAL_SIZE\n#boundaries = [steps_per_epoch * n for n in [30,50]]\n#values = [3e-4,3e-5,1e-5]\n#lr_sched = optimizers.schedules.PiecewiseConstantDecay(boundaries, values)\noptimizer = optimizers.Adam(INIT_LR)\n\nmodel.compile(optimizer=optimizer,\n              loss=\"sparse_categorical_crossentropy\",\n              metrics=[\"accuracy\",\"sparse_top_k_categorical_accuracy\"])","metadata":{"papermill":{"duration":1.167439,"end_time":"2023-03-06T23:40:40.38056","exception":false,"start_time":"2023-03-06T23:40:39.213121","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-08T21:33:20.23003Z","iopub.status.idle":"2023-03-08T21:33:20.2306Z","shell.execute_reply.started":"2023-03-08T21:33:20.230439Z","shell.execute_reply":"2023-03-08T21:33:20.230457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"papermill":{"duration":0.076179,"end_time":"2023-03-06T23:40:40.465871","exception":false,"start_time":"2023-03-06T23:40:40.389692","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-08T21:33:20.231838Z","iopub.status.idle":"2023-03-08T21:33:20.232445Z","shell.execute_reply.started":"2023-03-08T21:33:20.232285Z","shell.execute_reply":"2023-03-08T21:33:20.232303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"section-three\"></a>\n# 3. Training","metadata":{"papermill":{"duration":0.010606,"end_time":"2023-03-06T23:40:40.488156","exception":false,"start_time":"2023-03-06T23:40:40.47755","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def get_callbacks():\n    return [\n        tf.keras.callbacks.EarlyStopping(\n            monitor=\"val_loss\",\n            patience=STOP_PATIENCE,\n            restore_best_weights=True\n        ),\n        tf.keras.callbacks.ReduceLROnPlateau(\n            monitor=\"val_loss\",\n            patience=LR_PATIENCE,\n            factor=LR_FACTOR\n        )\n    ]","metadata":{"papermill":{"duration":0.019163,"end_time":"2023-03-06T23:40:40.518008","exception":false,"start_time":"2023-03-06T23:40:40.498845","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-08T21:33:20.23341Z","iopub.status.idle":"2023-03-08T21:33:20.234024Z","shell.execute_reply.started":"2023-03-08T21:33:20.233841Z","shell.execute_reply":"2023-03-08T21:33:20.233864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hist = model.fit(train_ds,\n          validation_data = val_ds,\n          callbacks = get_callbacks(),\n          epochs = EPOCHS,\n         )","metadata":{"papermill":{"duration":3608.52682,"end_time":"2023-03-07T00:40:49.055463","exception":false,"start_time":"2023-03-06T23:40:40.528643","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-08T21:33:20.235085Z","iopub.status.idle":"2023-03-08T21:33:20.235655Z","shell.execute_reply.started":"2023-03-08T21:33:20.235496Z","shell.execute_reply":"2023-03-08T21:33:20.235513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#ypreds = model.predict(val_ds)\n#_ = plt.hist(np.argmax(ypreds, axis=1), bins = range(1,251))\n#plt.show()","metadata":{"papermill":{"duration":1.018625,"end_time":"2023-03-07T00:40:51.004585","exception":false,"start_time":"2023-03-07T00:40:49.98596","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-08T21:33:20.236633Z","iopub.status.idle":"2023-03-08T21:33:20.237246Z","shell.execute_reply.started":"2023-03-08T21:33:20.237094Z","shell.execute_reply":"2023-03-08T21:33:20.23711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"section-four\"></a>\n# 4. TFLite inference model and conversion","metadata":{"papermill":{"duration":1.003587,"end_time":"2023-03-07T00:40:52.937708","exception":false,"start_time":"2023-03-07T00:40:51.934121","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def inference_mean(x):\n    nonzero = tf.boolean_mask(x, x[...,0] != 0.)\n    return tf.reduce_mean(nonzero, axis = 0, keepdims = True)\n    \ndef inference_std(x):\n    nonzero = tf.boolean_mask(x, x[...,0] != 0)\n    return tf.math.reduce_std(nonzero, axis = 0, keepdims = True)\n\ndef inference_flatten_features(x, n_features):\n    return tf.reshape(x, (-1,n_features * channels))\n\ndef inference_preprocess(X):\n    \n    X = X[...,:channels]\n    l_hand = tf.gather(X, L_LANDMARKS, axis=1)\n    r_hand = tf.gather(X, R_LANDMARKS, axis=1)\n    d_hand = l_hand + r_hand # only one hand is ever labeled in a single frame\n\n    l_visible = l_hand[...,0,1] != 0.\n    r_visible = r_hand[...,0,1] != 0.\n    \n    l_pct = tf.reduce_mean(tf.cast(l_visible, tf.float32), axis=-1, keepdims=True)\n    r_pct = tf.reduce_mean(tf.cast(r_visible, tf.float32), axis=-1, keepdims=True)\n    l_dominant = l_pct > r_pct\n    \n    cond = (l_dominant & l_visible) | (tf.math.logical_not(l_dominant) & r_visible)\n    X = tf.boolean_mask(X, cond)\n    d_hand = tf.boolean_mask(d_hand, cond)\n    \n    l_x = X[...,0]\n    zero_x = tf.zeros_like(l_x)\n    l_x = tf.where(tf.expand_dims(l_dominant,-1), l_x, zero_x) \n    X = X - 2 * tf.stack([l_x, zero_x, zero_x], -1)\n    \n    sections = [d_hand]\n    lengths = [len(L_LANDMARKS)]\n    if USE_POSE:\n        pose = tf.gather(X, POSE_LANDMARKS, axis=1)\n        sections.append(pose)\n        lengths.append(len(POSE_LANDMARKS))\n    if USE_FACE:\n        face = tf.gather(X, FACE_LANDMARKS, axis=1)\n        sections.append(face)\n        lengths.append(len(FACE_LANDMARKS))\n\n    # concatenating ragged tensors on something else than the last dimension loses shape information (why??)\n    X = tf.concat([flatten_features(xyz, l) for xyz, l in zip(sections, lengths)], -1)\n    X = tf.stack([X[...,i::channels] for i in range(channels)], -1)\n    \n    print(X.shape)\n    mu = inference_mean(X)\n    sigma = inference_std(X)\n    \n    X = (X - mu) / sigma\n    X = flatten_features(X, sum(lengths))\n    \n    return X","metadata":{"papermill":{"duration":0.952426,"end_time":"2023-03-07T00:40:54.889753","exception":false,"start_time":"2023-03-07T00:40:53.937327","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-08T21:33:20.238343Z","iopub.status.idle":"2023-03-08T21:33:20.238966Z","shell.execute_reply.started":"2023-03-08T21:33:20.23878Z","shell.execute_reply":"2023-03-08T21:33:20.238799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_inference_model(model):\n    \n\n    inputs = tf.keras.Input(shape=(543,3), name=\"inputs\")\n    \n    # fill nan\n    x = tf.where(tf.math.is_nan(inputs), tf.zeros_like(inputs), inputs)\n\n    x = inference_preprocess(x)\n\n    x = tf.expand_dims(x,0)\n    \n    # call trained model\n    out = model(x)\n    \n    # explicitly name the final (identity) layer for the submission format\n    out = layers.Activation(\"linear\", name=\"outputs\")(out)\n    \n    inference_model = tf.keras.Model(inputs=inputs, outputs=out)\n    inference_model.compile(loss=\"sparse_categorical_crossentropy\",\n                            metrics=\"accuracy\")\n    return inference_model","metadata":{"papermill":{"duration":1.029864,"end_time":"2023-03-07T00:40:56.912401","exception":false,"start_time":"2023-03-07T00:40:55.882537","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-08T21:33:20.240006Z","iopub.status.idle":"2023-03-08T21:33:20.240617Z","shell.execute_reply.started":"2023-03-08T21:33:20.240453Z","shell.execute_reply":"2023-03-08T21:33:20.240471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inference_model = get_inference_model(model)\ninference_model.summary(expand_nested=True)","metadata":{"papermill":{"duration":3.193708,"end_time":"2023-03-07T00:41:01.324737","exception":false,"start_time":"2023-03-07T00:40:58.131029","status":"completed"},"scrolled":true,"tags":[],"execution":{"iopub.status.busy":"2023-03-08T21:33:20.241642Z","iopub.status.idle":"2023-03-08T21:33:20.24246Z","shell.execute_reply.started":"2023-03-08T21:33:20.242301Z","shell.execute_reply":"2023-03-08T21:33:20.242318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"section-five\"></a>\n# 5. Generating submission","metadata":{"papermill":{"duration":0.95714,"end_time":"2023-03-07T00:41:03.329509","exception":false,"start_time":"2023-03-07T00:41:02.372369","status":"completed"},"tags":[]}},{"cell_type":"code","source":"converter = tf.lite.TFLiteConverter.from_keras_model(inference_model)\ntflite_model = converter.convert()\nmodel_path = \"model.tflite\"\n\nwith open(model_path, 'wb') as f:\n    f.write(tflite_model)\n!zip submission.zip $model_path","metadata":{"papermill":{"duration":25.382141,"end_time":"2023-03-07T00:41:29.733898","exception":false,"start_time":"2023-03-07T00:41:04.351757","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-08T21:33:20.243516Z","iopub.status.idle":"2023-03-08T21:33:20.244121Z","shell.execute_reply.started":"2023-03-08T21:33:20.243944Z","shell.execute_reply":"2023-03-08T21:33:20.243965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"section-six\"></a>\n# 6. Demonstrate TFLite inference","metadata":{"papermill":{"duration":1.027736,"end_time":"2023-03-07T00:41:31.936155","exception":false,"start_time":"2023-03-07T00:41:30.908419","status":"completed"},"tags":[]}},{"cell_type":"code","source":"data_dir = BASE_PATH + \"/\"\ndef load_relevant_data_subset(pq_path):\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(data_dir + pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)","metadata":{"papermill":{"duration":1.03608,"end_time":"2023-03-07T00:41:33.932492","exception":false,"start_time":"2023-03-07T00:41:32.896412","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-08T21:34:42.277436Z","iopub.execute_input":"2023-03-08T21:34:42.278171Z","iopub.status.idle":"2023-03-08T21:34:42.284406Z","shell.execute_reply.started":"2023-03-08T21:34:42.278135Z","shell.execute_reply":"2023-03-08T21:34:42.283337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install tflite-runtime\nimport tflite_runtime.interpreter as tflite\nfrom tqdm import tqdm","metadata":{"papermill":{"duration":152.016722,"end_time":"2023-03-07T00:44:06.966911","exception":true,"start_time":"2023-03-07T00:41:34.950189","status":"failed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-08T21:34:44.231966Z","iopub.execute_input":"2023-03-08T21:34:44.232318Z","iopub.status.idle":"2023-03-08T21:34:52.956591Z","shell.execute_reply.started":"2023-03-08T21:34:44.232286Z","shell.execute_reply":"2023-03-08T21:34:52.954447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_ds\ndel val_ds","metadata":{"execution":{"iopub.status.busy":"2023-03-08T21:34:52.95961Z","iopub.execute_input":"2023-03-08T21:34:52.960145Z","iopub.status.idle":"2023-03-08T21:34:53.193155Z","shell.execute_reply.started":"2023-03-08T21:34:52.960105Z","shell.execute_reply":"2023-03-08T21:34:53.191291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"interpreter = tflite.Interpreter(model_path)\nfound_signatures = list(interpreter.get_signature_list().keys())\n# if REQUIRED_SIGNATURE not in found_signatures:\n#     raise KernelEvalException('Required input signature not found.')\nprediction_fn = interpreter.get_signature_runner(\"serving_default\")\n\nn_vis = 5000\ny_trues = metadata.sign[:n_vis].map(sign_map).values\ny_preds = np.zeros(n_vis)\nlengths = np.zeros(n_vis)\nfor i in tqdm(range(n_vis)):\n    data = load_relevant_data_subset(metadata.path[i])\n    lengths[i] = len(data)\n    output = prediction_fn(inputs=data)\n    y_preds[i] = np.argmax(output[\"outputs\"])","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[],"execution":{"iopub.status.busy":"2023-03-08T21:35:47.15152Z","iopub.execute_input":"2023-03-08T21:35:47.151917Z","iopub.status.idle":"2023-03-08T21:38:17.573747Z","shell.execute_reply.started":"2023-03-08T21:35:47.15188Z","shell.execute_reply":"2023-03-08T21:38:17.572306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"signer = metadata.participant_id","metadata":{"execution":{"iopub.status.busy":"2023-03-08T21:38:23.733632Z","iopub.execute_input":"2023-03-08T21:38:23.734799Z","iopub.status.idle":"2023-03-08T21:38:23.73911Z","shell.execute_reply.started":"2023-03-08T21:38:23.73472Z","shell.execute_reply":"2023-03-08T21:38:23.738099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"correct_preds = y_preds == y_trues\nh_all, bins = np.histogram(lengths, bins=25)\nh_correct, _ = np.histogram(lengths[correct_preds], bins=bins)\nfig, ax1 = plt.subplots()\nax1.stairs(h_all, bins)\nax1.stairs(h_correct, bins)\nax1.set_yscale(\"log\")\nax1.set_xlabel('sequence length')\nax1.set_ylabel('Count')\n\n\nax2 = ax1.twinx()\ncolor = 'tab:red'\nwidth = bins[1] - bins[0]\nax2.plot(bins[1:] - width/2, h_correct / h_all, color=color)\nax2.tick_params(axis='y', labelcolor=color)\nax2.set_ylabel('proportion of correct predictions', color=color)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-08T21:38:27.371215Z","iopub.execute_input":"2023-03-08T21:38:27.371558Z","iopub.status.idle":"2023-03-08T21:38:27.927875Z","shell.execute_reply.started":"2023-03-08T21:38:27.371528Z","shell.execute_reply":"2023-03-08T21:38:27.926666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure()\nsigner_counts = signer[:n_vis].value_counts()\nsigner_corrects = signer[:n_vis][correct_preds].value_counts()\nsigner_counts.plot(kind=\"bar\", color=\"red\")\nsigner_corrects.plot(kind=\"bar\")\nproportion = signer_corrects / signer_counts\nplt.show()\nplt.figure()\n(signer_corrects / signer_counts).plot(kind=\"bar\")\nplt.show","metadata":{"execution":{"iopub.status.busy":"2023-03-08T21:38:43.339519Z","iopub.execute_input":"2023-03-08T21:38:43.339877Z","iopub.status.idle":"2023-03-08T21:38:43.844728Z","shell.execute_reply.started":"2023-03-08T21:38:43.339843Z","shell.execute_reply":"2023-03-08T21:38:43.844055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print((signer_corrects / signer_counts).sort_values())","metadata":{"execution":{"iopub.status.busy":"2023-03-08T21:38:43.909598Z","iopub.execute_input":"2023-03-08T21:38:43.911781Z","iopub.status.idle":"2023-03-08T21:38:43.921732Z","shell.execute_reply.started":"2023-03-08T21:38:43.911715Z","shell.execute_reply":"2023-03-08T21:38:43.919864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"validation accuracy: {np.mean(correct_preds)}\")\nconfmat = tf.math.confusion_matrix(y_trues,y_preds)\nplt.subplots(figsize=(11,11))\nplt.imshow(confmat, cmap=\"binary\")","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[],"execution":{"iopub.status.busy":"2023-03-08T21:38:47.65315Z","iopub.execute_input":"2023-03-08T21:38:47.653543Z","iopub.status.idle":"2023-03-08T21:38:48.24776Z","shell.execute_reply.started":"2023-03-08T21:38:47.653506Z","shell.execute_reply":"2023-03-08T21:38:48.247102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}