{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# import libraries\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nfrom tensorflow.keras import layers, optimizers","metadata":{"execution":{"iopub.status.busy":"2023-03-17T17:50:21.838297Z","iopub.execute_input":"2023-03-17T17:50:21.838591Z","iopub.status.idle":"2023-03-17T17:50:31.615614Z","shell.execute_reply.started":"2023-03-17T17:50:21.838563Z","shell.execute_reply":"2023-03-17T17:50:31.614526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# set files directories\nLANDMARK_FILES_DIR = \"/kaggle/input/asl-signs/train_landmark_files\"\nTRAIN_FILE = \"/kaggle/input/asl-signs/train.csv\"","metadata":{"execution":{"iopub.status.busy":"2023-03-17T17:51:13.174110Z","iopub.execute_input":"2023-03-17T17:51:13.175224Z","iopub.status.idle":"2023-03-17T17:51:13.180581Z","shell.execute_reply.started":"2023-03-17T17:51:13.175183Z","shell.execute_reply":"2023-03-17T17:51:13.179142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read the data, the type of the data\nsample = pd.read_parquet(\"/kaggle/input/asl-signs/train_landmark_files/16069/100015657.parquet\")\nsample.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-17T17:51:32.129637Z","iopub.execute_input":"2023-03-17T17:51:32.130325Z","iopub.status.idle":"2023-03-17T17:51:32.331382Z","shell.execute_reply.started":"2023-03-17T17:51:32.130287Z","shell.execute_reply":"2023-03-17T17:51:32.329978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pick the left hand and right hand points\nsample_left_hand = sample[sample.type == \"left_hand\"]\nsample_right_hand = sample[sample.type == \"right_hand\"]\n\n# edges that represents the hand edges\nedges = [(0,1),(1,2),(2,3),(3,4),(0,5),(0,17),(5,6),(6,7),(7,8),(5,9),(9,10),(10,11),(11,12),\n         (9,13),(13,14),(14,15),(15,16),(13,17),(17,18),(18,19),(19,20)]\n\n# plotting a single frame into matplotlib\ndef plot_frame(df, frame_id, ax):\n    df = df[df.frame == frame_id].sort_values(['landmark_index'])\n    x = list(df.x)\n    y = list(df.y)\n    \n    # plotting the points\n    ax.scatter(df.x, df.y, color='dodgerblue')\n    for i in range(len(x)):\n        ax.text(x[i], y[i], str(i))\n    \n    # plotting the edges that represents the hand\n    for edge in edges:\n        ax.plot([x[edge[0]], x[edge[1]]], [y[edge[0]], y[edge[1]]], color='salmon')\n        ax.set_xlabel(f\"Frame no. {frame_id}\")\n        ax.set_xticks([])\n        ax.set_yticks([])\n        ax.set_xticklabels([])\n        ax.set_yticklabels([])\n\n# plotting the multiple frames\ndef plot_frame_seq(df, frame_range, n_frames):\n    frames = np.linspace(frame_range[0],frame_range[1],n_frames, dtype = int, endpoint=True)\n    fig, ax = plt.subplots(n_frames, 1, figsize=(5,25))\n    for i in range(n_frames):\n        plot_frame(df, frames[i], ax[i])\n        \n    plt.show()\n\nplot_frame_seq(sample_left_hand, (178,186), 10)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T17:51:37.800997Z","iopub.execute_input":"2023-03-17T17:51:37.801935Z","iopub.status.idle":"2023-03-17T17:51:39.243907Z","shell.execute_reply.started":"2023-03-17T17:51:37.801897Z","shell.execute_reply":"2023-03-17T17:51:39.242804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Load Data**","metadata":{}},{"cell_type":"code","source":"# Set constants and pick important landmarks\nLANDMARK_IDX = [0,9,11,13,14,17,117,118,119,199,346,347,348] + list(range(468,543))\nDATA_PATH = \"/kaggle/input/saved-tfdataset-of-google-isl-recognition-data/GoogleISLDatasetBatched\"\nDS_CARDINALITY = 185\nVAL_SIZE  = 20\nN_SIGNS = 250\nROWS_PER_FRAME = 543","metadata":{"execution":{"iopub.status.busy":"2023-03-17T17:52:28.197395Z","iopub.execute_input":"2023-03-17T17:52:28.197783Z","iopub.status.idle":"2023-03-17T17:52:28.206325Z","shell.execute_reply.started":"2023-03-17T17:52:28.197749Z","shell.execute_reply":"2023-03-17T17:52:28.205291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess(ragged_batch, labels):\n    ragged_batch = tf.gather(ragged_batch, LANDMARK_IDX, axis=2)\n    ragged_batch = tf.where(tf.math.is_nan(ragged_batch), tf.zeros_like(ragged_batch), ragged_batch)\n    return tf.concat([ragged_batch[...,i] for i in range(3)],-1), labels\n\ndataset = tf.data.Dataset.load(DATA_PATH)\ndataset = dataset.map(preprocess)\nval_ds = dataset.take(VAL_SIZE).cache().prefetch(tf.data.AUTOTUNE)\ntrain_ds = dataset.skip(VAL_SIZE).cache().shuffle(20).prefetch(tf.data.AUTOTUNE)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T17:52:30.539746Z","iopub.execute_input":"2023-03-17T17:52:30.540188Z","iopub.status.idle":"2023-03-17T17:52:35.476387Z","shell.execute_reply.started":"2023-03-17T17:52:30.540150Z","shell.execute_reply":"2023-03-17T17:52:35.475337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Train Model**","metadata":{}},{"cell_type":"code","source":"# include early stopping and reducelr\ndef get_callbacks():\n    return [\n            tf.keras.callbacks.EarlyStopping(\n            monitor=\"val_accuracy\",\n            patience=10,\n            restore_best_weights=True\n        ),\n        tf.keras.callbacks.ReduceLROnPlateau(\n            monitor = \"val_accuracy\",\n            factor = 0.5,\n            patience = 3\n        ),\n    ]\n\n# a single dense block followed by a normalization block and relu activation\ndef dense_block(units, name):\n    fc = layers.Dense(units)\n    norm = layers.LayerNormalization()\n    act = layers.Activation(\"relu\")\n    return lambda x: act(norm(fc(x)))\n\n# the lstm block with the final dense block for the classification\ndef classifier(lstm_units):\n    lstm = layers.LSTM(lstm_units)\n    out = layers.Dense(N_SIGNS, activation=\"softmax\")\n    return lambda x: out(lstm(x))","metadata":{"execution":{"iopub.status.busy":"2023-03-17T17:52:59.495543Z","iopub.execute_input":"2023-03-17T17:52:59.496566Z","iopub.status.idle":"2023-03-17T17:52:59.504816Z","shell.execute_reply.started":"2023-03-17T17:52:59.496527Z","shell.execute_reply":"2023-03-17T17:52:59.503492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# choose the number of nodes per layer\nencoder_units = [512, 256] # tune this\nlstm_units = 250 # tune this\n\n#define the inputs (ragged batches of time series of landmark coordinates)\ninputs = tf.keras.Input(shape=(None,3*len(LANDMARK_IDX)), ragged=True)\n\n# dense encoder model\nx = inputs\nfor i, n in enumerate(encoder_units):\n    x = dense_block(n, f\"encoder_{i}\")(x)\n    \nx = layers.Dropout(0.1)(x)\n\n# classifier model\nout = classifier(lstm_units)(x)\n\nmodel = tf.keras.Model(inputs=inputs, outputs=out)\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-03-17T17:53:42.596823Z","iopub.execute_input":"2023-03-17T17:53:42.597576Z","iopub.status.idle":"2023-03-17T17:53:43.135121Z","shell.execute_reply.started":"2023-03-17T17:53:42.597536Z","shell.execute_reply":"2023-03-17T17:53:43.134319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# add a decreasing learning rate scheduler to help convergence\nsteps_per_epoch = DS_CARDINALITY - VAL_SIZE\nboundaries = [steps_per_epoch * n for n in [30,50,70]]\nvalues = [1e-3,1e-4,1e-5,1e-6]\nlr_sched = optimizers.schedules.PiecewiseConstantDecay(boundaries, values)\noptimizer = optimizers.Adam(lr_sched)\n\nmodel.compile(optimizer=optimizer,\n              loss=\"sparse_categorical_crossentropy\",\n              metrics=[\"accuracy\",\"sparse_top_k_categorical_accuracy\"])","metadata":{"execution":{"iopub.status.busy":"2023-03-17T17:53:48.013669Z","iopub.execute_input":"2023-03-17T17:53:48.014844Z","iopub.status.idle":"2023-03-17T17:53:48.072437Z","shell.execute_reply.started":"2023-03-17T17:53:48.014797Z","shell.execute_reply":"2023-03-17T17:53:48.071517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fit the model with 100 epochs iteration\nmodel.fit(train_ds,\n          validation_data = val_ds,\n          callbacks = get_callbacks(),\n          epochs = 100)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T17:53:54.513042Z","iopub.execute_input":"2023-03-17T17:53:54.513430Z","iopub.status.idle":"2023-03-17T18:55:10.703836Z","shell.execute_reply.started":"2023-03-17T17:53:54.513396Z","shell.execute_reply":"2023-03-17T18:55:10.702845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Submit Model**","metadata":{}},{"cell_type":"code","source":"model.summary(expand_nested=True)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T18:57:06.143167Z","iopub.execute_input":"2023-03-17T18:57:06.143548Z","iopub.status.idle":"2023-03-17T18:57:06.172935Z","shell.execute_reply.started":"2023-03-17T18:57:06.143515Z","shell.execute_reply":"2023-03-17T18:57:06.172166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# add a decreasing learning rate scheduler to help convergence\nsteps_per_epoch = DS_CARDINALITY - VAL_SIZE\nboundaries = [steps_per_epoch * n for n in [30,50,70]]\nvalues = [1e-3,1e-4,1e-5,1e-6]\nlr_sched = optimizers.schedules.PiecewiseConstantDecay(boundaries, values)\noptimizer = optimizers.Adam(lr_sched)\n\nmodel.compile(optimizer=optimizer,\n              loss=\"sparse_categorical_crossentropy\",\n              metrics=[\"accuracy\",\"sparse_top_k_categorical_accuracy\"])","metadata":{"execution":{"iopub.status.busy":"2023-03-17T18:58:23.410029Z","iopub.execute_input":"2023-03-17T18:58:23.410649Z","iopub.status.idle":"2023-03-17T18:58:23.431884Z","shell.execute_reply.started":"2023-03-17T18:58:23.410591Z","shell.execute_reply":"2023-03-17T18:58:23.430764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_inference_model(model):\n    inputs = tf.keras.Input(shape=(ROWS_PER_FRAME,3), name=\"inputs\")\n    \n    # drop most of the face mesh\n    x = tf.gather(inputs, LANDMARK_IDX, axis=1)\n\n    # fill nan\n    x = tf.where(tf.math.is_nan(x), tf.zeros_like(x), x)\n\n    # flatten landmark xyz coordinates ()\n    x = tf.concat([x[...,i] for i in range(3)], -1)\n\n    x = tf.expand_dims(x,0)\n    \n    # call trained model\n    out = model(x)\n    \n    # explicitly name the final (identity) layer for the submission format\n    out = layers.Activation(\"linear\", name=\"outputs\")(out)\n    \n    inference_model = tf.keras.Model(inputs=inputs, outputs=out)\n    inference_model.compile(loss=\"sparse_categorical_crossentropy\",\n                            metrics=\"accuracy\")\n    return inference_model","metadata":{"execution":{"iopub.status.busy":"2023-03-17T19:11:22.723277Z","iopub.execute_input":"2023-03-17T19:11:22.723987Z","iopub.status.idle":"2023-03-17T19:11:22.738115Z","shell.execute_reply.started":"2023-03-17T19:11:22.723945Z","shell.execute_reply":"2023-03-17T19:11:22.737323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inference_model = get_inference_model(model)\ninference_model.summary(expand_nested=True)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T19:26:37.585936Z","iopub.execute_input":"2023-03-17T19:26:37.586555Z","iopub.status.idle":"2023-03-17T19:26:37.974409Z","shell.execute_reply.started":"2023-03-17T19:26:37.586513Z","shell.execute_reply":"2023-03-17T19:26:37.973651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save the model\nconverter = tf.lite.TFLiteConverter.from_keras_model(inference_model)\ntflite_model = converter.convert()\nmodel_path = \"model.tflite\"\n\n# submit the model\nwith open(model_path, 'wb') as f:\n    f.write(tflite_model)\n!zip submission.zip $model_path","metadata":{"execution":{"iopub.status.busy":"2023-03-17T19:26:44.069426Z","iopub.execute_input":"2023-03-17T19:26:44.069972Z","iopub.status.idle":"2023-03-17T19:26:56.661599Z","shell.execute_reply.started":"2023-03-17T19:26:44.069928Z","shell.execute_reply":"2023-03-17T19:26:56.660378Z"},"trusted":true},"execution_count":null,"outputs":[]}]}