{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Isolated Sign Language Recognition with ConvLSTM1D\n\nIn this notebook, I will create Sign Language Recognition model using ConvLSTM1D. To build an efficient training pipeline, I will use TFRecord Dataset from https://www.kaggle.com/datasets/lonnieqin/islr-12-time-steps-tfrecords created by notebook https://www.kaggle.com/code/lonnieqin/islr-create-tfrecord for training.\n\nIt will take about 1 hour to finish runing this notebook using GPU.","metadata":{"papermill":{"duration":0.008117,"end_time":"2023-03-02T08:44:33.967124","exception":false,"start_time":"2023-03-02T08:44:33.959007","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Configuration","metadata":{"papermill":{"duration":0.006539,"end_time":"2023-03-02T08:44:33.994281","exception":false,"start_time":"2023-03-02T08:44:33.987742","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class CFG:\n    data_path = \"../input/asl-signs/\"\n    tf_record_path = \"../input/islr-12-time-steps-tfrecords/\"\n    sequence_length = 12\n    rows_per_frame = 543\n    is_training = False","metadata":{"papermill":{"duration":0.022144,"end_time":"2023-03-02T08:44:34.023146","exception":false,"start_time":"2023-03-02T08:44:34.001002","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-02T11:26:06.346348Z","iopub.execute_input":"2023-03-02T11:26:06.346876Z","iopub.status.idle":"2023-03-02T11:26:06.353817Z","shell.execute_reply.started":"2023-03-02T11:26:06.346838Z","shell.execute_reply":"2023-03-02T11:26:06.352181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import Libraries","metadata":{"papermill":{"duration":0.006464,"end_time":"2023-03-02T08:44:34.036262","exception":false,"start_time":"2023-03-02T08:44:34.029798","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tqdm import tqdm\nimport json\nimport time\nimport os\nfrom kaggle_datasets import KaggleDatasets","metadata":{"papermill":{"duration":7.496016,"end_time":"2023-03-02T08:44:41.539085","exception":false,"start_time":"2023-03-02T08:44:34.043069","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-02T11:26:09.973718Z","iopub.execute_input":"2023-03-02T11:26:09.974213Z","iopub.status.idle":"2023-03-02T11:26:09.981765Z","shell.execute_reply.started":"2023-03-02T11:26:09.974172Z","shell.execute_reply":"2023-03-02T11:26:09.979760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utilities","metadata":{"papermill":{"duration":0.006496,"end_time":"2023-03-02T08:44:41.597734","exception":false,"start_time":"2023-03-02T08:44:41.591238","status":"completed"},"tags":[]}},{"cell_type":"code","source":"ROWS_PER_FRAME = 543  # number of landmarks per frame\n\ndef load_relevant_data_subset_with_imputation(pq_path):\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    data.replace(np.nan, 0, inplace=True)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float16)\n\ndef load_relevant_data_subset(pq_path):\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)\n\ndef read_dict(file_path):\n    path = os.path.expanduser(file_path)\n    with open(path, \"r\") as f:\n        dic = json.load(f)\n    return dic","metadata":{"papermill":{"duration":0.017828,"end_time":"2023-03-02T08:44:41.622261","exception":false,"start_time":"2023-03-02T08:44:41.604433","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-02T11:26:12.264972Z","iopub.execute_input":"2023-03-02T11:26:12.265422Z","iopub.status.idle":"2023-03-02T11:26:12.276428Z","shell.execute_reply.started":"2023-03-02T11:26:12.265385Z","shell.execute_reply":"2023-03-02T11:26:12.274813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load data","metadata":{"papermill":{"duration":0.006542,"end_time":"2023-03-02T08:44:41.635478","exception":false,"start_time":"2023-03-02T08:44:41.628936","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train = pd.read_csv(f\"{CFG.data_path}train.csv\")\ntrain.head()","metadata":{"papermill":{"duration":0.206559,"end_time":"2023-03-02T08:44:41.848795","exception":false,"start_time":"2023-03-02T08:44:41.642236","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-02T11:26:16.026996Z","iopub.execute_input":"2023-03-02T11:26:16.027675Z","iopub.status.idle":"2023-03-02T11:26:16.154775Z","shell.execute_reply.started":"2023-03-02T11:26:16.027638Z","shell.execute_reply":"2023-03-02T11:26:16.153453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are 21 participants. Each of them created about 3000 to 5000 training records.","metadata":{"papermill":{"duration":0.007434,"end_time":"2023-03-02T08:44:41.863500","exception":false,"start_time":"2023-03-02T08:44:41.856066","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train.participant_id.nunique()","metadata":{"papermill":{"duration":0.024082,"end_time":"2023-03-02T08:44:41.894435","exception":false,"start_time":"2023-03-02T08:44:41.870353","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-02T11:26:18.052531Z","iopub.execute_input":"2023-03-02T11:26:18.052993Z","iopub.status.idle":"2023-03-02T11:26:18.064216Z","shell.execute_reply.started":"2023-03-02T11:26:18.052956Z","shell.execute_reply":"2023-03-02T11:26:18.062379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.participant_id.value_counts().plot(kind=\"bar\")","metadata":{"papermill":{"duration":0.339293,"end_time":"2023-03-02T08:44:42.240816","exception":false,"start_time":"2023-03-02T08:44:41.901523","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-02T11:26:20.281695Z","iopub.execute_input":"2023-03-02T11:26:20.282202Z","iopub.status.idle":"2023-03-02T11:26:20.661193Z","shell.execute_reply.started":"2023-03-02T11:26:20.282161Z","shell.execute_reply":"2023-03-02T11:26:20.659870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are 94477 training samples in total.","metadata":{"papermill":{"duration":0.007382,"end_time":"2023-03-02T08:44:42.255978","exception":false,"start_time":"2023-03-02T08:44:42.248596","status":"completed"},"tags":[]}},{"cell_type":"code","source":"len(train)","metadata":{"papermill":{"duration":0.016757,"end_time":"2023-03-02T08:44:42.280157","exception":false,"start_time":"2023-03-02T08:44:42.263400","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-02T11:26:23.859023Z","iopub.execute_input":"2023-03-02T11:26:23.860854Z","iopub.status.idle":"2023-03-02T11:26:23.869982Z","shell.execute_reply.started":"2023-03-02T11:26:23.860794Z","shell.execute_reply":"2023-03-02T11:26:23.868074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are 250 kinds of sign languages that we need to make prediction on.","metadata":{"papermill":{"duration":0.007305,"end_time":"2023-03-02T08:44:42.294963","exception":false,"start_time":"2023-03-02T08:44:42.287658","status":"completed"},"tags":[]}},{"cell_type":"code","source":"label_index = read_dict(f\"{CFG.data_path}sign_to_prediction_index_map.json\")\nindex_label = dict([(label_index[key], key) for key in label_index])\nprint(label_index)\ntrain[\"label\"] = train[\"sign\"].map(lambda sign: label_index[sign])\ntrain.head()","metadata":{"papermill":{"duration":0.059155,"end_time":"2023-03-02T08:44:42.361615","exception":false,"start_time":"2023-03-02T08:44:42.302460","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-02T11:26:25.629949Z","iopub.execute_input":"2023-03-02T11:26:25.630492Z","iopub.status.idle":"2023-03-02T11:26:25.695132Z","shell.execute_reply.started":"2023-03-02T11:26:25.630436Z","shell.execute_reply":"2023-03-02T11:26:25.693394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create Tensorflow Dataset","metadata":{"papermill":{"duration":0.007629,"end_time":"2023-03-02T08:44:42.377000","exception":false,"start_time":"2023-03-02T08:44:42.369371","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def decode_function(record_bytes):\n    return tf.io.parse_single_example(\n          # Data\n          record_bytes,\n          # Schema\n          {\n              \"feature\": tf.io.FixedLenFeature([CFG.sequence_length * CFG.rows_per_frame * 3], dtype=tf.float32),\n              \"label\": tf.io.FixedLenFeature([], dtype=tf.int64)\n          }\n      )\ndef preprocess(item):\n    features = item[\"feature\"]\n    features = tf.reshape(features, (CFG.sequence_length, 543, 3))\n    return features, item[\"label\"]         \ndef make_dataset(file_paths, batch_size=128, mode=\"train\"):\n    ds = tf.data.TFRecordDataset(file_paths)\n    ds = ds.map(decode_function)\n    ds = ds.map(preprocess)\n    options = tf.data.Options()\n    if mode == \"train\":\n        ds = ds.shuffle(1024)\n        options.experimental_deterministic = False\n    ds = ds.batch(batch_size, drop_remainder=True)\n    ds = ds.with_options(options) \n    ds = ds.cache().prefetch(tf.data.AUTOTUNE)\n    return ds","metadata":{"papermill":{"duration":0.019748,"end_time":"2023-03-02T08:44:42.404541","exception":false,"start_time":"2023-03-02T08:44:42.384793","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-02T11:26:30.673495Z","iopub.execute_input":"2023-03-02T11:26:30.673942Z","iopub.status.idle":"2023-03-02T11:26:30.685282Z","shell.execute_reply.started":"2023-03-02T11:26:30.673904Z","shell.execute_reply":"2023-03-02T11:26:30.683970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_ids = np.array(sorted(train.participant_id.unique()))\ntrain_ds = make_dataset([f\"{CFG.tf_record_path}{identifier}.tfrecords\" for identifier in unique_ids[:-4]])\nvalid_ds = make_dataset([f\"{CFG.tf_record_path}{identifier}.tfrecords\" for identifier in unique_ids[-4:]], mode=\"valid\")","metadata":{"papermill":{"duration":0.017938,"end_time":"2023-03-02T08:44:42.430150","exception":false,"start_time":"2023-03-02T08:44:42.412212","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-02T11:26:33.845464Z","iopub.execute_input":"2023-03-02T11:26:33.846245Z","iopub.status.idle":"2023-03-02T11:26:33.934210Z","shell.execute_reply.started":"2023-03-02T11:26:33.846194Z","shell.execute_reply":"2023-03-02T11:26:33.933049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Modeling","metadata":{"papermill":{"duration":0.007677,"end_time":"2023-03-02T08:44:42.445922","exception":false,"start_time":"2023-03-02T08:44:42.438245","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def conv1d_lstm_block(inputs, filters):\n    vector = tf.keras.layers.ConvLSTM1D(filters=32, kernel_size=8)(inputs)\n    for f in filters:\n        vector = tf.keras.layers.Conv1D(filters=f, kernel_size=8)(vector)\n        vector = tf.keras.layers.MaxPooling1D()(vector)\n    vector = tf.keras.layers.Dropout(0.3)(vector)\n    return vector\n\ndef get_model():\n    inputs = tf.keras.Input((CFG.sequence_length, 543, 3), dtype=tf.float32)\n    face_inputs = inputs[:, :, 0:468, :]\n    left_hand_inputs = inputs[:, :, 468:489, :]\n    pose_inputs = inputs[:, :, 489:522, :]\n    right_hand_inputs = inputs[:, :,522:,:]\n    face_vector = conv1d_lstm_block(face_inputs, [32, 64])\n    left_hand_vector = conv1d_lstm_block(left_hand_inputs, [64])\n    right_hand_vector = conv1d_lstm_block(right_hand_inputs, [64])\n    pose_vector = conv1d_lstm_block(pose_inputs, [64])\n    vector = tf.keras.layers.Concatenate(axis=1)([face_vector, left_hand_vector, right_hand_vector, pose_vector])\n    vector = tf.keras.layers.Flatten()(vector)\n    output = tf.keras.layers.Dense(250, activation=\"softmax\")(vector)\n    model = tf.keras.Model(inputs=inputs, outputs=output)\n    model.compile(\n        loss=tf.keras.losses.SparseCategoricalCrossentropy(), metrics=[\n            \"accuracy\",\n        ]\n    )\n    return model","metadata":{"papermill":{"duration":0.252992,"end_time":"2023-03-02T08:44:42.706685","exception":false,"start_time":"2023-03-02T08:44:42.453693","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-02T11:26:36.230565Z","iopub.execute_input":"2023-03-02T11:26:36.231769Z","iopub.status.idle":"2023-03-02T11:26:36.244946Z","shell.execute_reply.started":"2023-03-02T11:26:36.231718Z","shell.execute_reply":"2023-03-02T11:26:36.243408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = get_model()\nmodel.summary()\ntf.keras.utils.plot_model(model, show_shapes=True)","metadata":{"papermill":{"duration":3.974692,"end_time":"2023-03-02T08:44:46.689625","exception":false,"start_time":"2023-03-02T08:44:42.714933","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-02T11:26:41.479819Z","iopub.execute_input":"2023-03-02T11:26:41.480347Z","iopub.status.idle":"2023-03-02T11:26:43.104994Z","shell.execute_reply.started":"2023-03-02T11:26:41.480301Z","shell.execute_reply":"2023-03-02T11:26:43.103219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CFG.is_training:\n    file_name = \"model.h5\"\n    callbacks = [\n        tf.keras.callbacks.ModelCheckpoint(\n            file_name, \n            save_best_only=True, \n            restore_best_weights=True, \n            monitor=\"val_accuracy\",\n            mode=\"max\"\n        ),\n        tf.keras.callbacks.EarlyStopping(\n            patience=5, \n            monitor=\"val_accuracy\",\n            mode=\"max\"\n        )\n    ]\n    model.fit(train_ds, epochs=30, validation_data=valid_ds, callbacks=callbacks)\n    model = tf.keras.models.load_model(file_name)\nelse:\n    model = tf.keras.models.load_model(\"/kaggle/input/islr-convlstm1d/model.h5\")\nmodel.evaluate(valid_ds)   ","metadata":{"execution":{"iopub.status.busy":"2023-03-02T11:26:52.654003Z","iopub.execute_input":"2023-03-02T11:26:52.654464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create Model for inference","metadata":{"papermill":{"duration":0.069654,"end_time":"2023-03-02T08:47:03.916954","exception":false,"start_time":"2023-03-02T08:47:03.847300","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def get_inference_model(model):\n    inputs = tf.keras.Input((543, 3), dtype=tf.float32, name=\"inputs\")\n    vector = tf.image.resize(inputs, (CFG.sequence_length, 543))\n    vector = tf.where(tf.math.is_nan(vector), tf.zeros_like(vector), vector)\n    vector = tf.expand_dims(vector, axis=0)\n    vector = model(vector)\n    output = tf.keras.layers.Activation(activation=\"linear\", name=\"outputs\")(vector)\n    inference_model = tf.keras.Model(inputs=inputs, outputs=output) \n    inference_model.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy(), metrics=[\"accuracy\"])\n    return inference_model","metadata":{"papermill":{"duration":0.086334,"end_time":"2023-03-02T08:47:04.072776","exception":false,"start_time":"2023-03-02T08:47:03.986442","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-02T10:56:46.443200Z","iopub.execute_input":"2023-03-02T10:56:46.443625Z","iopub.status.idle":"2023-03-02T10:56:46.453574Z","shell.execute_reply.started":"2023-03-02T10:56:46.443592Z","shell.execute_reply":"2023-03-02T10:56:46.451899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inference_model = get_inference_model(model)\ninference_model.summary()\ntf.keras.utils.plot_model(inference_model, show_shapes=True)","metadata":{"papermill":{"duration":5.194577,"end_time":"2023-03-02T08:47:09.336507","exception":false,"start_time":"2023-03-02T08:47:04.141930","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-02T10:56:48.112122Z","iopub.execute_input":"2023-03-02T10:56:48.112576Z","iopub.status.idle":"2023-03-02T10:56:49.124709Z","shell.execute_reply.started":"2023-03-02T10:56:48.112538Z","shell.execute_reply":"2023-03-02T10:56:49.123071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create submission file","metadata":{"papermill":{"duration":0.071953,"end_time":"2023-03-02T08:47:09.784691","exception":false,"start_time":"2023-03-02T08:47:09.712738","status":"completed"},"tags":[]}},{"cell_type":"code","source":"converter = tf.lite.TFLiteConverter.from_keras_model(inference_model)\ntflite_model = converter.convert()\nmodel_path = \"model.tflite\"\n# Save the model.\nwith open(model_path, 'wb') as f:\n    f.write(tflite_model)\n!zip submission.zip $model_path","metadata":{"papermill":{"duration":153.429402,"end_time":"2023-03-02T08:49:43.286145","exception":false,"start_time":"2023-03-02T08:47:09.856743","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-02T10:57:00.750649Z","iopub.execute_input":"2023-03-02T10:57:00.751070Z","iopub.status.idle":"2023-03-02T10:57:32.224658Z","shell.execute_reply.started":"2023-03-02T10:57:00.751030Z","shell.execute_reply":"2023-03-02T10:57:32.223278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Making Predictions","metadata":{"papermill":{"duration":0.1124,"end_time":"2023-03-02T08:49:46.762554","exception":false,"start_time":"2023-03-02T08:49:46.650154","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install tflite-runtime","metadata":{"execution":{"iopub.execute_input":"2023-03-02T08:49:46.988990Z","iopub.status.busy":"2023-03-02T08:49:46.988541Z","iopub.status.idle":"2023-03-02T08:49:59.535732Z","shell.execute_reply":"2023-03-02T08:49:59.534490Z"},"papermill":{"duration":12.663338,"end_time":"2023-03-02T08:49:59.538473","exception":false,"start_time":"2023-03-02T08:49:46.875135","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tflite_runtime.interpreter as tflite\ninterpreter = tflite.Interpreter(model_path)\nfound_signatures = list(interpreter.get_signature_list().keys())\nprediction_fn = interpreter.get_signature_runner(\"serving_default\")\nfor i in tqdm(range(10000)):\n    frames = load_relevant_data_subset(f'/kaggle/input/asl-signs/{train.iloc[i].path}')\n    output = prediction_fn(inputs=frames)\n    if i < 100:\n        sign = np.argmax(output[\"outputs\"])\n        print(f\"Predicted label: {index_label[sign]}, Actual Label: {train.iloc[i].sign}\")","metadata":{"execution":{"iopub.execute_input":"2023-03-02T08:49:59.762557Z","iopub.status.busy":"2023-03-02T08:49:59.761446Z","iopub.status.idle":"2023-03-02T08:50:28.814165Z","shell.execute_reply":"2023-03-02T08:50:28.812801Z"},"papermill":{"duration":29.201279,"end_time":"2023-03-02T08:50:28.816671","exception":false,"start_time":"2023-03-02T08:49:59.615392","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}