{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# GISLR notebook\nworking notebook","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport json","metadata":{"execution":{"iopub.status.busy":"2023-04-30T20:32:53.994617Z","iopub.execute_input":"2023-04-30T20:32:53.995742Z","iopub.status.idle":"2023-04-30T20:32:54.038727Z","shell.execute_reply.started":"2023-04-30T20:32:53.995694Z","shell.execute_reply":"2023-04-30T20:32:54.037341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip show tensorflow","metadata":{"execution":{"iopub.status.busy":"2023-04-30T20:32:54.041546Z","iopub.execute_input":"2023-04-30T20:32:54.044762Z","iopub.status.idle":"2023-04-30T20:33:06.282470Z","shell.execute_reply.started":"2023-04-30T20:32:54.044728Z","shell.execute_reply":"2023-04-30T20:33:06.281242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install tslearn plotly pyarrow fastparquet","metadata":{"execution":{"iopub.status.busy":"2023-04-30T20:33:06.284962Z","iopub.execute_input":"2023-04-30T20:33:06.285366Z","iopub.status.idle":"2023-04-30T20:33:17.528338Z","shell.execute_reply.started":"2023-04-30T20:33:06.285313Z","shell.execute_reply":"2023-04-30T20:33:17.527120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# mode = \"debugging\"\nmode = \"training\"\n# mode = \"submission\"","metadata":{"execution":{"iopub.status.busy":"2023-04-30T20:33:17.533037Z","iopub.execute_input":"2023-04-30T20:33:17.533356Z","iopub.status.idle":"2023-04-30T20:33:17.538330Z","shell.execute_reply.started":"2023-04-30T20:33:17.533324Z","shell.execute_reply":"2023-04-30T20:33:17.537120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Understanding the data","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(\"/kaggle/input/asl-signs/train.csv\")\nprint(df_train.shape)\ndf_train.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-04-30T20:33:17.539887Z","iopub.execute_input":"2023-04-30T20:33:17.540596Z","iopub.status.idle":"2023-04-30T20:33:17.758334Z","shell.execute_reply.started":"2023-04-30T20:33:17.540557Z","shell.execute_reply":"2023-04-30T20:33:17.757218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"json_file_path = \"/kaggle/input/asl-signs/sign_to_prediction_index_map.json\"\nwith open(json_file_path, 'r') as j:\n     sign_dict = json.loads(j.read())\n        \nordered_signs = list(sign_dict.keys())\nprint(ordered_signs)\n","metadata":{"execution":{"iopub.status.busy":"2023-04-30T20:33:17.760327Z","iopub.execute_input":"2023-04-30T20:33:17.761003Z","iopub.status.idle":"2023-04-30T20:33:17.769796Z","shell.execute_reply.started":"2023-04-30T20:33:17.760962Z","shell.execute_reply":"2023-04-30T20:33:17.768564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROWS_PER_FRAME = 543  # number of landmarks per frame\n\ndef load_relevant_data_subset(pq_path):\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(pq_path, columns=data_columns).fillna(0)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)\n\ndef load_relevant_data(pq_path):\n    data = pd.read_parquet(pq_path).fillna(0)\n    return data\n","metadata":{"execution":{"iopub.status.busy":"2023-04-30T20:33:17.771641Z","iopub.execute_input":"2023-04-30T20:33:17.772083Z","iopub.status.idle":"2023-04-30T20:33:17.780487Z","shell.execute_reply.started":"2023-04-30T20:33:17.772046Z","shell.execute_reply":"2023-04-30T20:33:17.779318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx_plot = 3\npath_show = \"/kaggle/input/asl-signs/\"+df_train['path'].values[idx_plot]\nsign_plot = df_train['sign'].values[idx_plot]\npath_example = path_show.replace(\"_\", \"_\")\n\ndf = load_relevant_data(path_show)\ndf.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-04-30T20:33:17.782089Z","iopub.execute_input":"2023-04-30T20:33:17.783203Z","iopub.status.idle":"2023-04-30T20:33:17.890583Z","shell.execute_reply.started":"2023-04-30T20:33:17.783165Z","shell.execute_reply":"2023-04-30T20:33:17.889483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_show = \"/kaggle/input/asl-signs/\"+df_train['path'].values[1]\nsign_show = df_train['sign'].values[1]\n\ndf_example = load_relevant_data_subset(path_show)\n\nframes = df_example.shape[0]\nkeypoints = df_example.shape[1]\nposition = df_example.shape[2]\n\nprint(\"\\nNumber of frames:\", frames)\nprint(\"Keypoints:\", keypoints)\nprint(\"X, Y Z postions:\", position)\nprint(\"Total number of datapoints in this sequence:\", np.prod(df_example.shape))\n\n\npose_landmarks = 33\nface_landmarks = 468\nright_hand_landmarks = 21\nstart_left_hand = face_landmarks\nleft_hand_landmarks = 21\nstart_right_hand = face_landmarks + left_hand_landmarks + pose_landmarks\ntotal_landmarks = pose_landmarks + face_landmarks + right_hand_landmarks + left_hand_landmarks\n\n\nprint(\"\\nPose landmarks:\", pose_landmarks)\nprint(\"Face landmarks:\", face_landmarks)\nprint(\"Right hand landmarks:\", right_hand_landmarks)\nprint(\"Left hand landmarks:\", left_hand_landmarks)\nprint(\"Total landmarks/keypoints: \", total_landmarks)","metadata":{"execution":{"iopub.status.busy":"2023-04-30T20:33:17.892609Z","iopub.execute_input":"2023-04-30T20:33:17.893655Z","iopub.status.idle":"2023-04-30T20:33:17.915646Z","shell.execute_reply.started":"2023-04-30T20:33:17.893616Z","shell.execute_reply":"2023-04-30T20:33:17.914530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\n\nmax_sequence_length = 40\nlip_marks = [61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291, 78, 191, 80, 81, 82, 13, 312, 311, 310, 415, 308, 95, 88, 178, 87, 14, 317, 402, 318, 324, 146, 91, 181, 84, 17, 314, 405, 321, 375]  \n\nlips = lip_marks\nleft_hand = [*range(start_left_hand, start_left_hand+left_hand_landmarks, 1)]\nright_hand = [*range(start_right_hand, start_right_hand+right_hand_landmarks, 1)]\nmeaningful_keypoints = lips + left_hand + right_hand\ninput_length = len(meaningful_keypoints)*3\n\ndef get_data(file_paths, y_sign):\n    \n    X = np.empty((file_paths.shape[0], max_sequence_length, len(meaningful_keypoints)*3), dtype=float)\n\n    for i in tqdm(range(file_paths.shape[0])):\n        file_name = \"/kaggle/input/asl-signs/\"+file_paths[i]\n        data = load_relevant_data_subset(file_name)\n        \n        data = data[:, meaningful_keypoints]\n        \n        if data.shape[0] < max_sequence_length:\n            rows = max_sequence_length - data.shape[0]\n            data = np.append(np.zeros((rows, len(meaningful_keypoints), 3)), data, axis=0)\n        elif data.shape[0] > max_sequence_length:\n            data = data[-(max_sequence_length):]\n\n        X[i] = data.reshape(max_sequence_length, len(meaningful_keypoints)*3, order='F')\n        \n        del data\n        \n    X = np.asarray(X).astype(np.float32)\n        \n    y = []\n    for sign in y_sign:\n        y.append(sign_dict[sign])\n\n    y = np.array(y, dtype=int)\n\n    return X, y","metadata":{"execution":{"iopub.status.busy":"2023-04-30T20:33:17.920789Z","iopub.execute_input":"2023-04-30T20:33:17.921081Z","iopub.status.idle":"2023-04-30T20:33:17.932220Z","shell.execute_reply.started":"2023-04-30T20:33:17.921054Z","shell.execute_reply":"2023-04-30T20:33:17.930980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training the classifier","metadata":{}},{"cell_type":"markdown","source":"## LSTM\nLets start with a simple LSTM system","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras import datasets, layers, models, Input, optimizers\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow.keras.utils import pad_sequences","metadata":{"execution":{"iopub.status.busy":"2023-04-30T20:33:17.933894Z","iopub.execute_input":"2023-04-30T20:33:17.934691Z","iopub.status.idle":"2023-04-30T20:33:25.321445Z","shell.execute_reply.started":"2023-04-30T20:33:17.934650Z","shell.execute_reply":"2023-04-30T20:33:25.320350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## credits for this implementation of MultiHeadattention to Mark wijkhuizen notebook\n# tf.keras.layers.MultiHeadattention does not work on tflite 2.9.1\n# https://www.kaggle.com/code/markwijkhuizen/gislr-tf-data-processing-transformer-training\n\ndef scaled_dot_product(q,k,v, softmax):\n    #calculates Q . K(transpose)\n    qkt = tf.matmul(q,k,transpose_b=True)\n    #caculates scaling factor\n    dk = tf.math.sqrt(tf.cast(q.shape[-1],dtype=tf.float32))\n    scaled_qkt = qkt/dk\n    softmax = softmax(scaled_qkt)\n    \n    z = tf.matmul(softmax,v)\n    #shape: (m,Tx,depth), same shape as q,k,v\n    return z\n\nclass MultiHeadAttention(tf.keras.layers.Layer):\n    def __init__(self,d_model,num_of_heads):\n        super(MultiHeadAttention,self).__init__()\n        self.d_model = d_model\n        self.num_of_heads = num_of_heads\n        self.depth = d_model//num_of_heads\n        self.wq = [tf.keras.layers.Dense(self.depth) for i in range(num_of_heads)]\n        self.wk = [tf.keras.layers.Dense(self.depth) for i in range(num_of_heads)]\n        self.wv = [tf.keras.layers.Dense(self.depth) for i in range(num_of_heads)]\n        self.wo = tf.keras.layers.Dense(d_model)\n        self.softmax = tf.keras.layers.Softmax()\n        \n    def call(self,x):\n        \n        multi_attn = []\n        for i in range(self.num_of_heads):\n            Q = self.wq[i](x)\n            K = self.wk[i](x)\n            V = self.wv[i](x)\n            multi_attn.append(scaled_dot_product(Q,K,V, self.softmax))\n            \n        multi_head = tf.concat(multi_attn,axis=-1)\n        multi_head_attention = self.wo(multi_head)\n        return multi_head_attention","metadata":{"execution":{"iopub.status.busy":"2023-04-30T20:33:25.323307Z","iopub.execute_input":"2023-04-30T20:33:25.324154Z","iopub.status.idle":"2023-04-30T20:33:25.519878Z","shell.execute_reply.started":"2023-04-30T20:33:25.324114Z","shell.execute_reply":"2023-04-30T20:33:25.518686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Keypoint embedding and positional encoding for transformer architecture\nusing padding or masking to work with variable input lengths\nWorking on positional encoding but not yet functional, need to do some padding and/or masking to make this work.\n\nhttps://www.tensorflow.org/guide/keras/masking_and_padding","metadata":{"execution":{"iopub.status.busy":"2023-04-24T11:16:16.089106Z","iopub.execute_input":"2023-04-24T11:16:16.089508Z","iopub.status.idle":"2023-04-24T11:16:16.098095Z","shell.execute_reply.started":"2023-04-24T11:16:16.089472Z","shell.execute_reply":"2023-04-24T11:16:16.096921Z"}}},{"cell_type":"code","source":"class PositionalEncoding(tf.keras.layers.Layer):\n    def __init__(self, output_dim, **kwargs):\n        super(PositionalEncoding, self).__init__(**kwargs)\n        self.d = output_dim\n        \n        self.positional_embedding = tf.keras.layers.Embedding(32+1, 256)\n             \n    def get_position_encoding(self, seq_len, n=10000):\n        P = np.zeros((seq_len, self.d))\n        for k in range(seq_len):\n            for i in np.arange(int(self.d/2)):\n                denominator = np.power(n, 2*i/self.d)\n                P[k, 2*i] = np.sin(k/denominator)\n                P[k, 2*i+1] = np.cos(k/denominator)\n        return P\n \n \n    def call(self, inputs):\n        print(inputs.shape)\n        non_empty_frame_idxs = tf.keras.layers.Input(shape=(inputs.shape[1],), dtype=tf.float32)\n        max_frame_idxs = tf.clip_by_value(\n                tf.reduce_max(non_empty_frame_idxs, axis=1, keepdims=True),\n                1,\n                np.PINF,\n            )\n        normalised_non_empty_frame_idxs = tf.where(\n            tf.math.equal(non_empty_frame_idxs, -1.0),\n            32,\n            tf.cast(\n                non_empty_frame_idxs / max_frame_idxs * 32,\n                tf.int32,\n            ),\n        )\n        \n        print(normalised_non_empty_frame_idxs.shape)\n        \n        positional_encoding = self.positional_embedding(normalised_non_empty_frame_idxs)\n        return inputs + positional_encoding","metadata":{"execution":{"iopub.status.busy":"2023-04-30T20:33:25.521792Z","iopub.execute_input":"2023-04-30T20:33:25.522456Z","iopub.status.idle":"2023-04-30T20:33:25.535727Z","shell.execute_reply.started":"2023-04-30T20:33:25.522415Z","shell.execute_reply":"2023-04-30T20:33:25.534565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# a single dense block followed by a normalization block and relu activation\ndef dense_block(units):\n    fc = layers.Dense(units)\n    norm = layers.LayerNormalization()\n    act = layers.Activation(\"relu\")\n    return lambda x: act(norm(fc(x)))\n\n# transformer blocks\ndef transformer_block(key_dim, x):\n    mha = MultiHeadAttention(key_dim, 8)(x)\n    add1 = layers.add([mha, x])\n    norm1 = layers.LayerNormalization()(add1)\n\n    fc = layers.Dense(key_dim, activation=\"relu\")(norm1)\n    add2 = tf.math.add(fc, norm1)\n    norm2 = layers.LayerNormalization()(add2)\n\n    return norm2\n\n# the final dense block for the classification\ndef classifier_lstm(units):\n    lstm = layers.LSTM(units)\n    out = layers.Dense(250, activation=\"softmax\", name=\"outputs\")\n    return lambda x: out(lstm(x))\n    \ndef classifier_transformer():\n    out = layers.Dense(250, activation=\"softmax\", name=\"outputs\")\n    return lambda x: out(x)\n\ninputs = tf.keras.Input(shape=(None, input_length), ragged=True)\n# choose the number of nodes per layer\nembedding_units = [256, 128, 256, 512] # tune this\ntransformer_units = []#512, 512, 512, 512]#, 256, 256, 256]\n\n# # dense encoder model\nx = inputs\nprint(x.shape)\nfor units in embedding_units:\n    x = dense_block(units)(x)\n\n    \n# print(x.shape)\n# x = PositionalEncoding(transformer_units[0])(x)\n    \nfor t in transformer_units:\n    x = transformer_block(t, x)\n\n# classifier layer\nif len(transformer_units) > 0:\n    # Pooling\n    x = tf.keras.layers.GlobalAveragePooling1D()(x)\n    out = classifier_transformer()(x)\nelse:\n    out = classifier_lstm(embedding_units[-1])(x)\n\n\nmodel = tf.keras.Model(inputs=inputs, outputs=out)\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-04-30T20:33:25.537343Z","iopub.execute_input":"2023-04-30T20:33:25.537796Z","iopub.status.idle":"2023-04-30T20:33:29.157081Z","shell.execute_reply.started":"2023-04-30T20:33:25.537759Z","shell.execute_reply":"2023-04-30T20:33:29.156273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# add a decreasing learning rate scheduler to help convergence\nbatch_size = 1024\nvalidation_percentage = 0.001\nsteps_per_epoch = int(94477*(1-validation_percentage)) // batch_size\nboundaries = [steps_per_epoch * n for n in [40, 50, 60, 70, 80, 90, 100]]\nprint(boundaries)\nvalues = [1e-3, 3.5e-4, 1e-4, 3.5e-5, 1e-5, 3.5e-6, 1e-6, 1e-7]\nlr_sched = optimizers.schedules.PiecewiseConstantDecay(boundaries, values)\n\noptimizer = optimizers.Adam(lr_sched)\n# optimizer = optimizers.Adam()\n\nmodel.compile(optimizer=optimizer,\n              loss=tf.keras.losses.SparseCategoricalCrossentropy(name=\"loss\"),\n              metrics=[\"accuracy\",\"sparse_top_k_categorical_accuracy\"])","metadata":{"execution":{"iopub.status.busy":"2023-04-30T20:33:29.158186Z","iopub.execute_input":"2023-04-30T20:33:29.158557Z","iopub.status.idle":"2023-04-30T20:33:29.237195Z","shell.execute_reply.started":"2023-04-30T20:33:29.158500Z","shell.execute_reply":"2023-04-30T20:33:29.236077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_callbacks():\n    return [\n        tf.keras.callbacks.EarlyStopping(\n            monitor=\"accuracy\",\n            patience = 20,\n            restore_best_weights=True\n        ),\n        tf.keras.callbacks.ReduceLROnPlateau(\n            monitor = \"accuracy\",\n            factor = (0.1)**(0.5),\n            patience = 3\n        ),\n    ]","metadata":{"execution":{"iopub.status.busy":"2023-04-30T20:33:29.239392Z","iopub.execute_input":"2023-04-30T20:33:29.240490Z","iopub.status.idle":"2023-04-30T20:33:29.246688Z","shell.execute_reply.started":"2023-04-30T20:33:29.240445Z","shell.execute_reply":"2023-04-30T20:33:29.245504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if mode == \"training\":\n    file_paths = df_train['path'].values#[:1000]\n    y_sign = df_train['sign'].values#[:1000]\n    X, y = get_data(file_paths, y_sign)\n\n    X, X_val, y, y_val = train_test_split(X, y, test_size=validation_percentage, random_state=123)\n\n    print(X.shape)\n    print(X_val.shape)\n    print(y.shape)\n\n    history = model.fit(X, y,  \n                    epochs=125,\n                    batch_size=batch_size,\n                    validation_data=(X_val, y_val),\n                    verbose=2,\n                    callbacks=[get_callbacks()]\n                   )\n\n    loss = history.history['loss']\n    val_loss = history.history['val_loss']\n    accuracy = history.history['accuracy']\n    val_accuracy = history.history['val_accuracy']","metadata":{"execution":{"iopub.status.busy":"2023-04-30T20:33:29.250613Z","iopub.execute_input":"2023-04-30T20:33:29.250963Z","iopub.status.idle":"2023-04-30T20:35:17.287082Z","shell.execute_reply.started":"2023-04-30T20:33:29.250919Z","shell.execute_reply":"2023-04-30T20:35:17.286057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if mode == \"training\":\n    del X\n    del y\n    del X_val\n    del y_val","metadata":{"execution":{"iopub.status.busy":"2023-04-30T20:35:17.288789Z","iopub.execute_input":"2023-04-30T20:35:17.289150Z","iopub.status.idle":"2023-04-30T20:35:17.296777Z","shell.execute_reply.started":"2023-04-30T20:35:17.289112Z","shell.execute_reply":"2023-04-30T20:35:17.295599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if mode == \"training\":\n    import matplotlib.pyplot as plt\n    fig = plt.figure()\n\n    plt.plot(val_accuracy, label='val_accuracy')\n\n    plt.plot(accuracy, label='accuracy')\n\n    plt.xlabel('Epoch')\n    plt.ylabel('Accuracy')\n#     plt.ylim([0.5, 1])\n    plt.legend()\n\n#     test_loss, test_acc = model.evaluate(X,  y, verbose=1)","metadata":{"execution":{"iopub.status.busy":"2023-04-30T20:35:17.298482Z","iopub.execute_input":"2023-04-30T20:35:17.298989Z","iopub.status.idle":"2023-04-30T20:35:35.922669Z","shell.execute_reply.started":"2023-04-30T20:35:17.298951Z","shell.execute_reply":"2023-04-30T20:35:35.921510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Convert final model in a tf.lite model\nAdd a preprocessing pipeline and safe the model as tf.lite model","metadata":{}},{"cell_type":"code","source":"lips = lip_marks\nleft_hand = [*range(start_left_hand, start_left_hand+left_hand_landmarks, 1)]\nright_hand = [*range(start_right_hand, start_right_hand+right_hand_landmarks, 1)]\nmeaningful_keypoints = lips + left_hand + right_hand\n\n\ndef get_inference_model(model):\n    inputs = tf.keras.Input(shape=(ROWS_PER_FRAME,3), name=\"inputs\")\n    \n    # drop most of the face mesh\n    x = tf.gather(inputs, meaningful_keypoints, axis=1)\n\n    # fill nan\n    x = tf.where(tf.math.is_nan(x), tf.zeros_like(x), x)\n\n    # flatten landmark xyz coordinates ()\n    x = tf.concat([x[...,i] for i in range(3)], -1)\n\n    x = tf.expand_dims(x,0)\n    \n    # call trained model\n    out = model(x)\n    \n    # explicitly name the final (identity) layer for the submission format\n    outputs = layers.Activation(\"linear\", name=\"outputs\")(out)\n    \n    inference_model = tf.keras.Model(inputs=inputs, outputs=outputs)\n    inference_model.compile(loss=\"sparse_categorical_crossentropy\",\n                            metrics=\"accuracy\")\n    return inference_model","metadata":{"execution":{"iopub.status.busy":"2023-04-30T20:35:35.924316Z","iopub.execute_input":"2023-04-30T20:35:35.924730Z","iopub.status.idle":"2023-04-30T20:35:35.935303Z","shell.execute_reply.started":"2023-04-30T20:35:35.924690Z","shell.execute_reply":"2023-04-30T20:35:35.934206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inference_model = get_inference_model(model)\ninference_model.summary(expand_nested=True)","metadata":{"execution":{"iopub.status.busy":"2023-04-30T20:35:35.936760Z","iopub.execute_input":"2023-04-30T20:35:35.937208Z","iopub.status.idle":"2023-04-30T20:35:36.422060Z","shell.execute_reply.started":"2023-04-30T20:35:35.937167Z","shell.execute_reply":"2023-04-30T20:35:36.421247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if mode == \"training\":\n    converter = tf.lite.TFLiteConverter.from_keras_model(inference_model)\n    tflite_model = converter.convert()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-04-30T20:35:36.423144Z","iopub.execute_input":"2023-04-30T20:35:36.423505Z","iopub.status.idle":"2023-04-30T20:35:50.417336Z","shell.execute_reply.started":"2023-04-30T20:35:36.423466Z","shell.execute_reply":"2023-04-30T20:35:50.416231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if mode == \"training\":\n    with open('/kaggle/working/model.tflite', 'wb') as f:\n        f.write(tflite_model)","metadata":{"execution":{"iopub.status.busy":"2023-04-30T20:35:50.419150Z","iopub.execute_input":"2023-04-30T20:35:50.419552Z","iopub.status.idle":"2023-04-30T20:35:50.433992Z","shell.execute_reply.started":"2023-04-30T20:35:50.419497Z","shell.execute_reply":"2023-04-30T20:35:50.432977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if mode == \"submission\":\n    !cp -r /kaggle/input/tflite-model/model.tflite ./\n","metadata":{"execution":{"iopub.status.busy":"2023-04-30T20:35:50.435505Z","iopub.execute_input":"2023-04-30T20:35:50.436311Z","iopub.status.idle":"2023-04-30T20:35:50.441854Z","shell.execute_reply.started":"2023-04-30T20:35:50.436262Z","shell.execute_reply":"2023-04-30T20:35:50.440770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip submission.zip model.tflite","metadata":{"execution":{"iopub.status.busy":"2023-04-30T20:35:50.443290Z","iopub.execute_input":"2023-04-30T20:35:50.443847Z","iopub.status.idle":"2023-04-30T20:35:51.984681Z","shell.execute_reply.started":"2023-04-30T20:35:50.443807Z","shell.execute_reply":"2023-04-30T20:35:51.983366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission code","metadata":{}},{"cell_type":"code","source":"!pip install tflite-runtime==2.9.1\nimport tflite_runtime.interpreter as tflite\nimport numpy as np\nimport pandas as pd","metadata":{"execution":{"iopub.status.busy":"2023-04-30T20:35:51.986860Z","iopub.execute_input":"2023-04-30T20:35:51.987787Z","iopub.status.idle":"2023-04-30T20:36:02.179594Z","shell.execute_reply.started":"2023-04-30T20:35:51.987738Z","shell.execute_reply":"2023-04-30T20:36:02.178334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROWS_PER_FRAME = 543  # number of landmarks per frame\n\ndef load_relevant_data_subset(pq_path):\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2023-04-30T20:36:02.181467Z","iopub.execute_input":"2023-04-30T20:36:02.181896Z","iopub.status.idle":"2023-04-30T20:36:02.190061Z","shell.execute_reply.started":"2023-04-30T20:36:02.181843Z","shell.execute_reply":"2023-04-30T20:36:02.188780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mode = \"debugging\"\n\nif mode == \"debugging\":    \n    import tflite_runtime.interpreter as tflite\n    \n    model_path = \"/kaggle/working/model.tflite\"\n#     model_path = \"/kaggle/input/tflite-model/model.tflite\"\n    \n    interpreter = tflite.Interpreter(model_path)\n    print(\"interpreter loaded\")\n\n    prediction_fn = interpreter.get_signature_runner(\"serving_default\")\n    precision_count = 0\n    for i in range(10):\n        X = load_relevant_data_subset(\"/kaggle/input/asl-signs/\"+file_paths[i])\n        output = prediction_fn(inputs=X)\n        sign = np.argmax(output[\"outputs\"])\n        print(\"Ground truth\", y_sign[i], sign_dict[y_sign[i]], \"___ Prediction\", ordered_signs[sign], sign)\n        if sign == sign_dict[y_sign[i]]:\n            precision_count += 1\n\n    print(precision_count)\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2023-04-30T20:36:02.196386Z","iopub.execute_input":"2023-04-30T20:36:02.196960Z","iopub.status.idle":"2023-04-30T20:36:02.461038Z","shell.execute_reply.started":"2023-04-30T20:36:02.196930Z","shell.execute_reply":"2023-04-30T20:36:02.459827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if mode == \"debugging\":\n    model_path = \"/kaggle/working/model.tflite\"\n    tf.lite.experimental.Analyzer.analyze(model_path=model_path)        ","metadata":{"execution":{"iopub.status.busy":"2023-04-30T20:36:02.462716Z","iopub.execute_input":"2023-04-30T20:36:02.463119Z","iopub.status.idle":"2023-04-30T20:36:02.472091Z","shell.execute_reply.started":"2023-04-30T20:36:02.463079Z","shell.execute_reply":"2023-04-30T20:36:02.470928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}