{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Ensemble of two models\nUsed 2 notebooks:\n* https://www.kaggle.com/code/jvthunder/lstm-baseline-for-starters-sign-language (with my own train  - LB = 0.6);\n* https://www.kaggle.com/code/roberthatch/gislr-lb-0-63-on-the-shoulders (LB = 0.63)\n","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers, optimizers","metadata":{"execution":{"iopub.status.busy":"2023-03-27T14:58:34.120529Z","iopub.execute_input":"2023-03-27T14:58:34.121594Z","iopub.status.idle":"2023-03-27T14:58:34.127867Z","shell.execute_reply.started":"2023-03-27T14:58:34.121551Z","shell.execute_reply":"2023-03-27T14:58:34.126579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load models","metadata":{}},{"cell_type":"code","source":"asl_model = keras.models.load_model('/kaggle/input/gislr-tf-on-the-shoulders-s/models/asl_model')\nltsm_model = keras.models.load_model('/kaggle/input/googlecode/lstm5.h5')","metadata":{"execution":{"iopub.status.busy":"2023-03-27T14:58:34.900380Z","iopub.execute_input":"2023-03-27T14:58:34.901007Z","iopub.status.idle":"2023-03-27T14:58:36.109357Z","shell.execute_reply.started":"2023-03-27T14:58:34.900969Z","shell.execute_reply":"2023-03-27T14:58:36.108114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asl_model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-03-27T14:58:36.112457Z","iopub.execute_input":"2023-03-27T14:58:36.113208Z","iopub.status.idle":"2023-03-27T14:58:36.146470Z","shell.execute_reply.started":"2023-03-27T14:58:36.113161Z","shell.execute_reply":"2023-03-27T14:58:36.145713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ltsm_model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-03-27T14:58:36.147607Z","iopub.execute_input":"2023-03-27T14:58:36.148039Z","iopub.status.idle":"2023-03-27T14:58:36.177562Z","shell.execute_reply.started":"2023-03-27T14:58:36.147997Z","shell.execute_reply":"2023-03-27T14:58:36.176804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Params and utils","metadata":{}},{"cell_type":"code","source":"ROWS_PER_FRAME = 543\nLANDMARK_IDX = [0,9,11,13,14,17,117,118,119,199,346,347,348] + list(range(468,543))","metadata":{"execution":{"iopub.status.busy":"2023-03-27T14:58:36.179356Z","iopub.execute_input":"2023-03-27T14:58:36.179703Z","iopub.status.idle":"2023-03-27T14:58:36.185226Z","shell.execute_reply.started":"2023-03-27T14:58:36.179648Z","shell.execute_reply":"2023-03-27T14:58:36.184426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DROP_Z = False\n\nNUM_FRAMES = 15\nSEGMENTS = 3\n\nLEFT_HAND_OFFSET = 468\nPOSE_OFFSET = LEFT_HAND_OFFSET+21\nRIGHT_HAND_OFFSET = POSE_OFFSET+33\n\n## average over the entire face, and the entire 'pose'\naveraging_sets = [[0, 468], [POSE_OFFSET, 33]]\n\nlip_landmarks = [61, 185, 40, 39, 37,  0, 267, 269, 270, 409,\n                 291,146, 91,181, 84, 17, 314, 405, 321, 375, \n                 78, 191, 80, 81, 82, 13, 312, 311, 310, 415, \n                 95, 88, 178, 87, 14,317, 402, 318, 324, 308]\nleft_hand_landmarks = list(range(LEFT_HAND_OFFSET, LEFT_HAND_OFFSET+21))\nright_hand_landmarks = list(range(RIGHT_HAND_OFFSET, RIGHT_HAND_OFFSET+21))\n\npoint_landmarks = [item for sublist in [lip_landmarks, left_hand_landmarks, right_hand_landmarks] for item in sublist]\n\nLANDMARKS = len(point_landmarks) + len(averaging_sets)\nprint(LANDMARKS)\nif DROP_Z:\n    INPUT_SHAPE = (NUM_FRAMES,LANDMARKS*2)\nelse:\n    INPUT_SHAPE = (NUM_FRAMES,LANDMARKS*3)\n\nFLAT_INPUT_SHAPE = (INPUT_SHAPE[0] + 2 * (SEGMENTS + 1)) * INPUT_SHAPE[1]","metadata":{"execution":{"iopub.status.busy":"2023-03-27T14:58:36.967287Z","iopub.execute_input":"2023-03-27T14:58:36.967724Z","iopub.status.idle":"2023-03-27T14:58:36.979366Z","shell.execute_reply.started":"2023-03-27T14:58:36.967676Z","shell.execute_reply":"2023-03-27T14:58:36.978115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tf_nan_mean(x, axis=0):\n    return tf.reduce_sum(tf.where(tf.math.is_nan(x), tf.zeros_like(x), x), axis=axis) / tf.reduce_sum(tf.where(tf.math.is_nan(x), tf.zeros_like(x), tf.ones_like(x)), axis=axis)\n\ndef tf_nan_std(x, axis=0):\n    d = x - tf_nan_mean(x, axis=axis)\n    return tf.math.sqrt(tf_nan_mean(d * d, axis=axis))\n\ndef flatten_means_and_stds(x, axis=0):\n    # Get means and stds\n    x_mean = tf_nan_mean(x, axis=0)\n    x_std  = tf_nan_std(x,  axis=0)\n\n    x_out = tf.concat([x_mean, x_std], axis=0)\n    x_out = tf.reshape(x_out, (1, INPUT_SHAPE[1]*2))\n    x_out = tf.where(tf.math.is_finite(x_out), x_out, tf.zeros_like(x_out))\n    return x_out\n\nclass FeatureGen(tf.keras.layers.Layer):\n    def __init__(self):\n        super(FeatureGen, self).__init__()\n    \n    def call(self, x_in):\n        if DROP_Z:\n            x_in = x_in[:, :, 0:2]\n        x_list = [tf.expand_dims(tf_nan_mean(x_in[:, av_set[0]:av_set[0]+av_set[1], :], axis=1), axis=1) for av_set in averaging_sets]\n        x_list.append(tf.gather(x_in, point_landmarks, axis=1))\n        x = tf.concat(x_list, 1)\n\n        x_padded = x\n        for i in range(SEGMENTS):\n            p0 = tf.where( ((tf.shape(x_padded)[0] % SEGMENTS) > 0) & ((i % 2) != 0) , 1, 0)\n            p1 = tf.where( ((tf.shape(x_padded)[0] % SEGMENTS) > 0) & ((i % 2) == 0) , 1, 0)\n            paddings = [[p0, p1], [0, 0], [0, 0]]\n            x_padded = tf.pad(x_padded, paddings, mode=\"SYMMETRIC\")\n        x_list = tf.split(x_padded, SEGMENTS)\n        x_list = [flatten_means_and_stds(_x, axis=0) for _x in x_list]\n\n        x_list.append(flatten_means_and_stds(x, axis=0))\n        \n        ## Resize only dimension 0. Resize can't handle nan, so replace nan with that dimension's avg value to reduce impact.\n        x = tf.image.resize(tf.where(tf.math.is_finite(x), x, tf_nan_mean(x, axis=0)), [NUM_FRAMES, LANDMARKS])\n        x = tf.reshape(x, (1, INPUT_SHAPE[0]*INPUT_SHAPE[1]))\n        x = tf.where(tf.math.is_nan(x), tf.zeros_like(x), x)\n        x_list.append(x)\n        x = tf.concat(x_list, axis=1)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-03-27T14:58:37.709882Z","iopub.execute_input":"2023-03-27T14:58:37.710343Z","iopub.status.idle":"2023-03-27T14:58:37.737200Z","shell.execute_reply.started":"2023-03-27T14:58:37.710304Z","shell.execute_reply":"2023-03-27T14:58:37.735741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Make ensemble ","metadata":{}},{"cell_type":"code","source":"def get_inference_model(model,asl_model):\n    inputs = tf.keras.Input(shape=(ROWS_PER_FRAME,3), name=\"inputs\")\n    \n    # drop most of the face mesh\n    x = tf.gather(inputs, LANDMARK_IDX, axis=1)\n\n    # fill nan\n    x = tf.where(tf.math.is_nan(x), tf.zeros_like(x), x)\n\n    # flatten landmark xyz coordinates ()\n    x = tf.concat([x[...,i] for i in range(3)], -1)\n\n    x = tf.expand_dims(x,0)\n    \n    # call trained model\n    out = model(x)\n    \n    #model 2\n    prep_inputs = FeatureGen()\n#     x2 = prep_inputs(tf.cast(inputs, dtype=tf.float32))\n#     out2 = asl_model(x2)\n    x2 = prep_inputs(tf.cast(inputs, dtype=tf.float32))\n    out2 = asl_model(x2)\n    \n    #out3 = tf.keras.layers.Average()([out,out2])\n    out3 = tf.keras.layers.Add()([0.45*out,0.55*out2])\n    \n    # explicitly name the final (identity) layer for the submission format\n    out = layers.Activation(\"linear\", name=\"outputs\")(out3)\n    \n    inference_model = tf.keras.Model(inputs=inputs, outputs=out)\n    inference_model.compile(loss=\"sparse_categorical_crossentropy\",\n                            metrics=\"accuracy\")\n    return inference_model","metadata":{"execution":{"iopub.status.busy":"2023-03-27T14:58:51.613553Z","iopub.execute_input":"2023-03-27T14:58:51.614121Z","iopub.status.idle":"2023-03-27T14:58:51.628204Z","shell.execute_reply.started":"2023-03-27T14:58:51.614076Z","shell.execute_reply":"2023-03-27T14:58:51.626955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inference_model = get_inference_model(ltsm_model,asl_model)\ntf.keras.utils.plot_model(inference_model)","metadata":{"execution":{"iopub.status.busy":"2023-03-27T14:58:51.908634Z","iopub.execute_input":"2023-03-27T14:58:51.911941Z","iopub.status.idle":"2023-03-27T14:58:53.431619Z","shell.execute_reply.started":"2023-03-27T14:58:51.911888Z","shell.execute_reply":"2023-03-27T14:58:53.429624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inference_model.summary(expand_nested=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Convert model","metadata":{}},{"cell_type":"code","source":"converter = tf.lite.TFLiteConverter.from_keras_model(inference_model)\ntflite_model = converter.convert()\nmodel_path = \"model.tflite\"\n\nwith open(model_path, 'wb') as f:\n    f.write(tflite_model)\n!zip submission.zip $model_path","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}