{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"raw","source":"","metadata":{"papermill":{"duration":0.014869,"end_time":"2023-04-30T13:11:15.705499","exception":false,"start_time":"2023-04-30T13:11:15.690630","status":"completed"},"tags":[]}},{"cell_type":"raw","source":"Reference: https://www.kaggle.com/code/markwijkhuizen/gislr-tf-data-processing-transformer-training","metadata":{"papermill":{"duration":0.013099,"end_time":"2023-04-30T13:11:15.731969","exception":false,"start_time":"2023-04-30T13:11:15.718870","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nimport matplotlib.pyplot as plt\nimport matplotlib as mpl\nimport seaborn as sn\n\nfrom tqdm.notebook import tqdm\nfrom sklearn.model_selection import train_test_split, GroupShuffleSplit \n\nimport glob\nimport sys\nimport os\nimport math\nimport gc\nimport sys\nimport sklearn\nimport scipy","metadata":{"papermill":{"duration":9.10292,"end_time":"2023-04-30T13:11:24.849621","exception":false,"start_time":"2023-04-30T13:11:15.746701","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:10.107872Z","iopub.execute_input":"2023-05-01T22:14:10.108555Z","iopub.status.idle":"2023-05-01T22:14:18.321740Z","shell.execute_reply.started":"2023-05-01T22:14:10.108490Z","shell.execute_reply":"2023-05-01T22:14:18.320563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot Config","metadata":{"papermill":{"duration":0.01244,"end_time":"2023-04-30T13:11:24.875182","exception":false,"start_time":"2023-04-30T13:11:24.862742","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# MatplotLib Global Settings\nmpl.rcParams.update(mpl.rcParamsDefault)\nmpl.rcParams['xtick.labelsize'] = 14\nmpl.rcParams['ytick.labelsize'] = 14\nmpl.rcParams['axes.labelsize'] = 14\nmpl.rcParams['axes.titlesize'] = 16","metadata":{"papermill":{"duration":0.024409,"end_time":"2023-04-30T13:11:24.912365","exception":false,"start_time":"2023-04-30T13:11:24.887956","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:18.324204Z","iopub.execute_input":"2023-05-01T22:14:18.325418Z","iopub.status.idle":"2023-05-01T22:14:18.332783Z","shell.execute_reply.started":"2023-05-01T22:14:18.325375Z","shell.execute_reply":"2023-05-01T22:14:18.331296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{"papermill":{"duration":0.012684,"end_time":"2023-04-30T13:11:24.937820","exception":false,"start_time":"2023-04-30T13:11:24.925136","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# If True, processing data from scratch\n# If False, loads preprocessed data\nPREPROCESS_DATA = False\nTRAIN_MODEL = True\n\nN_ROWS = 543\nN_DIMS = 3\nDIM_NAMES = ['x', 'y', 'z']\nSEED = 42\nNUM_CLASSES = 250\n#IS_INTERACTIVE = os.environ['KAGGLE_KERNEL_RUN_TYPE'] == 'Interactive'\n#VERBOSE = 1 if IS_INTERACTIVE else 2\n\nSeq_len = 32\n\nBATCH_ALL_SIGNS_N = 4\nBATCH_SIZE = 256\nN_EPOCHS = 140\nLR_MAX = 1e-3\nN_WARMUP_EPOCHS = 0\n#WD_RATIO = 0.05\nWD_RATIO = 0.8\n#MASK_VAL = 4237","metadata":{"papermill":{"duration":0.02303,"end_time":"2023-04-30T13:11:24.973685","exception":false,"start_time":"2023-04-30T13:11:24.950655","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:18.335046Z","iopub.execute_input":"2023-05-01T22:14:18.335638Z","iopub.status.idle":"2023-05-01T22:14:18.345848Z","shell.execute_reply.started":"2023-05-01T22:14:18.335600Z","shell.execute_reply":"2023-05-01T22:14:18.344882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utils","metadata":{"papermill":{"duration":0.013181,"end_time":"2023-04-30T13:11:24.999666","exception":false,"start_time":"2023-04-30T13:11:24.986485","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Prints Shape and Dtype For List Of Variables\ndef print_shape_dtype(l, names):\n    for e, n in zip(l, names):\n        print(f'{n} shape: {e.shape}, dtype: {e.dtype}')","metadata":{"papermill":{"duration":0.022307,"end_time":"2023-04-30T13:11:25.034859","exception":false,"start_time":"2023-04-30T13:11:25.012552","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:18.349048Z","iopub.execute_input":"2023-05-01T22:14:18.349501Z","iopub.status.idle":"2023-05-01T22:14:18.357215Z","shell.execute_reply.started":"2023-05-01T22:14:18.349461Z","shell.execute_reply":"2023-05-01T22:14:18.356240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{"papermill":{"duration":0.013876,"end_time":"2023-04-30T13:11:25.061680","exception":false,"start_time":"2023-04-30T13:11:25.047804","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# # Read Training Data\n# if IS_INTERACTIVE or not PREPROCESS_DATA:\n#     train = pd.read_csv('/kaggle/input/asl-signs/train.csv').sample(int(5e3), random_state=SEED)\n# else:\n#     train = pd.read_csv('/kaggle/input/asl-signs/train.csv')\n\ntrain = pd.read_csv('/kaggle/input/asl-signs/train.csv')\n\nN_SAMPLES = len(train)\nprint(f'N_SAMPLES: {N_SAMPLES}')","metadata":{"papermill":{"duration":0.254812,"end_time":"2023-04-30T13:11:25.329325","exception":false,"start_time":"2023-04-30T13:11:25.074513","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:18.358448Z","iopub.execute_input":"2023-05-01T22:14:18.358742Z","iopub.status.idle":"2023-05-01T22:14:18.589253Z","shell.execute_reply.started":"2023-05-01T22:14:18.358701Z","shell.execute_reply":"2023-05-01T22:14:18.587194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Add File Path","metadata":{"papermill":{"duration":0.012965,"end_time":"2023-04-30T13:11:25.355855","exception":false,"start_time":"2023-04-30T13:11:25.342890","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Get complete file path to file\ndef get_file_path(path):\n    return f'/kaggle/input/asl-signs/{path}'\n\ntrain['file_path'] = train['path'].apply(get_file_path)","metadata":{"papermill":{"duration":0.059158,"end_time":"2023-04-30T13:11:25.428011","exception":false,"start_time":"2023-04-30T13:11:25.368853","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:18.590910Z","iopub.execute_input":"2023-05-01T22:14:18.591284Z","iopub.status.idle":"2023-05-01T22:14:18.633109Z","shell.execute_reply.started":"2023-05-01T22:14:18.591245Z","shell.execute_reply":"2023-05-01T22:14:18.632149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"papermill":{"duration":0.035306,"end_time":"2023-04-30T13:11:25.477201","exception":false,"start_time":"2023-04-30T13:11:25.441895","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:18.634563Z","iopub.execute_input":"2023-05-01T22:14:18.634938Z","iopub.status.idle":"2023-05-01T22:14:18.652373Z","shell.execute_reply.started":"2023-05-01T22:14:18.634901Z","shell.execute_reply":"2023-05-01T22:14:18.651247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ordinally Encode Sign","metadata":{"papermill":{"duration":0.013481,"end_time":"2023-04-30T13:11:25.504095","exception":false,"start_time":"2023-04-30T13:11:25.490614","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import json\nwith open(\"/kaggle/input/asl-signs/sign_to_prediction_index_map.json\") as f:\n    sign_map = json.load(f)\nsign_list = list(sign_map.keys())","metadata":{"papermill":{"duration":0.026904,"end_time":"2023-04-30T13:11:25.544270","exception":false,"start_time":"2023-04-30T13:11:25.517366","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:18.654310Z","iopub.execute_input":"2023-05-01T22:14:18.654732Z","iopub.status.idle":"2023-05-01T22:14:18.663034Z","shell.execute_reply.started":"2023-05-01T22:14:18.654696Z","shell.execute_reply":"2023-05-01T22:14:18.661986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#sign_map","metadata":{"papermill":{"duration":0.022161,"end_time":"2023-04-30T13:11:25.579947","exception":false,"start_time":"2023-04-30T13:11:25.557786","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:18.664920Z","iopub.execute_input":"2023-05-01T22:14:18.665311Z","iopub.status.idle":"2023-05-01T22:14:18.671801Z","shell.execute_reply.started":"2023-05-01T22:14:18.665270Z","shell.execute_reply":"2023-05-01T22:14:18.670844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mapping_sign_code(sign: str):\n    return sign_map[sign]","metadata":{"papermill":{"duration":0.021896,"end_time":"2023-04-30T13:11:25.615600","exception":false,"start_time":"2023-04-30T13:11:25.593704","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:18.677629Z","iopub.execute_input":"2023-05-01T22:14:18.677888Z","iopub.status.idle":"2023-05-01T22:14:18.682727Z","shell.execute_reply.started":"2023-05-01T22:14:18.677864Z","shell.execute_reply":"2023-05-01T22:14:18.681591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sign_list[:4]","metadata":{"papermill":{"duration":0.024114,"end_time":"2023-04-30T13:11:25.652862","exception":false,"start_time":"2023-04-30T13:11:25.628748","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:18.684340Z","iopub.execute_input":"2023-05-01T22:14:18.685383Z","iopub.status.idle":"2023-05-01T22:14:18.694888Z","shell.execute_reply.started":"2023-05-01T22:14:18.685341Z","shell.execute_reply":"2023-05-01T22:14:18.693410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['sign_code'] = train['sign'].apply(mapping_sign_code)","metadata":{"papermill":{"duration":0.060515,"end_time":"2023-04-30T13:11:25.726481","exception":false,"start_time":"2023-04-30T13:11:25.665966","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:18.696552Z","iopub.execute_input":"2023-05-01T22:14:18.697027Z","iopub.status.idle":"2023-05-01T22:14:18.740861Z","shell.execute_reply.started":"2023-05-01T22:14:18.696993Z","shell.execute_reply":"2023-05-01T22:14:18.739866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Add ordinally Encoded Sign (assign number to each sign name)\ntrain['sign_ord'] = train['sign'].astype('category').cat.codes\n\n\n# Dictionaries to translate sign <-> ordinal encoded sign\nSIGN2ORD = train[['sign', 'sign_ord']].set_index('sign').squeeze().to_dict()\nORD2SIGN = train[['sign_ord', 'sign']].set_index('sign_ord').squeeze().to_dict()","metadata":{"papermill":{"duration":0.211492,"end_time":"2023-04-30T13:11:25.951140","exception":false,"start_time":"2023-04-30T13:11:25.739648","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:18.742433Z","iopub.execute_input":"2023-05-01T22:14:18.742804Z","iopub.status.idle":"2023-05-01T22:14:18.917544Z","shell.execute_reply.started":"2023-05-01T22:14:18.742764Z","shell.execute_reply":"2023-05-01T22:14:18.916471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train.head(10))\ndisplay(train.info())","metadata":{"papermill":{"duration":0.061319,"end_time":"2023-04-30T13:11:26.025882","exception":false,"start_time":"2023-04-30T13:11:25.964563","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:18.919130Z","iopub.execute_input":"2023-05-01T22:14:18.919520Z","iopub.status.idle":"2023-05-01T22:14:18.959513Z","shell.execute_reply.started":"2023-05-01T22:14:18.919480Z","shell.execute_reply":"2023-05-01T22:14:18.958451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Process Data Tensorflow","metadata":{"papermill":{"duration":0.013481,"end_time":"2023-04-30T13:11:26.053701","exception":false,"start_time":"2023-04-30T13:11:26.040220","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Source: https://www.kaggle.com/competitions/asl-signs/overview/evaluation\nROWS_PER_FRAME = 543  # number of landmarks per frame\n\ndef load_relevant_data_subset(pq_path):\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)","metadata":{"papermill":{"duration":0.02482,"end_time":"2023-04-30T13:11:26.092383","exception":false,"start_time":"2023-04-30T13:11:26.067563","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:18.960917Z","iopub.execute_input":"2023-05-01T22:14:18.961837Z","iopub.status.idle":"2023-05-01T22:14:18.969168Z","shell.execute_reply.started":"2023-05-01T22:14:18.961798Z","shell.execute_reply":"2023-05-01T22:14:18.968172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n    Tensorflow layer to process data in TFLite\n    Data needs to be processed in the model itself, so we can not use Python\n\"\"\" \nclass PreprocessLayer(tf.keras.layers.Layer):\n    def __init__(self, Seq_len):\n        super(PreprocessLayer, self).__init__()\n        self.Seq_len = Seq_len\n        \n    def pad_edge(self, t, repeats, side):\n        if side == 'LEFT':\n            return tf.concat((tf.repeat(t[:1], repeats=repeats, axis=0), t), axis=0)\n        elif side == 'RIGHT':\n            return tf.concat((t, tf.repeat(t[-1:], repeats=repeats, axis=0)), axis=0)\n    \n    @tf.function(input_signature=(tf.TensorSpec(shape=[None,N_ROWS,N_DIMS], dtype=tf.float32),),)\n    # @tf.function\n    def call(self, data0):\n        # Number of Frames in Video\n        N_FRAMES0 = tf.shape(data0)[0]\n        \n        # Filter Out Frames With Empty Hand Data\n        frames_hands_nansum = tf.experimental.numpy.nanmean(tf.gather(data0, HAND_IDXS0, axis=1), axis=[1,2])\n        non_empty_frames_idxs = tf.where(frames_hands_nansum > 0)\n        non_empty_frames_idxs = tf.squeeze(non_empty_frames_idxs, axis=1)\n        data = tf.gather(data0, non_empty_frames_idxs, axis=0)\n        \n        # Cast Indices in float32 to be compatible with Tensorflow Lite\n        non_empty_frames_idxs = tf.cast(non_empty_frames_idxs, tf.float32) \n\n        \n        # Number of Frames in Filtered Video\n        N_FRAMES = tf.shape(data)[0]\n        \n        # Gather Relevant Landmark Columns\n        data = tf.gather(data, LANDMARK_IDXS0, axis=1)\n        \n        # Video fits in Seq_len\n        if N_FRAMES < self.Seq_len:\n            # Pad With -1 to indicate padding\n            non_empty_frames_idxs = tf.pad(non_empty_frames_idxs, [[0, self.Seq_len-N_FRAMES]], constant_values=-1)\n            # Pad Data With Zeros\n            data = tf.pad(data, [[0, self.Seq_len-N_FRAMES], [0,0], [0,0]], constant_values=0)\n            # Fill NaN Values With 0\n            data = tf.where(tf.math.is_nan(data), 0.0, data)\n            return data, non_empty_frames_idxs\n        # Video needs to be downsampled to Seq_len\n        else:\n            # Repeat\n            if N_FRAMES < self.Seq_len**2:\n                repeats = tf.math.floordiv(self.Seq_len * self.Seq_len, N_FRAMES0)\n                data = tf.repeat(data, repeats=repeats, axis=0)\n                non_empty_frames_idxs = tf.repeat(non_empty_frames_idxs, repeats=repeats, axis=0)\n\n            # Pad To Multiple Of Input Size\n            pool_size = tf.math.floordiv(len(data), self.Seq_len)\n            if tf.math.mod(len(data), self.Seq_len) > 0:\n                pool_size += 1\n\n            if pool_size == 1:\n                pad_size = (pool_size * self.Seq_len) - len(data)\n            else:\n                pad_size = (pool_size * self.Seq_len) % len(data)\n\n            # Pad Start/End with Start/End value\n            pad_left = tf.math.floordiv(pad_size, 2) + tf.math.floordiv(self.Seq_len, 2)\n            pad_right = tf.math.floordiv(pad_size, 2) + tf.math.floordiv(self.Seq_len, 2)\n            if tf.math.mod(pad_size, 2) > 0:\n                pad_right += 1\n\n            # Pad By Concatenating Left/Right Edge Values\n            data = self.pad_edge(data, pad_left, 'LEFT')\n            data = self.pad_edge(data, pad_right, 'RIGHT')\n\n            # Pad Non Empty Frame Indices\n            non_empty_frames_idxs = self.pad_edge(non_empty_frames_idxs, pad_left, 'LEFT')\n            non_empty_frames_idxs = self.pad_edge(non_empty_frames_idxs, pad_right, 'RIGHT')\n\n            # Reshape to Mean Pool\n            data = tf.reshape(data, [self.Seq_len, -1, N_COLS, N_DIMS])\n            non_empty_frames_idxs = tf.reshape(non_empty_frames_idxs, [self.Seq_len, -1])\n\n            # Mean Pool\n            data = tf.experimental.numpy.nanmean(data, axis=1)\n            non_empty_frames_idxs = tf.experimental.numpy.nanmean(non_empty_frames_idxs, axis=1)\n\n            # Fill NaN Values With 0\n            data = tf.where(tf.math.is_nan(data), 0.0, data)\n            \n            return data, non_empty_frames_idxs\n    ","metadata":{"papermill":{"duration":0.039253,"end_time":"2023-04-30T13:11:26.145426","exception":false,"start_time":"2023-04-30T13:11:26.106173","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:18.970839Z","iopub.execute_input":"2023-05-01T22:14:18.971273Z","iopub.status.idle":"2023-05-01T22:14:18.994136Z","shell.execute_reply.started":"2023-05-01T22:14:18.971198Z","shell.execute_reply":"2023-05-01T22:14:18.992788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocess_layer32 = PreprocessLayer(32)\npreprocess_layer64 = PreprocessLayer(64)","metadata":{"papermill":{"duration":0.0328,"end_time":"2023-04-30T13:11:26.191913","exception":false,"start_time":"2023-04-30T13:11:26.159113","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:18.995949Z","iopub.execute_input":"2023-05-01T22:14:18.996513Z","iopub.status.idle":"2023-05-01T22:14:19.020703Z","shell.execute_reply.started":"2023-05-01T22:14:18.996477Z","shell.execute_reply":"2023-05-01T22:14:19.019415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Interpolate NaN Values","metadata":{"papermill":{"duration":0.013684,"end_time":"2023-04-30T13:11:26.219486","exception":false,"start_time":"2023-04-30T13:11:26.205802","status":"completed"},"tags":[]}},{"cell_type":"code","source":"\"\"\"\n    face: 0:468\n    left_hand: 468:489\n    pose: 489:522\n    right_hand: 522:544\n        \n\"\"\"\ndef get_data(file_path, preprocess_layer):\n    # Load Raw Data\n    data = load_relevant_data_subset(file_path)\n    # Process Data Using Tensorflow\n    data = preprocess_layer(data)\n    \n    return data","metadata":{"papermill":{"duration":0.022698,"end_time":"2023-04-30T13:11:26.255862","exception":false,"start_time":"2023-04-30T13:11:26.233164","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:19.022459Z","iopub.execute_input":"2023-05-01T22:14:19.022845Z","iopub.status.idle":"2023-05-01T22:14:19.029618Z","shell.execute_reply.started":"2023-05-01T22:14:19.022807Z","shell.execute_reply":"2023-05-01T22:14:19.028278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"USE_TYPES = ['left_hand', 'pose', 'right_hand']\nSTART_IDX = 468\nLIPS_IDXS0 = np.array([\n        61, 185, 40, 39, 37, 0, 267, 269, 270, 409,\n        291, 146, 91, 181, 84, 17, 314, 405, 321, 375,\n        78, 191, 80, 81, 82, 13, 312, 311, 310, 415,\n        95, 88, 178, 87, 14, 317, 402, 318, 324, 308,\n    ])\n# Landmark indices in original data\nLEFT_HAND_IDXS0 = np.arange(468,489)\nRIGHT_HAND_IDXS0 = np.arange(522,543)\nPOSE_IDXS0 = np.arange(502, 512)\nLANDMARK_IDXS0 = np.concatenate((LIPS_IDXS0, LEFT_HAND_IDXS0, RIGHT_HAND_IDXS0, POSE_IDXS0))\nHAND_IDXS0 = np.concatenate((LEFT_HAND_IDXS0, RIGHT_HAND_IDXS0), axis=0)\nN_COLS = LANDMARK_IDXS0.size\n# Landmark indices in processed data\nLIPS_IDXS = np.argwhere(np.isin(LANDMARK_IDXS0, LIPS_IDXS0)).squeeze()\nLEFT_HAND_IDXS = np.argwhere(np.isin(LANDMARK_IDXS0, LEFT_HAND_IDXS0)).squeeze()\nRIGHT_HAND_IDXS = np.argwhere(np.isin(LANDMARK_IDXS0, RIGHT_HAND_IDXS0)).squeeze()\nHAND_IDXS = np.argwhere(np.isin(LANDMARK_IDXS0, HAND_IDXS0)).squeeze()\nPOSE_IDXS = np.argwhere(np.isin(LANDMARK_IDXS0, POSE_IDXS0)).squeeze()\n\nprint(f'# HAND_IDXS: {len(HAND_IDXS)}, N_COLS: {N_COLS}')","metadata":{"papermill":{"duration":0.030409,"end_time":"2023-04-30T13:11:26.299807","exception":false,"start_time":"2023-04-30T13:11:26.269398","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:19.031770Z","iopub.execute_input":"2023-05-01T22:14:19.032241Z","iopub.status.idle":"2023-05-01T22:14:19.047253Z","shell.execute_reply.started":"2023-05-01T22:14:19.032203Z","shell.execute_reply":"2023-05-01T22:14:19.046038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LIPS_START = 0\nLEFT_HAND_START = LIPS_IDXS.size\nRIGHT_HAND_START = LEFT_HAND_START + LEFT_HAND_IDXS.size\nPOSE_START = RIGHT_HAND_START + RIGHT_HAND_IDXS.size\n\nprint(f'LIPS_START: {LIPS_START}, LEFT_HAND_START: {LEFT_HAND_START}, RIGHT_HAND_START: {RIGHT_HAND_START}, POSE_START: {POSE_START}')","metadata":{"papermill":{"duration":0.023621,"end_time":"2023-04-30T13:11:26.337105","exception":false,"start_time":"2023-04-30T13:11:26.313484","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:19.048865Z","iopub.execute_input":"2023-05-01T22:14:19.049859Z","iopub.status.idle":"2023-05-01T22:14:19.057368Z","shell.execute_reply.started":"2023-05-01T22:14:19.049818Z","shell.execute_reply":"2023-05-01T22:14:19.056035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Dataset","metadata":{"papermill":{"duration":0.013741,"end_time":"2023-04-30T13:11:26.364916","exception":false,"start_time":"2023-04-30T13:11:26.351175","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Get the full dataset\ndef get_x_y(X_name:str, y_name: str, NON_EMPTY_FRAME_IDXS_name: str, Seq_len: int, preprocess_layer, PREPROCESS_DATA):\n    \n    def get_data(file_path, preprocess_layer):\n    # Load Raw Data\n        data = load_relevant_data_subset(file_path)\n        # Process Data Using Tensorflow\n        data = preprocess_layer(data)\n        return data\n    # Save X/y\n    if PREPROCESS_DATA:\n        \n        # Create arrays to save data\n        X = np.zeros([N_SAMPLES, Seq_len, N_COLS, N_DIMS], dtype=np.float32)\n        y = np.zeros([N_SAMPLES], dtype=np.int32)\n        NON_EMPTY_FRAME_IDXS = np.full([N_SAMPLES, Seq_len], -1, dtype=np.float32)\n\n        for row_idx, (file_path, sign_ord) in enumerate(tqdm(train[['file_path', 'sign_ord']].values)):\n            if row_idx % 5000 == 0:\n                print(f'Generated {row_idx}/{N_SAMPLES}')\n\n            data, non_empty_frame_idxs = get_data(file_path, preprocess_layer)\n            X[row_idx] = data\n            y[row_idx] = sign_ord\n            NON_EMPTY_FRAME_IDXS[row_idx] = non_empty_frame_idxs\n            if np.isnan(data).sum() > 0:\n                print(row_idx)\n                return data\n            \n        np.save(X_name, X)\n        np.save(y_name, y)\n        np.save(NON_EMPTY_FRAME_IDXS_name, NON_EMPTY_FRAME_IDXS)\n    \n    else:        \n        X = np.load(X_name)\n        y = np.load(y_name)\n        NON_EMPTY_FRAME_IDXS = np.load(NON_EMPTY_FRAME_IDXS_name)\n\n    return X, y, NON_EMPTY_FRAME_IDXS","metadata":{"papermill":{"duration":0.028006,"end_time":"2023-04-30T13:11:26.406691","exception":false,"start_time":"2023-04-30T13:11:26.378685","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:19.059023Z","iopub.execute_input":"2023-05-01T22:14:19.060260Z","iopub.status.idle":"2023-05-01T22:14:19.071239Z","shell.execute_reply.started":"2023-05-01T22:14:19.060220Z","shell.execute_reply":"2023-05-01T22:14:19.070523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Statistics - Lips","metadata":{"papermill":{"duration":0.013723,"end_time":"2023-04-30T13:11:26.434194","exception":false,"start_time":"2023-04-30T13:11:26.420471","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# LIPS_MEAN32, LIPS_STD32 = get_lips_mean_std(X32)","metadata":{"papermill":{"duration":0.021744,"end_time":"2023-04-30T13:11:26.470388","exception":false,"start_time":"2023-04-30T13:11:26.448644","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:19.072810Z","iopub.execute_input":"2023-05-01T22:14:19.073523Z","iopub.status.idle":"2023-05-01T22:14:19.083817Z","shell.execute_reply.started":"2023-05-01T22:14:19.073481Z","shell.execute_reply":"2023-05-01T22:14:19.082947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LIPS_MEAN32 = np.load('/kaggle/input/transformer-weights/LIPS_MEAN32.npy')\nLIPS_STD32 =np.load('/kaggle/input/transformer-weights/LIPS_STD32.npy')","metadata":{"papermill":{"duration":0.037836,"end_time":"2023-04-30T13:11:26.522148","exception":false,"start_time":"2023-04-30T13:11:26.484312","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:19.085296Z","iopub.execute_input":"2023-05-01T22:14:19.086174Z","iopub.status.idle":"2023-05-01T22:14:19.101836Z","shell.execute_reply.started":"2023-05-01T22:14:19.086083Z","shell.execute_reply":"2023-05-01T22:14:19.100964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LIPS_MEAN64 = np.load('/kaggle/input/transformer-weights/LIPS_MEAN64.npy')\nLIPS_STD64 = np.load('/kaggle/input/transformer-weights/LIPS_STD64.npy')","metadata":{"papermill":{"duration":0.034455,"end_time":"2023-04-30T13:11:26.570514","exception":false,"start_time":"2023-04-30T13:11:26.536059","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:19.103148Z","iopub.execute_input":"2023-05-01T22:14:19.103930Z","iopub.status.idle":"2023-05-01T22:14:19.117354Z","shell.execute_reply.started":"2023-05-01T22:14:19.103890Z","shell.execute_reply":"2023-05-01T22:14:19.116480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Statistics - Hands","metadata":{"papermill":{"duration":0.013679,"end_time":"2023-04-30T13:11:26.597646","exception":false,"start_time":"2023-04-30T13:11:26.583967","status":"completed"},"tags":[]}},{"cell_type":"code","source":"LEFT_HANDS_MEAN32 = np.load('/kaggle/input/transformer-weights/LEFT_HANDS_MEAN32.npy')\nLEFT_HANDS_STD32 = np.load('/kaggle/input/transformer-weights/LEFT_HANDS_STD32.npy')\nRIGHT_HANDS_MEAN32 = np.load('/kaggle/input/transformer-weights/RIGHT_HANDS_MEAN32.npy')\nRIGHT_HANDS_STD32 = np.load('/kaggle/input/transformer-weights/RIGHT_HANDS_STD32.npy')","metadata":{"papermill":{"duration":0.051269,"end_time":"2023-04-30T13:11:26.662556","exception":false,"start_time":"2023-04-30T13:11:26.611287","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:19.118862Z","iopub.execute_input":"2023-05-01T22:14:19.119224Z","iopub.status.idle":"2023-05-01T22:14:19.145212Z","shell.execute_reply.started":"2023-05-01T22:14:19.119186Z","shell.execute_reply":"2023-05-01T22:14:19.144251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LEFT_HANDS_MEAN64 = np.load('/kaggle/input/transformer-weights/LEFT_HANDS_MEAN64.npy')\nLEFT_HANDS_STD64 = np.load('/kaggle/input/transformer-weights/LEFT_HANDS_STD64.npy')\nRIGHT_HANDS_MEAN64 = np.load('/kaggle/input/transformer-weights/RIGHT_HANDS_MEAN64.npy')\nRIGHT_HANDS_STD64 = np.load('/kaggle/input/transformer-weights/RIGHT_HANDS_STD64.npy')","metadata":{"papermill":{"duration":0.046662,"end_time":"2023-04-30T13:11:26.723471","exception":false,"start_time":"2023-04-30T13:11:26.676809","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:19.146558Z","iopub.execute_input":"2023-05-01T22:14:19.147273Z","iopub.status.idle":"2023-05-01T22:14:19.167138Z","shell.execute_reply.started":"2023-05-01T22:14:19.147237Z","shell.execute_reply":"2023-05-01T22:14:19.166278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Statistics - Pose","metadata":{"papermill":{"duration":0.013444,"end_time":"2023-04-30T13:11:26.750648","exception":false,"start_time":"2023-04-30T13:11:26.737204","status":"completed"},"tags":[]}},{"cell_type":"code","source":"POSE_MEAN32 = np.load('/kaggle/input/transformer-weights/POSE_MEAN32.npy')\nPOSE_STD32 = np.load('/kaggle/input/transformer-weights/POSE_STD32.npy')","metadata":{"papermill":{"duration":0.033093,"end_time":"2023-04-30T13:11:26.797666","exception":false,"start_time":"2023-04-30T13:11:26.764573","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:19.169595Z","iopub.execute_input":"2023-05-01T22:14:19.169854Z","iopub.status.idle":"2023-05-01T22:14:19.179629Z","shell.execute_reply.started":"2023-05-01T22:14:19.169830Z","shell.execute_reply":"2023-05-01T22:14:19.178598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"POSE_MEAN64 = np.load('/kaggle/input/transformer-weights/POSE_MEAN64.npy')\nPOSE_STD64 = np.load('/kaggle/input/transformer-weights/POSE_STD64.npy')","metadata":{"papermill":{"duration":0.035384,"end_time":"2023-04-30T13:11:26.846979","exception":false,"start_time":"2023-04-30T13:11:26.811595","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:19.186259Z","iopub.execute_input":"2023-05-01T22:14:19.187097Z","iopub.status.idle":"2023-05-01T22:14:19.197046Z","shell.execute_reply.started":"2023-05-01T22:14:19.187068Z","shell.execute_reply":"2023-05-01T22:14:19.196061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Samples","metadata":{"papermill":{"duration":0.013567,"end_time":"2023-04-30T13:11:26.874479","exception":false,"start_time":"2023-04-30T13:11:26.860912","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Custom sampler to get a batch containing N times all signs\ndef get_train_batch_all_signs(X, y, NON_EMPTY_FRAME_IDXS, X_batch_name, NON_EMPTY_FRAME_IDXS_name, Seq_len, n=BATCH_ALL_SIGNS_N):\n    # Arrays to store batch in\n    X_batch = np.zeros([NUM_CLASSES*n, Seq_len, N_COLS, N_DIMS], dtype=np.float32)\n    y_batch = np.arange(0, NUM_CLASSES, step=1/n, dtype=np.float32).astype(np.int64)\n    non_empty_frame_idxs_batch = np.zeros([NUM_CLASSES*n, Seq_len], dtype=np.float32)\n    \n    # Dictionary mapping ordinally encoded sign to corresponding sample indices\n    CLASS2IDXS = {}\n    for i in range(NUM_CLASSES):\n        CLASS2IDXS[i] = np.argwhere(y == i).squeeze().astype(np.int32)\n            \n    while True:\n        # Fill batch arrays\n        for i in range(NUM_CLASSES):\n            idxs = np.random.choice(CLASS2IDXS[i], n)\n            X_batch[i*n:(i+1)*n] = X[idxs]\n            non_empty_frame_idxs_batch[i*n:(i+1)*n] = NON_EMPTY_FRAME_IDXS[idxs]\n        yield { X_batch_name: X_batch, NON_EMPTY_FRAME_IDXS_name: non_empty_frame_idxs_batch }, y_batch\n        # yield { 'frames': X_batch, 'non_empty_frame_idxs': non_empty_frame_idxs_batch }, y_batch","metadata":{"papermill":{"duration":0.026528,"end_time":"2023-04-30T13:11:26.914832","exception":false,"start_time":"2023-04-30T13:11:26.888304","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:19.198659Z","iopub.execute_input":"2023-05-01T22:14:19.199363Z","iopub.status.idle":"2023-05-01T22:14:19.208710Z","shell.execute_reply.started":"2023-05-01T22:14:19.199328Z","shell.execute_reply":"2023-05-01T22:14:19.207769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Config","metadata":{"papermill":{"duration":0.013625,"end_time":"2023-04-30T13:11:26.942051","exception":false,"start_time":"2023-04-30T13:11:26.928426","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Epsilon value for layer normalisation\nLAYER_NORM_EPS = 1e-6\n# Epsilon value for layer normalisation\nLAYER_NORM_EPS = 1e-6\n\n# Dense layer units for landmarks\nemb_size = 96\nLIPS_UNITS = emb_size\nHANDS_UNITS = emb_size\nPOSE_UNITS = emb_size\n# final embedding and transformer embedding size\nUNITS = 2*emb_size\n\n# Transformer\nNUM_BLOCKS = 2\nMLP_RATIO = 2\n\n# Dropout\nEMBEDDING_DROPOUT = 0.3\nMLP_DROPOUT_RATIO = 0.3\nCLASSIFIER_DROPOUT_RATIO = 0.4\n\n# Initiailizers\nINIT_HE_UNIFORM = tf.keras.initializers.he_uniform\nINIT_GLOROT_UNIFORM = tf.keras.initializers.glorot_uniform\nINIT_ZEROS = tf.keras.initializers.constant(0.0)\n# Activations\nGELU = tf.keras.activations.gelu","metadata":{"papermill":{"duration":0.02364,"end_time":"2023-04-30T13:11:26.979470","exception":false,"start_time":"2023-04-30T13:11:26.955830","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:19.210264Z","iopub.execute_input":"2023-05-01T22:14:19.211018Z","iopub.status.idle":"2023-05-01T22:14:19.221973Z","shell.execute_reply.started":"2023-05-01T22:14:19.210982Z","shell.execute_reply":"2023-05-01T22:14:19.221032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transformer\n\nNeed to implement transformer from scratch as TFLite does not support the native TF implementation of MultiHeadAttention.","metadata":{"papermill":{"duration":0.013496,"end_time":"2023-04-30T13:11:27.006623","exception":false,"start_time":"2023-04-30T13:11:26.993127","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# based on: https://stackoverflow.com/questions/67342988/verifying-the-implementation-of-multihead-attention-in-transformer\n# replaced softmax with softmax layer to support masked softmax\ndef scaled_dot_product(q, k, v, softmax, attention_mask):\n    #calculates Q . K(transpose)\n    qkt = tf.matmul(q,k,transpose_b=True)\n    #caculates scaling factor\n    dk = tf.math.sqrt(tf.cast(q.shape[-1],dtype=tf.float32))\n    scaled_qkt = qkt/dk\n    softmax = softmax(scaled_qkt, mask=attention_mask)\n    \n    z = tf.matmul(softmax,v)\n    #shape: (m,Tx,depth), same shape as q,k,v\n    return z\n\nclass ISLRMultiHeadAttention(tf.keras.layers.Layer):\n    def __init__(self,d_model,num_of_heads):\n        super(ISLRMultiHeadAttention,self).__init__()\n        self.d_model = d_model\n        self.num_of_heads = num_of_heads\n        self.depth = d_model//num_of_heads\n        self.wq = [tf.keras.layers.Dense(self.depth) for i in range(num_of_heads)]\n        self.wk = [tf.keras.layers.Dense(self.depth) for i in range(num_of_heads)]\n        self.wv = [tf.keras.layers.Dense(self.depth) for i in range(num_of_heads)]\n        self.wo = tf.keras.layers.Dense(d_model)\n        self.softmax = tf.keras.layers.Softmax()\n        \n    def call(self,x, attention_mask):\n        \n        multi_attn = []\n        for i in range(self.num_of_heads):\n            Q = self.wq[i](x)\n            K = self.wk[i](x)\n            V = self.wv[i](x)\n            multi_attn.append(scaled_dot_product(Q,K,V, self.softmax, attention_mask))\n            \n        multi_head = tf.concat(multi_attn,axis=-1)\n        multi_head_attention = self.wo(multi_head)\n        return multi_head_attention","metadata":{"papermill":{"duration":0.029145,"end_time":"2023-04-30T13:11:27.049478","exception":false,"start_time":"2023-04-30T13:11:27.020333","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:19.223566Z","iopub.execute_input":"2023-05-01T22:14:19.224591Z","iopub.status.idle":"2023-05-01T22:14:19.236944Z","shell.execute_reply.started":"2023-05-01T22:14:19.224538Z","shell.execute_reply":"2023-05-01T22:14:19.236006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Full Transformer\nclass ISLRTransformer(tf.keras.Model):\n    def __init__(self, num_blocks, transformer_name):\n        super(ISLRTransformer, self).__init__(name=transformer_name)\n        self.num_blocks = num_blocks\n        self.transformer_name = transformer_name\n    \n    def build(self, input_shape):\n        self.ln_1s = []\n        self.mhas = []\n        self.ln_2s = []\n        self.mlps = []\n        # Make Transformer Blocks\n        for i in range(self.num_blocks):\n            # First Layer Normalisation\n            self.ln_1s.append(tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS, name=self.transformer_name+f'block{i}_layernorm1'))\n            # Multi Head Attention\n            self.mhas.append(ISLRMultiHeadAttention(UNITS, 8))\n            # Second Layer Normalisation\n            self.ln_2s.append(tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS, name=self.transformer_name+f'block{i}_layernorm2'))\n            # Multi Layer Perception\n            self.mlps.append(tf.keras.Sequential([\n                tf.keras.layers.Dense(UNITS * MLP_RATIO, activation=GELU, kernel_initializer=INIT_GLOROT_UNIFORM, name= self.transformer_name +f\"block{i}_dense1\"),\n                tf.keras.layers.Dropout(MLP_DROPOUT_RATIO),\n                tf.keras.layers.Dense(UNITS, kernel_initializer=INIT_HE_UNIFORM, name= self.transformer_name +f\"block{i}__dense2\"),\n            ]))\n        \n    def call(self, x, attention_mask):\n        # Iterate input over transformer blocks\n        for ln_1, mha, ln_2, mlp in zip(self.ln_1s, self.mhas, self.ln_2s, self.mlps):\n            x1 = ln_1(x)\n            attention_output = mha(x1, attention_mask)\n            x2 = x1 + attention_output\n            x3 = ln_2(x2)\n            x3 = mlp(x3)\n            x = x3 + x2\n    \n        return x","metadata":{"papermill":{"duration":0.028517,"end_time":"2023-04-30T13:11:27.091753","exception":false,"start_time":"2023-04-30T13:11:27.063236","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:19.239597Z","iopub.execute_input":"2023-05-01T22:14:19.239886Z","iopub.status.idle":"2023-05-01T22:14:19.252283Z","shell.execute_reply.started":"2023-05-01T22:14:19.239859Z","shell.execute_reply":"2023-05-01T22:14:19.251311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Landmark Embedding","metadata":{"papermill":{"duration":0.013585,"end_time":"2023-04-30T13:11:27.118969","exception":false,"start_time":"2023-04-30T13:11:27.105384","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class LandmarkEmbedding(tf.keras.Model):\n    def __init__(self, units, name):\n        super(LandmarkEmbedding, self).__init__(name=f'{name}_embedding')\n        self.units = units\n        \n    def build(self, input_shape):\n        # Embedding for missing landmark in frame, initizlied with zeros\n        self.empty_embedding = self.add_weight(\n            name=f'{self.name}_empty_embedding',\n            shape=[self.units],\n            initializer=INIT_ZEROS,\n        )\n        # Embedding\n        self.dense = tf.keras.Sequential([\n            tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_1', use_bias=False, kernel_initializer=INIT_GLOROT_UNIFORM, activation=GELU),\n            tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_2', use_bias=False, kernel_initializer=INIT_HE_UNIFORM),\n        ], name=f'{self.name}_dense')\n\n    def call(self, x):\n        return tf.where(\n                # Checks whether landmark is missing in frame\n                tf.reduce_sum(x, axis=2, keepdims=True) == 0,\n                # If so, the empty embedding is used\n                self.empty_embedding,\n                # Otherwise the landmark data is embedded\n                self.dense(x),\n            )","metadata":{"papermill":{"duration":0.027813,"end_time":"2023-04-30T13:11:27.160943","exception":false,"start_time":"2023-04-30T13:11:27.133130","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:19.254012Z","iopub.execute_input":"2023-05-01T22:14:19.255002Z","iopub.status.idle":"2023-05-01T22:14:19.267571Z","shell.execute_reply.started":"2023-05-01T22:14:19.254966Z","shell.execute_reply":"2023-05-01T22:14:19.266419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Embedding","metadata":{"papermill":{"duration":0.013577,"end_time":"2023-04-30T13:11:27.188201","exception":false,"start_time":"2023-04-30T13:11:27.174624","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class ISLREmbedding(tf.keras.Model):\n    def __init__(self, Seq_len, landmark_weights_name):\n        super(ISLREmbedding, self).__init__()\n        self.Seq_len = Seq_len\n        self.landmark_weights_name = landmark_weights_name\n        \n    def get_diffs(self, l):\n        S = l.shape[2]\n        other = tf.expand_dims(l, 3)\n        other = tf.repeat(other, S, axis=3)\n        other = tf.transpose(other, [0,1,3,2])\n        diffs = tf.expand_dims(l, 3) - other\n        diffs = tf.reshape(diffs, [-1, self.Seq_len, S*S])\n        return diffs\n\n    def build(self, input_shape):\n        # Positional Embedding, initialized with zeros\n        self.positional_embedding = tf.keras.layers.Embedding(self.Seq_len+1, emb_size, embeddings_initializer=INIT_ZEROS)\n        # Embedding layer for Landmarks\n        self.lips_embedding = LandmarkEmbedding(emb_size, 'Seq'+str(self.Seq_len)+'lips')\n        self.left_hand_embedding = LandmarkEmbedding(emb_size, 'Seq'+str(self.Seq_len)+'left_hand')\n        self.right_hand_embedding = LandmarkEmbedding(emb_size, 'Seq'+str(self.Seq_len)+'right_hand')\n        self.pose_embedding = LandmarkEmbedding(emb_size, 'Seq'+str(self.Seq_len)+'pose')\n        # Landmark Weights\n        self.landmark_weights = tf.Variable(tf.zeros([4], dtype=tf.float32), name=self.landmark_weights_name + str(self.Seq_len))\n        # self.lip_weights = tf.Variable(tf.random.normal((LIPS_UNITS, LIPS_UNITS), 0, 0.01, dtype=tf.float32), name='lip_weights')\n        # self.lhand_weights = tf.Variable(tf.random.normal((HANDS_UNITS, HANDS_UNITS), 0, 0.01, dtype=tf.float32), name='lhand_weights')\n        # self.rhand_weights = tf.Variable(tf.random.normal((HANDS_UNITS, HANDS_UNITS), 0, 0.01, dtype=tf.float32), name='rhand_weights')\n        # self.pose_weights = tf.Variable(tf.random.normal((POSE_UNITS, POSE_UNITS), 0, 0.01, dtype=tf.float32), name='pose_weights')\n        #self.landmark_weights = tf.Variable(tf.zeros([3], dtype=tf.float32), name='landmark_weights')\n        # Fully Connected Layers for combined landmarks\n        self.fc = tf.keras.Sequential([\n            tf.keras.layers.Dense(emb_size, name='Seq'+ str(self.Seq_len)+'fully_connected_1', use_bias=False, kernel_initializer=INIT_GLOROT_UNIFORM, activation=GELU),\n            tf.keras.layers.Dense(emb_size, name='Seq'+ str(self.Seq_len)+'fully_connected_2', use_bias=False, kernel_initializer=INIT_HE_UNIFORM),\n        ], name='Seq'+ str(self.Seq_len)+'fc')\n\n\n    def call(self, lips0, left_hand0, right_hand0, pose0, non_empty_frame_idxs, training=False):\n        # Lips\n        lips_embedding = self.lips_embedding(lips0)\n        # Left Hand\n        left_hand_embedding = self.left_hand_embedding(left_hand0)\n        # Right Hand\n        right_hand_embedding = self.right_hand_embedding(right_hand0)\n        # Pose\n        pose_embedding = self.pose_embedding(pose0)\n        # Merge Embeddings of all landmarks with mean pooling\n        x = tf.stack((lips_embedding, left_hand_embedding, right_hand_embedding, pose_embedding), axis=3)\n        #x = tf.stack((lips_embedding, left_hand_embedding, right_hand_embedding), axis=3)\n        # Merge Landmarks with trainable attention weights\n        x = x * tf.nn.softmax(self.landmark_weights)\n        x = tf.reduce_sum(x, axis=3)\n        # Fully Connected Layers\n        x = self.fc(x)\n        # Add Positional Embedding\n        normalised_non_empty_frame_idxs = tf.where(\n            tf.math.equal(non_empty_frame_idxs, -1.0),\n            self.Seq_len,\n            tf.cast(\n                non_empty_frame_idxs / tf.reduce_max(non_empty_frame_idxs, axis=1, keepdims=True) * self.Seq_len,\n                tf.int32,\n            ),\n        )\n        x = x + self.positional_embedding(normalised_non_empty_frame_idxs)\n        \n        return x","metadata":{"papermill":{"duration":0.035439,"end_time":"2023-04-30T13:11:27.237537","exception":false,"start_time":"2023-04-30T13:11:27.202098","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:19.271027Z","iopub.execute_input":"2023-05-01T22:14:19.271346Z","iopub.status.idle":"2023-05-01T22:14:19.288352Z","shell.execute_reply.started":"2023-05-01T22:14:19.271321Z","shell.execute_reply":"2023-05-01T22:14:19.287271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model(Seq_len1, \n              Seq_len2,\n              LIPS_MEAN1, \n              LIPS_STD1, \n              LEFT_HANDS_MEAN1, \n              LEFT_HANDS_STD1, \n              RIGHT_HANDS_MEAN1, \n              RIGHT_HANDS_STD1,\n              POSE_MEAN1,\n                POSE_STD1,\n              LIPS_MEAN2, \n              LIPS_STD2, \n              LEFT_HANDS_MEAN2, \n              LEFT_HANDS_STD2, \n              RIGHT_HANDS_MEAN2, \n              RIGHT_HANDS_STD2,\n              POSE_MEAN2,\n                POSE_STD2,\n             name):\n    \n    ######## Model1 (model32)  ##########\n    # Inputs\n    frames1 = tf.keras.layers.Input([Seq_len1, N_COLS, N_DIMS], dtype=tf.float32, name='frames1')\n    non_empty_frame_idxs1 = tf.keras.layers.Input([Seq_len1], dtype=tf.float32, name='non_empty_frame_idxs1')\n    # Padding Mask\n    mask1 = tf.cast(tf.math.not_equal(non_empty_frame_idxs1, -1), tf.float32)\n    mask1 = tf.expand_dims(mask1, axis=2)\n    \n    \"\"\"\n        left_hand: 468:489\n        pose: 489:522\n        right_hand: 522:543\n    \"\"\"\n    x1 = frames1\n    x1 = tf.slice(x1, [0,0,0,0], [-1, Seq_len1, N_COLS, 2])\n    # LIPS\n    lips1 = tf.slice(x1, [0,0,LIPS_START,0], [-1,Seq_len1, 40, 2])\n    lips1 = tf.where(\n        tf.math.equal(lips1, 0.0),\n            0.0,\n            (lips1 - LIPS_MEAN1) / LIPS_STD1,\n        )\n    lips1 = tf.reshape(lips1, [-1, Seq_len1, 40*2])\n    # LEFT HAND\n    left_hand1 = tf.slice(x1, [0,0,40,0], [-1, Seq_len1, 21, 2])\n    left_hand1 = tf.where(\n            tf.math.equal(left_hand1, 0.0),\n            0.0,\n            (left_hand1 - LEFT_HANDS_MEAN1) / LEFT_HANDS_STD1,\n        )\n    left_hand1 = tf.reshape(left_hand1, [-1, Seq_len1, 21*2])\n    # RIGHT HAND\n    right_hand1 = tf.slice(x1, [0,0,61,0], [-1, Seq_len1, 21, 2])\n    right_hand1 = tf.where(\n            tf.math.equal(right_hand1, 0.0),\n            0.0,\n            (right_hand1 - RIGHT_HANDS_MEAN1) / RIGHT_HANDS_STD1,\n        )\n    right_hand1 = tf.reshape(right_hand1, [-1, Seq_len1, 21*2])\n    # POSE\n    pose1 = tf.slice(x1, [0,0,82,0], [-1, Seq_len1, 10, 2])\n    pose1 = tf.where(\n            tf.math.equal(pose1, 0.0),\n            0.0,\n            (pose1 - POSE_MEAN1) / POSE_STD1,\n        )\n    pose1 = tf.reshape(pose1, [-1, Seq_len1, 10*2])\n    \n    x1 = lips1, left_hand1, right_hand1, pose1\n        \n    x1 = ISLREmbedding(Seq_len1, 'landmark_weights1')(lips1, left_hand1, right_hand1, pose1, non_empty_frame_idxs1) # pose data is used here\n    # x = Embedding()(lips, left_hand, right_hand, non_empty_frame_idxs)\n    \n    # Encoder Transformer Blocks\n    x1 = tf.keras.layers.Bidirectional(tf.keras.layers.GRU(emb_size, return_sequences=True), name='bidir_gru_1')(x1)\n\n    x1 = ISLRTransformer(NUM_BLOCKS, 'transformer1')(x1, mask1)\n    \n    # Pooling\n    x1 = tf.reduce_sum(x1 * mask1, axis=1) / tf.reduce_sum(mask1, axis=1)\n    # Classification Layer\n    x1 = tf.keras.layers.LayerNormalization(name='x1_layernorm')(x1)\n    # x1 = tf.keras.layers.Dropout(CLASSIFIER_DROPOUT_RATIO)(x1)\n    \n                                ##### Model2 ######\n    # Inputs\n    frames2 = tf.keras.layers.Input([Seq_len2, N_COLS, N_DIMS], dtype=tf.float32, name='frames2')\n    non_empty_frame_idxs2 = tf.keras.layers.Input([Seq_len2], dtype=tf.float32, name='non_empty_frame_idxs2')\n    # Padding Mask\n    mask2 = tf.cast(tf.math.not_equal(non_empty_frame_idxs2, -1), tf.float32)\n    mask2 = tf.expand_dims(mask2, axis=2)\n    \n    \"\"\"\n        left_hand: 468:489\n        pose: 489:522\n        right_hand: 522:543\n    \"\"\"\n    x2 = frames2\n    x2 = tf.slice(x2, [0,0,0,0], [-1, Seq_len2, N_COLS, 2])\n    # LIPS\n    lips2 = tf.slice(x2, [0,0,LIPS_START,0], [-1,Seq_len2, 40, 2])\n    lips2 = tf.where(\n        tf.math.equal(lips2, 0.0),\n            0.0,\n            (lips2 - LIPS_MEAN2) / LIPS_STD2,\n        )\n    lips2 = tf.reshape(lips2, [-1, Seq_len2, 40*2])\n    # LEFT HAND\n    left_hand2 = tf.slice(x2, [0,0,40,0], [-1, Seq_len2, 21, 2])\n    left_hand2 = tf.where(\n            tf.math.equal(left_hand2, 0.0),\n            0.0,\n            (left_hand2 - LEFT_HANDS_MEAN2) / LEFT_HANDS_STD2,\n        )\n    left_hand2 = tf.reshape(left_hand2, [-1, Seq_len2, 21*2])\n    # RIGHT HAND\n    right_hand2 = tf.slice(x2, [0,0,61,0], [-1, Seq_len2, 21, 2])\n    right_hand2 = tf.where(\n            tf.math.equal(right_hand2, 0.0),\n            0.0,\n            (right_hand2 - RIGHT_HANDS_MEAN2) / RIGHT_HANDS_STD2,\n        )\n    right_hand2 = tf.reshape(right_hand2, [-1, Seq_len2, 21*2])\n    # POSE\n    pose2 = tf.slice(x2, [0,0,82,0], [-1, Seq_len2, 10, 2])\n    pose2 = tf.where(\n            tf.math.equal(pose2, 0.0),\n            0.0,\n            (pose2 - POSE_MEAN2) / POSE_STD2,\n        )\n    pose2 = tf.reshape(pose2, [-1, Seq_len2, 10*2])\n    \n    x2 = lips2, left_hand2, right_hand2, pose2\n        \n    x2 = ISLREmbedding(Seq_len2, 'landmark_weights2')(lips2, left_hand2, right_hand2, pose2, non_empty_frame_idxs2) # pose data is used here\n    # x = Embedding()(lips, left_hand, right_hand, non_empty_frame_idxs)\n    \n    # Encoder Transformer Blocks\n    x2 = tf.keras.layers.Bidirectional(tf.keras.layers.GRU(emb_size, return_sequences=True), name='bidir_gru_2')(x2)\n\n    x2 = ISLRTransformer(NUM_BLOCKS, 'transformer2')(x2, mask2)\n    \n    # Pooling\n    x2 = tf.reduce_sum(x2 * mask2, axis=1) / tf.reduce_sum(mask2, axis=1)\n    # Classification Layer\n    x2 = tf.keras.layers.LayerNormalization(name='x2_layernorm')(x2)\n    # x2 = tf.keras.layers.Dropout(CLASSIFIER_DROPOUT_RATIO)(x2)\n    #x = tf.keras.layers.Dense(NUM_CLASSES, activation=None, kernel_initializer=INIT_GLOROT_UNIFORM)(x)\n    x = tf.keras.layers.concatenate([x1, x2], axis=-1)\n    \n    x = tf.keras.layers.LayerNormalization(name='combine_x_layernorm')(x)\n    x = tf.keras.layers.Dropout(CLASSIFIER_DROPOUT_RATIO)(x)\n    x = tf.keras.layers.Dense(NUM_CLASSES, \n                          kernel_initializer=INIT_GLOROT_UNIFORM, name='final_dense')(x)\n    x = tf.keras.layers.Softmax()(x)\n    outputs = x\n\n    #model = tf.keras.models.Model(inputs=[model32.input, model64.input], outputs=outputs)\n    \n    # Create Tensorflow Model\n    model = tf.keras.models.Model(inputs=[frames1, non_empty_frame_idxs1, frames2, non_empty_frame_idxs2], \n                                  outputs=outputs,\n                                 name = name)\n    \n    \n    return model","metadata":{"papermill":{"duration":0.046961,"end_time":"2023-04-30T13:11:27.298174","exception":false,"start_time":"2023-04-30T13:11:27.251213","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:14:19.290994Z","iopub.execute_input":"2023-05-01T22:14:19.291289Z","iopub.status.idle":"2023-05-01T22:14:19.320184Z","shell.execute_reply.started":"2023-05-01T22:14:19.291263Z","shell.execute_reply":"2023-05-01T22:14:19.319157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model2 = get_model(  32,\n                    64,\n                    LIPS_MEAN32, \n                    LIPS_STD32, \n                    LEFT_HANDS_MEAN32, \n                    LEFT_HANDS_STD32, \n                    RIGHT_HANDS_MEAN32, \n                    RIGHT_HANDS_STD32,\n                    POSE_MEAN32,\n                    POSE_STD32,\n                    LIPS_MEAN64, \n                    LIPS_STD64, \n                    LEFT_HANDS_MEAN64, \n                    LEFT_HANDS_STD64, \n                    RIGHT_HANDS_MEAN64, \n                    RIGHT_HANDS_STD64,\n                    POSE_MEAN64,\n                    POSE_STD64,\n                    \"GRU_trans2x_fold6\")","metadata":{"execution":{"iopub.status.busy":"2023-05-01T22:14:27.113862Z","iopub.execute_input":"2023-05-01T22:14:27.114405Z","iopub.status.idle":"2023-05-01T22:14:31.477528Z","shell.execute_reply.started":"2023-05-01T22:14:27.114359Z","shell.execute_reply":"2023-05-01T22:14:31.476488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model3 = get_model(  32,\n                    64,\n                    LIPS_MEAN32, \n                    LIPS_STD32, \n                    LEFT_HANDS_MEAN32, \n                    LEFT_HANDS_STD32, \n                    RIGHT_HANDS_MEAN32, \n                    RIGHT_HANDS_STD32,\n                    POSE_MEAN32,\n                    POSE_STD32,\n                    LIPS_MEAN64, \n                    LIPS_STD64, \n                    LEFT_HANDS_MEAN64, \n                    LEFT_HANDS_STD64, \n                    RIGHT_HANDS_MEAN64, \n                    RIGHT_HANDS_STD64,\n                    POSE_MEAN64,\n                    POSE_STD64,\n                    \"GRU_trans2x_fold7\")","metadata":{"execution":{"iopub.status.busy":"2023-05-01T22:14:31.479109Z","iopub.execute_input":"2023-05-01T22:14:31.479493Z","iopub.status.idle":"2023-05-01T22:14:36.022699Z","shell.execute_reply.started":"2023-05-01T22:14:31.479452Z","shell.execute_reply":"2023-05-01T22:14:36.021683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"emb_size = 80\nUNITS = 2*emb_size\nmodel1 = get_model(  32,\n                    64,\n                    LIPS_MEAN32, \n                    LIPS_STD32, \n                    LEFT_HANDS_MEAN32, \n                    LEFT_HANDS_STD32, \n                    RIGHT_HANDS_MEAN32, \n                    RIGHT_HANDS_STD32,\n                    POSE_MEAN32,\n                    POSE_STD32,\n                    LIPS_MEAN64, \n                    LIPS_STD64, \n                    LEFT_HANDS_MEAN64, \n                    LEFT_HANDS_STD64, \n                    RIGHT_HANDS_MEAN64, \n                    RIGHT_HANDS_STD64,\n                    POSE_MEAN64,\n                    POSE_STD64,\n                    \"GRU_trans2x_fold5\")","metadata":{"papermill":{"duration":8.164663,"end_time":"2023-04-30T13:11:35.476508","exception":false,"start_time":"2023-04-30T13:11:27.311845","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:22:45.512698Z","iopub.execute_input":"2023-05-01T22:22:45.513902Z","iopub.status.idle":"2023-05-01T22:22:50.285342Z","shell.execute_reply.started":"2023-05-01T22:22:45.513842Z","shell.execute_reply":"2023-05-01T22:22:50.284290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.backend.clear_session()\n\n# Simple Categorical Crossentropy Loss\nloss = tf.keras.losses.SparseCategoricalCrossentropy()\n\n# Adam Optimizer with weight decay\n# optimizer = tfa.optimizers.AdamW(learning_rate=1e-3, weight_decay=1e-5, clipnorm=1.0)\noptimizer = tfa.optimizers.AdamW(learning_rate=1e-3, weight_decay=1e-5, clipnorm=0.6)\n\n# TopK Metrics\nmetrics = [\n    tf.keras.metrics.SparseCategoricalAccuracy(name='acc'),\n    tf.keras.metrics.SparseTopKCategoricalAccuracy(k=5, name='top_5_acc'),\n    tf.keras.metrics.SparseTopKCategoricalAccuracy(k=10, name='top_10_acc'),\n]\n\nmodel1.compile(loss=loss, optimizer=optimizer, metrics=metrics)\nmodel2.compile(loss=loss, optimizer=optimizer, metrics=metrics)\nmodel3.compile(loss=loss, optimizer=optimizer, metrics=metrics)","metadata":{"papermill":{"duration":0.097677,"end_time":"2023-04-30T13:11:35.624713","exception":false,"start_time":"2023-04-30T13:11:35.527036","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:22:50.287240Z","iopub.execute_input":"2023-05-01T22:22:50.287588Z","iopub.status.idle":"2023-05-01T22:22:50.494890Z","shell.execute_reply.started":"2023-05-01T22:22:50.287550Z","shell.execute_reply":"2023-05-01T22:22:50.493861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Plot model summary\n#model.summary(expand_nested=True)","metadata":{"papermill":{"duration":0.024302,"end_time":"2023-04-30T13:11:35.663643","exception":false,"start_time":"2023-04-30T13:11:35.639341","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:22:50.496640Z","iopub.execute_input":"2023-05-01T22:22:50.497045Z","iopub.status.idle":"2023-05-01T22:22:50.504657Z","shell.execute_reply.started":"2023-05-01T22:22:50.497004Z","shell.execute_reply":"2023-05-01T22:22:50.503594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#tf.keras.utils.plot_model(model, show_shapes=True, show_dtype=True, show_layer_names=True, expand_nested=True, show_layer_activations=True)","metadata":{"papermill":{"duration":0.023642,"end_time":"2023-04-30T13:11:35.701184","exception":false,"start_time":"2023-04-30T13:11:35.677542","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:22:50.508916Z","iopub.execute_input":"2023-05-01T22:22:50.509276Z","iopub.status.idle":"2023-05-01T22:22:50.516915Z","shell.execute_reply.started":"2023-05-01T22:22:50.509243Z","shell.execute_reply":"2023-05-01T22:22:50.515857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model1.load_weights('/kaggle/input/transformer-weights/BiGRU_transformer2x_emb80_EmbDrop005_mlpdrop005_clsdrop01_seqlen32_seqlen64_fold5A_weights.h5')","metadata":{"papermill":{"duration":1.260188,"end_time":"2023-04-30T13:11:36.975569","exception":false,"start_time":"2023-04-30T13:11:35.715381","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:23:14.029465Z","iopub.execute_input":"2023-05-01T22:23:14.030487Z","iopub.status.idle":"2023-05-01T22:23:14.453526Z","shell.execute_reply.started":"2023-05-01T22:23:14.030445Z","shell.execute_reply":"2023-05-01T22:23:14.452456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model2.load_weights('/kaggle/input/transformer-weights/BiGRU_transformer2x_emb96_EmbDrop005_mlpdrop005_clsdrop01_seqlen32_seqlen64_fold6A_weights.h5')","metadata":{"execution":{"iopub.status.busy":"2023-05-01T22:23:16.726712Z","iopub.execute_input":"2023-05-01T22:23:16.727636Z","iopub.status.idle":"2023-05-01T22:23:17.160692Z","shell.execute_reply.started":"2023-05-01T22:23:16.727584Z","shell.execute_reply":"2023-05-01T22:23:17.159671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model3.load_weights('/kaggle/input/transformer-weights/BiGRU_transformer2x_emb96_EmbDrop005_mlpdrop005_clsdrop01_seqlen32_seqlen64_fold7A_weights.h5')","metadata":{"execution":{"iopub.status.busy":"2023-05-01T22:23:19.532405Z","iopub.execute_input":"2023-05-01T22:23:19.533039Z","iopub.status.idle":"2023-05-01T22:23:19.911331Z","shell.execute_reply.started":"2023-05-01T22:23:19.532996Z","shell.execute_reply":"2023-05-01T22:23:19.910297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission\n\nSubmission code loosley based on [this notebook](https://www.kaggle.com/code/dschettler8845/gislr-learn-eda-baseline#baseline) by [Darien Schettler\n](https://www.kaggle.com/dschettler8845)","metadata":{"papermill":{"duration":0.013716,"end_time":"2023-04-30T13:11:37.003535","exception":false,"start_time":"2023-04-30T13:11:36.989819","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# TFLite model for submission\nclass TFLiteModel(tf.Module):\n    def __init__(self, model1, model2, model3):\n        super(TFLiteModel, self).__init__()\n\n        # Load the feature generation and main models\n        self.preprocess_layer32 = PreprocessLayer(32)\n        self.preprocess_layer64 = PreprocessLayer(64)\n        self.model1 = model1\n        self.model2 = model2\n        self.model3 = model3\n        \n    @tf.function(input_signature=[tf.TensorSpec(shape=[None, N_ROWS, N_DIMS], dtype=tf.float32, name='inputs')])\n    def __call__(self, inputs):\n        # Preprocess Data\n        x32, non_empty_frame_idxs32 = self.preprocess_layer32(inputs)\n        #print(\"x32 shape: \", x32.shape)\n        x64, non_empty_frame_idxs64 = self.preprocess_layer64(inputs)\n        # print(\"x64 shape: \", x64.shape)\n        # Add Batch Dimension\n        x32 = tf.expand_dims(x32, axis=0)\n        x64 = tf.expand_dims(x64, axis=0)\n        non_empty_frame_idxs32 = tf.expand_dims(non_empty_frame_idxs32, axis=0)\n        non_empty_frame_idxs64 = tf.expand_dims(non_empty_frame_idxs64, axis=0)\n        # Make Prediction\n        outputs1 = self.model1({'frames1': x32, \n                              'non_empty_frame_idxs1': non_empty_frame_idxs32, \n                              'frames2': x64, \n                              'non_empty_frame_idxs2': non_empty_frame_idxs64 })\n        \n        outputs2 = self.model2({'frames1': x32, \n                              'non_empty_frame_idxs1': non_empty_frame_idxs32, \n                              'frames2': x64, \n                              'non_empty_frame_idxs2': non_empty_frame_idxs64 })\n        outputs3 = self.model3({'frames1': x32, \n                              'non_empty_frame_idxs1': non_empty_frame_idxs32, \n                              'frames2': x64, \n                              'non_empty_frame_idxs2': non_empty_frame_idxs64 })\n#         # Squeeze Output 1x250 -> 250\n        outputs = tf.concat([outputs1, outputs2, outputs3], axis=0)\n        outputs = tf.reduce_mean(outputs, axis=0)\n#         outputs = tf.squeeze(outputs, axis=0)\n        # Return a dictionary with the output tensor\n        return {'outputs': outputs}\n\n# Define TF Lite Model\ntflite_keras_model = TFLiteModel(model1, model2, model3)\n\n# Sanity Check\ndata_idx = 5\ndemo_raw_data = load_relevant_data_subset(train['file_path'].values[data_idx])\nprint(f'demo_raw_data shape: {demo_raw_data.shape}, dtype: {demo_raw_data.dtype}')\ndemo_output = tflite_keras_model(demo_raw_data)[\"outputs\"]\nprint(f'demo_output shape: {demo_output.shape}, dtype: {demo_output.dtype}')\ndemo_prediction = demo_output.numpy().argmax()\nprint(f'demo_prediction: {demo_prediction}, correct: {train.iloc[data_idx][\"sign_ord\"]}')","metadata":{"papermill":{"duration":9.144442,"end_time":"2023-04-30T13:11:46.161742","exception":false,"start_time":"2023-04-30T13:11:37.017300","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:23:24.810577Z","iopub.execute_input":"2023-05-01T22:23:24.811805Z","iopub.status.idle":"2023-05-01T22:23:36.787910Z","shell.execute_reply.started":"2023-05-01T22:23:24.811757Z","shell.execute_reply":"2023-05-01T22:23:36.786894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create Model Converter\nkeras_model_converter = tf.lite.TFLiteConverter.from_keras_model(tflite_keras_model)\n# Convert Model\n# keras_model_converter.optimizations = [tf.lite.Optimize.DEFAULT]\n# keras_model_converter.target_spec.supported_types = [tf.float16]\ntflite_model = keras_model_converter.convert()\n# Write Model\nwith open('model.tflite', 'wb') as f:\n    f.write(tflite_model)\n    \n# Zip Model\n!zip submission.zip /kaggle/working/model.tflite","metadata":{"papermill":{"duration":132.762424,"end_time":"2023-04-30T13:13:58.938453","exception":false,"start_time":"2023-04-30T13:11:46.176029","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:23:36.790006Z","iopub.execute_input":"2023-05-01T22:23:36.790486Z","iopub.status.idle":"2023-05-01T22:29:42.304861Z","shell.execute_reply.started":"2023-05-01T22:23:36.790445Z","shell.execute_reply":"2023-05-01T22:29:42.303444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"demo_raw_data.shape","metadata":{"papermill":{"duration":0.026521,"end_time":"2023-04-30T13:13:58.979550","exception":false,"start_time":"2023-04-30T13:13:58.953029","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:29:42.307997Z","iopub.execute_input":"2023-05-01T22:29:42.308344Z","iopub.status.idle":"2023-05-01T22:29:42.320174Z","shell.execute_reply.started":"2023-05-01T22:29:42.308310Z","shell.execute_reply":"2023-05-01T22:29:42.319135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Verify TFLite model can be loaded and used for prediction\n!pip install tflite-runtime\nimport tflite_runtime.interpreter as tflite\n\ninterpreter = tflite.Interpreter(\"model.tflite\")\nfound_signatures = list(interpreter.get_signature_list().keys())\nprediction_fn = interpreter.get_signature_runner(\"serving_default\")\n\n# x32, mask32 = preprocess_layer32(demo_raw_data)\n\n# x64, mask64 = preprocess_layer32(demo_raw_data)\nimport time\nst = time.time()\nfor i in range(100):\n    output = prediction_fn(inputs=demo_raw_data)\nprint(\"Inf. time for each sample\", (time.time()-st)/100)\n\n#output = prediction_fn(inputs=demo_raw_data)\n\nsign = output['outputs'].argmax()\n\nprint(\"max probility:\", output['outputs'].max())\n\nprint(\"PRED : \", ORD2SIGN.get(sign), f'[{sign}]')\nprint(\"TRUE : \", train.sign.values[data_idx], f'[{train.sign_ord.values[data_idx]}]')","metadata":{"papermill":{"duration":12.047828,"end_time":"2023-04-30T13:14:11.041207","exception":false,"start_time":"2023-04-30T13:13:58.993379","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-05-01T22:29:42.324007Z","iopub.execute_input":"2023-05-01T22:29:42.324401Z","iopub.status.idle":"2023-05-01T22:29:57.026552Z","shell.execute_reply.started":"2023-05-01T22:29:42.324372Z","shell.execute_reply":"2023-05-01T22:29:57.025160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.014392,"end_time":"2023-04-30T13:14:11.070698","exception":false,"start_time":"2023-04-30T13:14:11.056306","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.014325,"end_time":"2023-04-30T13:14:11.099720","exception":false,"start_time":"2023-04-30T13:14:11.085395","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}