{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":46105,"databundleVersionId":5087314,"sourceType":"competition"},{"sourceId":5315518,"sourceType":"datasetVersion","datasetId":3036481}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import\n","metadata":{}},{"cell_type":"code","source":"# Importing necessary libraries\n\nimport numpy as np  # NumPy is a library for numerical operations in Python\nimport pandas as pd  # Pandas is a data manipulation library\nimport tensorflow as tf  # TensorFlow is an open-source machine learning library\nimport tensorflow_addons as tfa  # TensorFlow Addons provides additional functionality for TensorFlow\nimport matplotlib.pyplot as plt  # Matplotlib is a plotting library\nimport matplotlib as mpl  # Matplotlib is often imported as mpl for setting global parameters\nimport seaborn as sn  # Seaborn is a statistical data visualization library based on Matplotlib\n\nfrom tqdm.notebook import tqdm  # tqdm is a library for displaying progress bars in loops\nfrom sklearn.model_selection import train_test_split, GroupShuffleSplit  # Scikit-learn is a machine learning library\n\nimport glob  # Glob is a module for working with file paths using wildcards\nimport sys  # sys provides access to some variables used or maintained by the Python interpreter\nimport os  # os provides a way of interacting with the operating system\nimport math  # math provides mathematical functions\nimport gc  # gc is the garbage collector module for managing memory\nimport sklearn  # Scikit-learn is a machine learning library\nimport scipy  # SciPy is a library for scientific and technical computing\n\n# Printing version information\nprint(f'Tensorflow V{tf.__version__}')  # Print TensorFlow version\nprint(f'Keras V{tf.keras.__version__}')  # Print Keras version (part of TensorFlow)\nprint(f'Python V{sys.version}')  # Print Python version\n","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:59:56.737583Z","iopub.execute_input":"2023-11-22T07:59:56.738344Z","iopub.status.idle":"2023-11-22T08:00:05.157387Z","shell.execute_reply.started":"2023-11-22T07:59:56.738305Z","shell.execute_reply":"2023-11-22T08:00:05.156289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot Configuratuion\nThis code block modifies the default settings of Matplotlib for various plot elements. It ensures that these settings are applied globally for all plots created in the same session. The parameters being set include the font sizes for x-axis and y-axis tick labels, axes labels, and axes titles. Adjusting these parameters can help in customizing the appearance of your plots according to your preferences.\n\n\n\n\n\n","metadata":{}},{"cell_type":"code","source":"# MatplotLib Global Settings\n\n# Resetting Matplotlib default parameters to avoid unexpected behavior\nmpl.rcParams.update(mpl.rcParamsDefault)\n\n# Setting the font size for x-axis tick labels to 16\nmpl.rcParams['xtick.labelsize'] = 16\n\n# Setting the font size for y-axis tick labels to 16\nmpl.rcParams['ytick.labelsize'] = 16\n\n# Setting the font size for the axes labels to 18\nmpl.rcParams['axes.labelsize'] = 18\n\n# Setting the font size for the axes titles to 24\nmpl.rcParams['axes.titlesize'] = 24","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:00:05.159254Z","iopub.execute_input":"2023-11-22T08:00:05.159938Z","iopub.status.idle":"2023-11-22T08:00:05.168872Z","shell.execute_reply.started":"2023-11-22T08:00:05.159902Z","shell.execute_reply":"2023-11-22T08:00:05.167815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configuration\n","metadata":{}},{"cell_type":"code","source":"# If True, processing data from scratch; If False, loads preprocessed data\nPREPROCESS_DATA = False\n\n# If True, training the model; If False, skips the training step\nTRAIN_MODEL = True\n\n# If True, use 10% of participants as the validation set; If False, use all data for training\nUSE_VAL = False\n\n# Number of rows in the data\nN_ROWS = 543\n\n# Number of dimensions in the data\nN_DIMS = 3\n\n# Names of the dimensions\nDIM_NAMES = ['x', 'y', 'z']\n\n# Seed for random number generation to ensure reproducibility\nSEED = 42\n\n# Number of classes in the classification task\nNUM_CLASSES = 250\n\n# Flag indicating whether the environment is interactive (e.g., Kaggle kernel) or not\nIS_INTERACTIVE = os.environ['KAGGLE_KERNEL_RUN_TYPE'] == 'Interactive'\n\n# Verbosity level for printing output during training\nVERBOSE = 1 if IS_INTERACTIVE else 2\n\n# Input size for the model\nINPUT_SIZE = 64\n\n# Number of signals in each batch\nBATCH_ALL_SIGNS_N = 4\n\n# Batch size for training\nBATCH_SIZE = 256\n\n# Number of epochs for training\nN_EPOCHS = 100\n\n# Maximum learning rate during training\nLR_MAX = 1e-3\n\n# Number of warm-up epochs with a lower learning rate\nN_WARMUP_EPOCHS = 0\n\n# Weight decay ratio for regularization\nWD_RATIO = 0.05\n\n# Mask value used in data processing\nMASK_VAL = 4237\n","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:00:05.170221Z","iopub.execute_input":"2023-11-22T08:00:05.170541Z","iopub.status.idle":"2023-11-22T08:00:05.18775Z","shell.execute_reply.started":"2023-11-22T08:00:05.170511Z","shell.execute_reply":"2023-11-22T08:00:05.186851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utils\nThis is a function print_shape_dtype that takes two lists as input (l and names). The function iterates through the elements of both lists in parallel, printing the shape and data type of each element","metadata":{}},{"cell_type":"code","source":"# Prints Shape and Dtype For List Of Variables\ndef print_shape_dtype(l, names):\n    # Iterate through elements of both lists in parallel\n    for e, n in zip(l, names):\n        # Print the name, shape, and dtype of each variable\n        print(f'{n} shape: {e.shape}, dtype: {e.dtype}')","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:00:05.190239Z","iopub.execute_input":"2023-11-22T08:00:05.190614Z","iopub.status.idle":"2023-11-22T08:00:05.197253Z","shell.execute_reply.started":"2023-11-22T08:00:05.190583Z","shell.execute_reply":"2023-11-22T08:00:05.196253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train\nThis code block reads training data from a CSV file. The behavior depends on the values of the \"IS_INTERACTIVE\" and \"PREPROCESS_DATA\" variables.\n\n* If the environment is interactive (\"IS_INTERACTIVE\" is True) or if data preprocessing is not required (\"PREPROCESS_DATA\" is False), it reads a random sample of 5000 rows from the training data CSV file.\n* If the environment is not interactive and data preprocessing is required, it reads the entire training data from the CSV file.","metadata":{}},{"cell_type":"code","source":"# Read Training Data\nif IS_INTERACTIVE or not PREPROCESS_DATA:\n    # Read a random sample of 5000 rows if interactive or if preprocessing is not required\n    train = pd.read_csv('/kaggle/input/asl-signs/train.csv').sample(int(5e3), random_state=SEED)\nelse:\n    # Read the entire training data if not interactive and preprocessing is required\n    train = pd.read_csv('/kaggle/input/asl-signs/train.csv')\n\n# Get the number of samples in the training data\nN_SAMPLES = len(train)\n\n# Print the number of samples\nprint(f'N_SAMPLES: {N_SAMPLES}')\n","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:00:05.198514Z","iopub.execute_input":"2023-11-22T08:00:05.198894Z","iopub.status.idle":"2023-11-22T08:00:05.423496Z","shell.execute_reply.started":"2023-11-22T08:00:05.198865Z","shell.execute_reply":"2023-11-22T08:00:05.422385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Add File Path\nThis code defines a function get_file_path that takes a relative path (path) as an argument and returns the complete file path by appending it to the base directory (/kaggle/input/asl-signs/). The function is then applied to the 'path' column of the 'train' DataFrame, and the result is stored in a new column 'file_path'.","metadata":{}},{"cell_type":"code","source":"# Get complete file path to file\ndef get_file_path(path):\n    # Concatenate the base directory with the given relative path\n    return f'/kaggle/input/asl-signs/{path}'\n\n# Apply the get_file_path function to the 'path' column of the 'train' DataFrame\ntrain['file_path'] = train['path'].apply(get_file_path)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:00:05.424696Z","iopub.execute_input":"2023-11-22T08:00:05.425012Z","iopub.status.idle":"2023-11-22T08:00:05.437198Z","shell.execute_reply.started":"2023-11-22T08:00:05.424981Z","shell.execute_reply":"2023-11-22T08:00:05.436107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ordinally Encode Sign\nThis code block adds a new column 'sign_ord' to the 'train' DataFrame, which contains ordinal encodings for the 'sign' column. It also creates two dictionaries, 'SIGN2ORD' and 'ORD2SIGN', for translating between sign names and their ordinal encodings.","metadata":{}},{"cell_type":"code","source":"# Add ordinally Encoded Sign (assign number to each sign name)\ntrain['sign_ord'] = train['sign'].astype('category').cat.codes\n\n# Dictionaries to translate sign <-> ordinal encoded sign\nSIGN2ORD = train[['sign', 'sign_ord']].set_index('sign').squeeze().to_dict()\nORD2SIGN = train[['sign_ord', 'sign']].set_index('sign_ord').squeeze().to_dict()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:00:05.438686Z","iopub.execute_input":"2023-11-22T08:00:05.439477Z","iopub.status.idle":"2023-11-22T08:00:05.464746Z","shell.execute_reply.started":"2023-11-22T08:00:05.439426Z","shell.execute_reply":"2023-11-22T08:00:05.463929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"1. **train['sign_ord'] = train['sign'].astype('category').cat.codes**: This line creates a new column 'sign_ord' in the 'train' DataFrame by assigning numerical codes to each unique value in the 'sign' column. The astype('category').cat.codes combination is a common way to perform ordinal encoding in pandas.\n\n1. **SIGN2ORD = train[['sign', 'sign_ord']].set_index('sign').squeeze().to_dict()**: This line creates a dictionary 'SIGN2ORD' where the sign names are the keys, and the corresponding ordinal encodings are the values. It uses the pandas DataFrame methods to set the 'sign' column as the index, squeezes the result to a Series, and converts it to a dictionary.\n\n1. **ORD2SIGN = train[['sign_ord', 'sign']].set_index('sign_ord').squeeze().to_dict()**: Similarly, this line creates a dictionary 'ORD2SIGN' where the ordinal encodings are the keys, and the corresponding sign names are the values.\n\nThese dictionaries can be useful for quickly translating between sign names and their corresponding numerical codes and vice versa.","metadata":{}},{"cell_type":"code","source":"# Display the first 30 rows of the 'train' DataFrame\ndisplay(train.head(30))\n\n# Display information about the 'train' DataFrame\ndisplay(train.info())","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:00:05.465963Z","iopub.execute_input":"2023-11-22T08:00:05.466251Z","iopub.status.idle":"2023-11-22T08:00:05.501348Z","shell.execute_reply.started":"2023-11-22T08:00:05.466223Z","shell.execute_reply":"2023-11-22T08:00:05.500396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* The DataFrame has 5000 entries (rows).\n* The index is of type Int64Index.\n* There are six columns in total: 'path', 'participant_id', 'sequence_id', 'sign', 'file_path', and 'sign_ord'.\n* Each column has 5000 non-null entries, indicating that there are no missing values.\n* The data types of the columns are as follows:\n    * 'path' and 'sign' are of type object (likely strings).\n    * 'participant_id' and 'sequence_id' are of type int64.\n    * 'file_path' is of type object (likely a string).\n    * 'sign_ord' is of type int16.\n* The total memory usage of the DataFrame is approximately 244.1 KB.","metadata":{}},{"cell_type":"markdown","source":"# Video Statistics","metadata":{}},{"cell_type":"code","source":"# Set the number of iterations based on conditions\nN = int(1e3) if (IS_INTERACTIVE or not PREPROCESS_DATA) else int(10e3)\n\n# Create arrays to store analysis results\nN_UNIQUE_FRAMES = np.zeros(N, dtype=np.uint16)\nN_MISSING_FRAMES = np.zeros(N, dtype=np.uint16)\nMAX_FRAME = np.zeros(N, dtype=np.uint16)\n\n# Define percentiles for analysis\nPERCENTILES = [0.01, 0.05, 0.25, 0.50, 0.75, 0.95, 0.99, 0.999]\n\n# Iterate over a sampled subset of file paths from the 'train' DataFrame\nfor idx, file_path in enumerate(tqdm(train['file_path'].sample(N, random_state=SEED))):\n    # Read data from a Parquet file into a DataFrame\n    df = pd.read_parquet(file_path)\n\n    # Analyze and store information in the arrays\n    N_UNIQUE_FRAMES[idx] = df['frame'].nunique()\n    N_MISSING_FRAMES[idx] = (df['frame'].max() - df['frame'].min()) - df['frame'].nunique() + 1\n    MAX_FRAME[idx] = df['frame'].max()\n","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:00:05.502702Z","iopub.execute_input":"2023-11-22T08:00:05.503153Z","iopub.status.idle":"2023-11-22T08:00:27.686151Z","shell.execute_reply.started":"2023-11-22T08:00:05.503114Z","shell.execute_reply":"2023-11-22T08:00:27.685023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The code extracts and analyzes information related to the 'frame' column from each DataFrame and stores the results in the corresponding arrays. This code snippet is performing some statistical analysis on the 'frame' column of the data stored in Parquet files and collecting various statistics for each iteration","metadata":{}},{"cell_type":"code","source":"# Display descriptive statistics for the number of unique frames\ndisplay(pd.Series(N_UNIQUE_FRAMES).describe(percentiles=PERCENTILES).to_frame('N_UNIQUE_FRAMES'))\n\n# Create a histogram for the number of unique frames\nplt.figure(figsize=(15,8))\nplt.title('Number of Unique Frames', size=24)\n\n# Plot the histogram\npd.Series(N_UNIQUE_FRAMES).plot(kind='hist', bins=128)\n\n# Adjust plot settings\nplt.grid()\nxlim = math.ceil(plt.xlim()[1])\nplt.xlim(0, xlim)\nplt.xticks(np.arange(0, xlim+25, 25))\n\n# Show the plot\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:00:27.691996Z","iopub.execute_input":"2023-11-22T08:00:27.692443Z","iopub.status.idle":"2023-11-22T08:00:28.242142Z","shell.execute_reply.started":"2023-11-22T08:00:27.69241Z","shell.execute_reply":"2023-11-22T08:00:28.241142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In the histogram plot, the x-axis represents the range of the number of unique frames in each video, and the y-axis represents the frequency (or count) of videos falling within each range. Here's a breakdown:\n\n* X-axis: Represents the number of unique frames in each video. The range is divided into bins, and each bin represents a specific range of unique frame counts. In this case, there are 128 bins (bins=128) used for the histogram.\n\n* Y-axis: Represents the frequency or count of videos that fall within each bin on the x-axis. For example, if there are 10 videos with 50 unique frames, there will be a bar at the corresponding bin on the x-axis with a height of 10.\n\nThe histogram provides a visual representation of the distribution of unique frame counts in the dataset.","metadata":{}},{"cell_type":"code","source":"# Display descriptive statistics for the number of missing frames\ndisplay(pd.Series(N_MISSING_FRAMES).describe(percentiles=PERCENTILES).to_frame('N_MISSING_FRAMES'))\n\n# Create a histogram for the number of missing frames\nplt.figure(figsize=(15,8))\nplt.title('Number of Missing Frames', size=24)\n\n# Plot the histogram\npd.Series(N_MISSING_FRAMES).plot(kind='hist', bins=128)\n\n# Adjust plot settings\nplt.grid()\nplt.xlim(0, math.ceil(plt.xlim()[1]))\n\n# Show the plot\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:00:28.243314Z","iopub.execute_input":"2023-11-22T08:00:28.243615Z","iopub.status.idle":"2023-11-22T08:00:28.739977Z","shell.execute_reply.started":"2023-11-22T08:00:28.243586Z","shell.execute_reply":"2023-11-22T08:00:28.738944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Display descriptive statistics for the maximum frame number\ndisplay(pd.Series(MAX_FRAME).describe(percentiles=PERCENTILES).to_frame('MAX_FRAME'))\n\n# Create a histogram for the maximum frame number\nplt.figure(figsize=(15,8))\nplt.title('Maximum Frames Index', size=24)\n\n# Plot the histogram\npd.Series(MAX_FRAME).plot(kind='hist', bins=128)\n\n# Adjust plot settings\nplt.grid()\nplt.xlim(0, math.ceil(plt.xlim()[1]))\n\n# Show the plot\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:00:28.741294Z","iopub.execute_input":"2023-11-22T08:00:28.741623Z","iopub.status.idle":"2023-11-22T08:00:29.202062Z","shell.execute_reply.started":"2023-11-22T08:00:28.741591Z","shell.execute_reply":"2023-11-22T08:00:29.200974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Landmark Indices","metadata":{}},{"cell_type":"code","source":"# List of types to use\nUSE_TYPES = ['left_hand', 'pose', 'right_hand']\n\n# Start index\nSTART_IDX = 468\n\n# Landmark indices for lips\nLIPS_IDXS0 = np.array([\n        61, 185, 40, 39, 37, 0, 267, 269, 270, 409,\n        291, 146, 91, 181, 84, 17, 314, 405, 321, 375,\n        78, 191, 80, 81, 82, 13, 312, 311, 310, 415,\n        95, 88, 178, 87, 14, 317, 402, 318, 324, 308,\n    ])\n\n# Landmark indices in original data\nLEFT_HAND_IDXS0 = np.arange(468,489)\nRIGHT_HAND_IDXS0 = np.arange(522,543)\nLEFT_POSE_IDXS0 = np.array([502, 504, 506, 508, 510])\nRIGHT_POSE_IDXS0 = np.array([503, 505, 507, 509, 511])\nLANDMARK_IDXS_LEFT_DOMINANT0 = np.concatenate((LIPS_IDXS0, LEFT_HAND_IDXS0, LEFT_POSE_IDXS0))\nLANDMARK_IDXS_RIGHT_DOMINANT0 = np.concatenate((LIPS_IDXS0, RIGHT_HAND_IDXS0, RIGHT_POSE_IDXS0))\nHAND_IDXS0 = np.concatenate((LEFT_HAND_IDXS0, RIGHT_HAND_IDXS0), axis=0)\nN_COLS = LANDMARK_IDXS_LEFT_DOMINANT0.size\n\n# Landmark indices in processed data\nLIPS_IDXS = np.argwhere(np.isin(LANDMARK_IDXS_LEFT_DOMINANT0, LIPS_IDXS0)).squeeze()\nLEFT_HAND_IDXS = np.argwhere(np.isin(LANDMARK_IDXS_LEFT_DOMINANT0, LEFT_HAND_IDXS0)).squeeze()\nRIGHT_HAND_IDXS = np.argwhere(np.isin(LANDMARK_IDXS_LEFT_DOMINANT0, RIGHT_HAND_IDXS0)).squeeze()\nHAND_IDXS = np.argwhere(np.isin(LANDMARK_IDXS_LEFT_DOMINANT0, HAND_IDXS0)).squeeze()\nPOSE_IDXS = np.argwhere(np.isin(LANDMARK_IDXS_LEFT_DOMINANT0, LEFT_POSE_IDXS0)).squeeze()\n\nprint(f'# HAND_IDXS: {len(HAND_IDXS)}, N_COLS: {N_COLS}')","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:00:29.203356Z","iopub.execute_input":"2023-11-22T08:00:29.20368Z","iopub.status.idle":"2023-11-22T08:00:29.21894Z","shell.execute_reply.started":"2023-11-22T08:00:29.203649Z","shell.execute_reply":"2023-11-22T08:00:29.217969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Starting indices for different types of landmarks\nLIPS_START = 0\nLEFT_HAND_START = LIPS_IDXS.size\nRIGHT_HAND_START = LEFT_HAND_START + LEFT_HAND_IDXS.size\nPOSE_START = RIGHT_HAND_START + RIGHT_HAND_IDXS.size\n\n# Print the starting indices\nprint(f'LIPS_START: {LIPS_START}, LEFT_HAND_START: {LEFT_HAND_START}, RIGHT_HAND_START: {RIGHT_HAND_START}, POSE_START: {POSE_START}')\n","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:00:29.220243Z","iopub.execute_input":"2023-11-22T08:00:29.220597Z","iopub.status.idle":"2023-11-22T08:00:29.228719Z","shell.execute_reply.started":"2023-11-22T08:00:29.220567Z","shell.execute_reply":"2023-11-22T08:00:29.227606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n\n- `LIPS_START`: The starting index for lips landmarks is set to 0.\n- `LEFT_HAND_START`: The starting index for left hand landmarks is set to the size of the `LIPS_IDXS` array.\n- `RIGHT_HAND_START`: The starting index for right hand landmarks is set to the sum of the sizes of the `LEFT_HAND_IDXS` and `RIGHT_HAND_IDXS` arrays.\n- `POSE_START`: The starting index for pose landmarks is set to the sum of the sizes of the `RIGHT_HAND_IDXS` and `POSE_IDXS` arrays.\n\nThe printed output shows the calculated starting indices for each type of landmark. These starting indices are useful for accessing specific types of landmarks within the processed arrays.","metadata":{}},{"cell_type":"markdown","source":"# Process Data Tensorflow","metadata":{}},{"cell_type":"code","source":"# Source: https://www.kaggle.com/competitions/asl-signs/overview/evaluation\nROWS_PER_FRAME = 543  # number of landmarks per frame\n\ndef load_relevant_data_subset(pq_path):\n    # Specify the columns to load from the Parquet file\n    data_columns = ['x', 'y', 'z']\n    \n    # Read relevant data from the Parquet file\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    \n    # Calculate the number of frames in the data\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    \n    # Reshape the data into a 3D array representing frames, landmarks, and coordinates\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    \n    # Convert the data to float32\n    return data.astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:00:29.230287Z","iopub.execute_input":"2023-11-22T08:00:29.230909Z","iopub.status.idle":"2023-11-22T08:00:29.240635Z","shell.execute_reply.started":"2023-11-22T08:00:29.230866Z","shell.execute_reply":"2023-11-22T08:00:29.239724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- `ROWS_PER_FRAME`: The number of landmarks per frame is set to 543.\n- `load_relevant_data_subset(pq_path)`: This function takes a Parquet file path (`pq_path`) as input.\n  - `data_columns`: Specifies the columns ('x', 'y', 'z') to load from the Parquet file.\n  - `data = pd.read_parquet(pq_path, columns=data_columns)`: Reads the relevant data from the Parquet file using Pandas.\n  - `n_frames = int(len(data) / ROWS_PER_FRAME)`: Calculates the number of frames in the data by dividing the total number of rows by the number of landmarks per frame.\n  - `data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))`: Reshapes the data into a 3D array representing frames, landmarks, and coordinates.\n  - `return data.astype(np.float32)`: Returns the reshaped data as a NumPy array with the data type set to float32.\n\nThis function is designed to load and preprocess landmark data from Parquet files in a format suitable for further analysis or modeling related to the ASL Signs competition.","metadata":{}},{"cell_type":"code","source":"\"\"\"\nTensorflow layer to process data in TFLite\nData needs to be processed in the model itself, so we can not use Python\n\"\"\"\n\nclass PreprocessLayer(tf.keras.layers.Layer):\n    \n    #Initialization\n    def __init__(self):\n        super(PreprocessLayer, self).__init__()\n        \n        # Define normalization correction constants\n        normalisation_correction = tf.constant([\n                    # Add 0.50 to left hand (original right hand) and substract 0.50 of right hand (original left hand)\n                    [0] * len(LIPS_IDXS) + [0.50] * len(LEFT_HAND_IDXS) + [0.50] * len(POSE_IDXS),\n                    # Y coordinates stay intact\n                    [0] * len(LANDMARK_IDXS_LEFT_DOMINANT0),\n                    # Z coordinates stay intact\n                    [0] * len(LANDMARK_IDXS_LEFT_DOMINANT0),\n                ],\n                dtype=tf.float32,\n            )\n        self.normalisation_correction = tf.transpose(normalisation_correction, [1,0])\n    \n    #Padding Method\n    def pad_edge(self, t, repeats, side):\n        if side == 'LEFT':\n            return tf.concat((tf.repeat(t[:1], repeats=repeats, axis=0), t), axis=0)\n        elif side == 'RIGHT':\n            return tf.concat((t, tf.repeat(t[-1:], repeats=repeats, axis=0)), axis=0)\n    \n    @tf.function(\n        input_signature=(tf.TensorSpec(shape=[None,N_ROWS,N_DIMS], dtype=tf.float32),),\n    )\n    \n    #Call Method\n    def call(self, data0):\n        # Number of Frames in Video\n        N_FRAMES0 = tf.shape(data0)[0]\n        \n        # Find dominant hand by comparing summed absolute coordinates\n        left_hand_sum = tf.math.reduce_sum(tf.where(tf.math.is_nan(tf.gather(data0, LEFT_HAND_IDXS0, axis=1)), 0, 1))\n        right_hand_sum = tf.math.reduce_sum(tf.where(tf.math.is_nan(tf.gather(data0, RIGHT_HAND_IDXS0, axis=1)), 0, 1))\n        left_dominant = left_hand_sum >= right_hand_sum\n        \n        # Count non NaN Hand values in each frame for the dominant hand\n        if left_dominant:\n            frames_hands_non_nan_sum = tf.math.reduce_sum(\n                    tf.where(tf.math.is_nan(tf.gather(data0, LEFT_HAND_IDXS0, axis=1)), 0, 1),\n                    axis=[1, 2],\n                )\n        else:\n            frames_hands_non_nan_sum = tf.math.reduce_sum(\n                    tf.where(tf.math.is_nan(tf.gather(data0, RIGHT_HAND_IDXS0, axis=1)), 0, 1),\n                    axis=[1, 2],\n                )\n        \n        # Find frames indices with coordinates of dominant hand\n        non_empty_frames_idxs = tf.where(frames_hands_non_nan_sum > 0)\n        non_empty_frames_idxs = tf.squeeze(non_empty_frames_idxs, axis=1)\n        # Filter frames\n        data = tf.gather(data0, non_empty_frames_idxs, axis=0)\n        \n        # Cast Indices in float32 to be compatible with Tensorflow Lite\n        non_empty_frames_idxs = tf.cast(non_empty_frames_idxs, tf.float32)\n        # Normalize to start with 0\n        non_empty_frames_idxs -= tf.reduce_min(non_empty_frames_idxs)\n        \n        # Number of Frames in Filtered Video\n        N_FRAMES = tf.shape(data)[0]\n        \n        # Gather Relevant Landmark Columns\n        if left_dominant:\n            data = tf.gather(data, LANDMARK_IDXS_LEFT_DOMINANT0, axis=1)\n        else:\n            data = tf.gather(data, LANDMARK_IDXS_RIGHT_DOMINANT0, axis=1)\n            data = (\n                    self.normalisation_correction + (\n                        (data - self.normalisation_correction) * tf.where(self.normalisation_correction != 0, -1.0, 1.0))\n                )\n        \n        # Video fits in INPUT_SIZE\n        if N_FRAMES < INPUT_SIZE:\n            # Pad With -1 to indicate padding\n            non_empty_frames_idxs = tf.pad(non_empty_frames_idxs, [[0, INPUT_SIZE-N_FRAMES]], constant_values=-1)\n            # Pad Data With Zeros\n            data = tf.pad(data, [[0, INPUT_SIZE-N_FRAMES], [0,0], [0,0]], constant_values=0)\n            # Fill NaN Values With 0\n            data = tf.where(tf.math.is_nan(data), 0.0, data)\n            return data, non_empty_frames_idxs\n        # Video needs to be downsampled to INPUT_SIZE\n        else:\n            # Repeat\n            if N_FRAMES < INPUT_SIZE**2:\n                repeats = tf.math.floordiv(INPUT_SIZE * INPUT_SIZE, N_FRAMES0)\n                data = tf.repeat(data, repeats=repeats, axis=0)\n                non_empty_frames_idxs = tf.repeat(non_empty_frames_idxs, repeats=repeats, axis=0)\n\n            # Pad To Multiple Of Input Size\n            pool_size = tf.math.floordiv(len(data), INPUT_SIZE)\n            if tf.math.mod(len(data), INPUT_SIZE) > 0:\n                pool_size += 1\n\n            if pool_size == 1:\n                pad_size = (pool_size * INPUT_SIZE) - len(data)\n            else:\n                pad_size = (pool_size * INPUT_SIZE) % len(data)\n\n            # Pad Start/End with Start/End value\n            pad_left = tf.math.floordiv(pad_size, 2) + tf.math.floordiv(INPUT_SIZE, 2)\n            pad_right = tf.math.floordiv(pad_size, 2) + tf.math.floordiv(INPUT_SIZE, 2)\n            if tf.math.mod(pad_size, 2) > 0:\n                pad_right += 1\n\n            # Pad By Concatenating Left/Right Edge Values\n            data = self.pad_edge(data, pad_left, 'LEFT')\n            data = self.pad_edge(data, pad_right, 'RIGHT')\n\n            # Pad Non Empty Frame Indices\n            non_empty_frames_idxs = self.pad_edge(non_empty_frames_idxs, pad_left, 'LEFT')\n            non_empty_frames_idxs = self.pad_edge(non_empty_frames_idxs, pad_right, 'RIGHT')\n\n            # Reshape to Mean Pool\n            data = tf.reshape(data, [INPUT_SIZE, -1, N_COLS, N_DIMS])\n            non_empty_frames_idxs = tf.reshape(non_empty_frames_idxs, [INPUT_SIZE, -1])\n\n            # Mean Pool\n            data = tf.experimental.numpy.nanmean(data, axis=1)\n            non_empty_frames_idxs = tf.experimental.numpy.nanmean(non_empty_frames_idxs, axis=1)\n\n            # Fill NaN Values With 0\n            data = tf.where(tf.math.is_nan(data), 0.0, data)\n            \n            return data, non_empty_frames_idxs\n# Installation     \npreprocess_layer = PreprocessLayer()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:00:29.242107Z","iopub.execute_input":"2023-11-22T08:00:29.242937Z","iopub.status.idle":"2023-11-22T08:00:31.819335Z","shell.execute_reply.started":"2023-11-22T08:00:29.242896Z","shell.execute_reply":"2023-11-22T08:00:31.818439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"1. Initialization:\nThis initializes the layer and defines a normalization correction tensor, which is then transposed for later use.\n1. Padding Method:\nThis method pads a given tensor along a specified side by repeating the edge values.\n1. 'call' Method:\nThis method is the main entry point during the forward pass. It takes input data data0 and performs various operations, including filtering frames, handling NaN values, normalization, padding, and downsampling. The code includes logic to handle cases where the number of frames is less than INPUT_SIZE and cases where downsampling is needed. It utilizes the pad_edge method for padding and applies mean pooling for downsampling.\n1. Instantiation:\nThis creates an instance of the PreprocessLayer for use in a TensorFlow model.","metadata":{}},{"cell_type":"markdown","source":"# Interpolate NaN Values","metadata":{}},{"cell_type":"code","source":"\"\"\"\n    face: 0:468\n    left_hand: 468:489\n    pose: 489:522\n    right_hand: 522:544\n        \n\"\"\"\ndef get_data(file_path):\n    # Load Raw Data\n    data = load_relevant_data_subset(file_path)\n    # Process Data Using Tensorflow\n    data = preprocess_layer(data)\n    \n    return data","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:00:31.820621Z","iopub.execute_input":"2023-11-22T08:00:31.820963Z","iopub.status.idle":"2023-11-22T08:00:31.826321Z","shell.execute_reply.started":"2023-11-22T08:00:31.82093Z","shell.execute_reply":"2023-11-22T08:00:31.825253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Dataset","metadata":{}},{"cell_type":"code","source":"# Get the full dataset\ndef preprocess_data():\n    # Create arrays to save data\n    X = np.zeros([N_SAMPLES, INPUT_SIZE, N_COLS, N_DIMS], dtype=np.float32)\n    y = np.zeros([N_SAMPLES], dtype=np.int32)\n    NON_EMPTY_FRAME_IDXS = np.full([N_SAMPLES, INPUT_SIZE], -1, dtype=np.float32)\n\n    # Fill X/y\n    for row_idx, (file_path, sign_ord) in enumerate(tqdm(train[['file_path', 'sign_ord']].values)):\n        # Log message every 5000 samples\n        if row_idx % 5000 == 0:\n            print(f'Generated {row_idx}/{N_SAMPLES}')\n\n        data, non_empty_frame_idxs = get_data(file_path)\n        X[row_idx] = data\n        y[row_idx] = sign_ord\n        NON_EMPTY_FRAME_IDXS[row_idx] = non_empty_frame_idxs\n        # Sanity check, data should not contain NaN values\n        if np.isnan(data).sum() > 0:\n            print(row_idx)\n            return data\n\n    # Save X/y\n    np.save('X.npy', X)\n    np.save('y.npy', y)\n    np.save('NON_EMPTY_FRAME_IDXS.npy', NON_EMPTY_FRAME_IDXS)\n    \n    # Save Validation\n    splitter = GroupShuffleSplit(test_size=0.10, n_splits=2, random_state=SEED)\n    PARTICIPANT_IDS = train['participant_id'].values\n    train_idxs, val_idxs = next(splitter.split(X, y, groups=PARTICIPANT_IDS))\n\n    # Save Train\n    X_train = X[train_idxs]\n    NON_EMPTY_FRAME_IDXS_TRAIN = NON_EMPTY_FRAME_IDXS[train_idxs]\n    y_train = y[train_idxs]\n    np.save('X_train.npy', X_train)\n    np.save('y_train.npy', y_train)\n    np.save('NON_EMPTY_FRAME_IDXS_TRAIN.npy', NON_EMPTY_FRAME_IDXS_TRAIN)\n    # Save Validation\n    X_val = X[val_idxs]\n    NON_EMPTY_FRAME_IDXS_VAL = NON_EMPTY_FRAME_IDXS[val_idxs]\n    y_val = y[val_idxs]\n    np.save('X_val.npy', X_val)\n    np.save('y_val.npy', y_val)\n    np.save('NON_EMPTY_FRAME_IDXS_VAL.npy', NON_EMPTY_FRAME_IDXS_VAL)\n    # Split Statistics\n    print(f'Patient ID Intersection Train/Val: {set(PARTICIPANT_IDS[train_idxs]).intersection(PARTICIPANT_IDS[val_idxs])}')\n    print(f'X_train shape: {X_train.shape}, X_val shape: {X_val.shape}')\n    print(f'y_train shape: {y_train.shape}, y_val shape: {y_val.shape}')","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:00:31.82785Z","iopub.execute_input":"2023-11-22T08:00:31.828216Z","iopub.status.idle":"2023-11-22T08:00:31.84249Z","shell.execute_reply.started":"2023-11-22T08:00:31.828179Z","shell.execute_reply":"2023-11-22T08:00:31.841618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Preprocess All Data From Scratch\nif PREPROCESS_DATA:\n    preprocess_data()\n    ROOT_DIR = '.'\nelse:\n    ROOT_DIR = '/kaggle/input/gislr-dataset-public'\n    \n# Load Data\nif USE_VAL:\n    # Load Train\n    X_train = np.load(f'{ROOT_DIR}/X_train.npy')\n    y_train = np.load(f'{ROOT_DIR}/y_train.npy')\n    NON_EMPTY_FRAME_IDXS_TRAIN = np.load(f'{ROOT_DIR}/NON_EMPTY_FRAME_IDXS_TRAIN.npy')\n    # Load Val\n    X_val = np.load(f'{ROOT_DIR}/X_val.npy')\n    y_val = np.load(f'{ROOT_DIR}/y_val.npy')\n    NON_EMPTY_FRAME_IDXS_VAL = np.load(f'{ROOT_DIR}/NON_EMPTY_FRAME_IDXS_VAL.npy')\n    # Define validation Data\n    validation_data = ({ 'frames': X_val, 'non_empty_frame_idxs': NON_EMPTY_FRAME_IDXS_VAL }, y_val)\nelse:\n    X_train = np.load(f'{ROOT_DIR}/X.npy')\n    y_train = np.load(f'{ROOT_DIR}/y.npy')\n    NON_EMPTY_FRAME_IDXS_TRAIN = np.load(f'{ROOT_DIR}/NON_EMPTY_FRAME_IDXS.npy')\n    validation_data = None\n\n# Train \nprint_shape_dtype([X_train, y_train, NON_EMPTY_FRAME_IDXS_TRAIN], ['X_train', 'y_train', 'NON_EMPTY_FRAME_IDXS_TRAIN'])\n# Val\nif USE_VAL:\n    print_shape_dtype([X_val, y_val, NON_EMPTY_FRAME_IDXS_VAL], ['X_val', 'y_val', 'NON_EMPTY_FRAME_IDXS_VAL'])\n# Sanity Check\nprint(f'# NaN Values X_train: {np.isnan(X_train).sum()}')","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:00:31.843629Z","iopub.execute_input":"2023-11-22T08:00:31.843927Z","iopub.status.idle":"2023-11-22T08:01:30.297289Z","shell.execute_reply.started":"2023-11-22T08:00:31.843899Z","shell.execute_reply":"2023-11-22T08:01:30.29619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Class Count\ndisplay(pd.Series(y_train).value_counts().to_frame('Class Count').iloc[[0,1,2,3,4, -5,-4,-3,-2,-1]])","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:01:30.298629Z","iopub.execute_input":"2023-11-22T08:01:30.298932Z","iopub.status.idle":"2023-11-22T08:01:30.312903Z","shell.execute_reply.started":"2023-11-22T08:01:30.298902Z","shell.execute_reply":"2023-11-22T08:01:30.311815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Number Of Frames","metadata":{}},{"cell_type":"code","source":"# Vast majority of samples fits has less than 32 non empty frames\nN_EMPTY_FRAMES = (NON_EMPTY_FRAME_IDXS_TRAIN != -1).sum(axis=1) \nN_EMPTY_FRAMES_WATERFALL = []\nfor n in tqdm(range(1,INPUT_SIZE+1)):\n    N_EMPTY_FRAMES_WATERFALL.append(sum(N_EMPTY_FRAMES >= n) / len(NON_EMPTY_FRAME_IDXS_TRAIN) * 100)\n\nplt.figure(figsize=(18,10))\nplt.title('Waterfall Plot For Number Of Non Empty Frames')\npd.Series(N_EMPTY_FRAMES_WATERFALL).plot(kind='bar')\nplt.grid(axis='y')\nplt.xticks(np.arange(INPUT_SIZE), np.arange(1, INPUT_SIZE+1))\nplt.xlabel('Number of Non Empty Frames', size=16)\nplt.yticks(np.arange(0, 100+10, 10))\nplt.ylim(0, 100)\nplt.ylabel('Percentage of Samples With At Least N Non Empty Frames', size=16)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:01:30.314083Z","iopub.execute_input":"2023-11-22T08:01:30.31435Z","iopub.status.idle":"2023-11-22T08:01:50.736466Z","shell.execute_reply.started":"2023-11-22T08:01:30.314323Z","shell.execute_reply":"2023-11-22T08:01:50.735332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Percentage of Frames Filled","metadata":{}},{"cell_type":"code","source":"# Percentage of frames filled, this is the maximum fill percentage of each landmark\nP_DATA_FILLED = (NON_EMPTY_FRAME_IDXS_TRAIN != -1).sum() / NON_EMPTY_FRAME_IDXS_TRAIN.size * 100\nprint(f'P_DATA_FILLED: {P_DATA_FILLED:.2f}%')","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:01:50.738104Z","iopub.execute_input":"2023-11-22T08:01:50.739154Z","iopub.status.idle":"2023-11-22T08:01:50.755178Z","shell.execute_reply.started":"2023-11-22T08:01:50.739106Z","shell.execute_reply":"2023-11-22T08:01:50.75387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Statistics - Lips","metadata":{}},{"cell_type":"code","source":"# Percentage of Lips Measurements\nP_LEFT_LIPS_MEASUREMENTS = (X_train[:,:,LIPS_IDXS] != 0).sum() / X_train[:,:,LIPS_IDXS].size / P_DATA_FILLED * 1e4\nprint(f'P_LEFT_LIPS_MEASUREMENTS: {P_LEFT_LIPS_MEASUREMENTS:.2f}%')","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:01:50.756566Z","iopub.execute_input":"2023-11-22T08:01:50.756906Z","iopub.status.idle":"2023-11-22T08:02:06.010145Z","shell.execute_reply.started":"2023-11-22T08:01:50.756874Z","shell.execute_reply":"2023-11-22T08:02:06.009022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_lips_mean_std():\n    # LIPS\n    LIPS_MEAN_X = np.zeros([LIPS_IDXS.size], dtype=np.float32)\n    LIPS_MEAN_Y = np.zeros([LIPS_IDXS.size], dtype=np.float32)\n    LIPS_STD_X = np.zeros([LIPS_IDXS.size], dtype=np.float32)\n    LIPS_STD_Y = np.zeros([LIPS_IDXS.size], dtype=np.float32)\n\n    fig, axes = plt.subplots(3, 1, figsize=(15, N_DIMS*6))\n\n    for col, ll in enumerate(tqdm( np.transpose(X_train[:,:,LIPS_IDXS], [2,3,0,1]).reshape([LIPS_IDXS.size, N_DIMS, -1]) )):\n        for dim, l in enumerate(ll):\n            v = l[np.nonzero(l)]\n            if dim == 0: # X\n                LIPS_MEAN_X[col] = v.mean()\n                LIPS_STD_X[col] = v.std()\n            if dim == 1: # Y\n                LIPS_MEAN_Y[col] = v.mean()\n                LIPS_STD_Y[col] = v.std()\n\n            axes[dim].boxplot(v, notch=False, showfliers=False, positions=[col], whis=[5,95])\n\n    for ax, dim_name in zip(axes, DIM_NAMES):\n        ax.set_title(f'Lips {dim_name.upper()} Dimension', size=24)\n        ax.tick_params(axis='x', labelsize=8)\n        ax.grid(axis='y')\n\n    plt.subplots_adjust(hspace=0.50)\n    plt.show()\n\n    LIPS_MEAN = np.array([LIPS_MEAN_X, LIPS_MEAN_Y]).T\n    LIPS_STD = np.array([LIPS_STD_X, LIPS_STD_Y]).T\n    \n    return LIPS_MEAN, LIPS_STD\n\nLIPS_MEAN, LIPS_STD = get_lips_mean_std()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:02:06.011621Z","iopub.execute_input":"2023-11-22T08:02:06.011954Z","iopub.status.idle":"2023-11-22T08:02:27.128428Z","shell.execute_reply.started":"2023-11-22T08:02:06.011921Z","shell.execute_reply":"2023-11-22T08:02:27.127402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Statistics - Hands","metadata":{}},{"cell_type":"code","source":"# Verify Normalised to Left Hand Dominant\nP_LEFT_HAND_MEASUREMENTS = (X_train[:,:,LEFT_HAND_IDXS] != 0).sum() / X_train[:,:,LEFT_HAND_IDXS].size / P_DATA_FILLED * 1e4\n# P_RIGHT_HAND_MEASUREMENTS = (X_train[:,:,RIGHT_HAND_IDXS] != 0).sum() / X_train[:,:,RIGHT_HAND_IDXS].size / P_DATA_FILLED * 1e4\nprint(f'P_LEFT_HAND_MEASUREMENTS: {P_LEFT_HAND_MEASUREMENTS:.2f}%')","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:02:27.130013Z","iopub.execute_input":"2023-11-22T08:02:27.130521Z","iopub.status.idle":"2023-11-22T08:02:35.012179Z","shell.execute_reply.started":"2023-11-22T08:02:27.130479Z","shell.execute_reply":"2023-11-22T08:02:35.011147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_left_right_hand_mean_std():\n    # LEFT HAND\n    LEFT_HANDS_MEAN_X = np.zeros([LEFT_HAND_IDXS.size], dtype=np.float32)\n    LEFT_HANDS_MEAN_Y = np.zeros([LEFT_HAND_IDXS.size], dtype=np.float32)\n    LEFT_HANDS_STD_X = np.zeros([LEFT_HAND_IDXS.size], dtype=np.float32)\n    LEFT_HANDS_STD_Y = np.zeros([LEFT_HAND_IDXS.size], dtype=np.float32)\n\n    fig, axes = plt.subplots(3, 1, figsize=(15, N_DIMS*6))\n\n    for col, ll in enumerate(tqdm( np.transpose(X_train[:,:,LEFT_HAND_IDXS], [2,3,0,1]).reshape([LEFT_HAND_IDXS.size, N_DIMS, -1]) )):\n        for dim, l in enumerate(ll):\n            v = l[np.nonzero(l)]\n            if dim == 0: # X\n                LEFT_HANDS_MEAN_X[col] = v.mean()\n                LEFT_HANDS_STD_X[col] = v.std()\n            if dim == 1: # Y\n                LEFT_HANDS_MEAN_Y[col] = v.mean()\n                LEFT_HANDS_STD_Y[col] = v.std()\n            # Plot\n            axes[dim].boxplot(v, notch=False, showfliers=False, positions=[col], whis=[5,95])\n\n    for ax, dim_name in zip(axes, DIM_NAMES):\n        ax.set_title(f'Hands {dim_name.upper()} Dimension', size=24)\n        ax.tick_params(axis='x', labelsize=8)\n        ax.grid(axis='y')\n\n    plt.subplots_adjust(hspace=0.50)\n    plt.show()\n\n    LEFT_HANDS_MEAN = np.array([LEFT_HANDS_MEAN_X, LEFT_HANDS_MEAN_Y]).T\n    LEFT_HANDS_STD = np.array([LEFT_HANDS_STD_X, LEFT_HANDS_STD_Y]).T\n    \n    return LEFT_HANDS_MEAN, LEFT_HANDS_STD\n\nLEFT_HANDS_MEAN, LEFT_HANDS_STD = get_left_right_hand_mean_std()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:02:35.013738Z","iopub.execute_input":"2023-11-22T08:02:35.014247Z","iopub.status.idle":"2023-11-22T08:02:46.979727Z","shell.execute_reply.started":"2023-11-22T08:02:35.014203Z","shell.execute_reply":"2023-11-22T08:02:46.978691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Statistics - Pose","metadata":{}},{"cell_type":"code","source":"# Percentage of Lips Measurements\nP_POSE_MEASUREMENTS = (X_train[:,:,POSE_IDXS] != 0).sum() / X_train[:,:,POSE_IDXS].size / P_DATA_FILLED * 1e4\nprint(f'P_POSE_MEASUREMENTS: {P_POSE_MEASUREMENTS:.2f}%')","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:02:46.981085Z","iopub.execute_input":"2023-11-22T08:02:46.981403Z","iopub.status.idle":"2023-11-22T08:02:48.947514Z","shell.execute_reply.started":"2023-11-22T08:02:46.981373Z","shell.execute_reply":"2023-11-22T08:02:48.946406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_pose_mean_std():\n    # POSE\n    POSE_MEAN_X = np.zeros([POSE_IDXS.size], dtype=np.float32)\n    POSE_MEAN_Y = np.zeros([POSE_IDXS.size], dtype=np.float32)\n    POSE_STD_X = np.zeros([POSE_IDXS.size], dtype=np.float32)\n    POSE_STD_Y = np.zeros([POSE_IDXS.size], dtype=np.float32)\n\n    fig, axes = plt.subplots(3, 1, figsize=(15, N_DIMS*6))\n\n    for col, ll in enumerate(tqdm( np.transpose(X_train[:,:,POSE_IDXS], [2,3,0,1]).reshape([POSE_IDXS.size, N_DIMS, -1]) )):\n        for dim, l in enumerate(ll):\n            v = l[np.nonzero(l)]\n            if dim == 0: # X\n                POSE_MEAN_X[col] = v.mean()\n                POSE_STD_X[col] = v.std()\n            if dim == 1: # Y\n                POSE_MEAN_Y[col] = v.mean()\n                POSE_STD_Y[col] = v.std()\n\n            axes[dim].boxplot(v, notch=False, showfliers=False, positions=[col], whis=[5,95])\n\n    for ax, dim_name in zip(axes, DIM_NAMES):\n        ax.set_title(f'Pose {dim_name.upper()} Dimension', size=24)\n        ax.tick_params(axis='x', labelsize=8)\n        ax.grid(axis='y')\n\n    plt.subplots_adjust(hspace=0.50)\n    plt.show()\n\n    POSE_MEAN = np.array([POSE_MEAN_X, POSE_MEAN_Y]).T\n    POSE_STD = np.array([POSE_STD_X, POSE_STD_Y]).T\n    \n    return POSE_MEAN, POSE_STD\n\nPOSE_MEAN, POSE_STD = get_pose_mean_std()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:02:48.95492Z","iopub.execute_input":"2023-11-22T08:02:48.955326Z","iopub.status.idle":"2023-11-22T08:02:52.083134Z","shell.execute_reply.started":"2023-11-22T08:02:48.955293Z","shell.execute_reply":"2023-11-22T08:02:52.082115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Samples","metadata":{}},{"cell_type":"code","source":"# Custom sampler to get a batch containing N times all signs\ndef get_train_batch_all_signs(X, y, NON_EMPTY_FRAME_IDXS, n=BATCH_ALL_SIGNS_N):\n    # Arrays to store batch in\n    X_batch = np.zeros([NUM_CLASSES*n, INPUT_SIZE, N_COLS, N_DIMS], dtype=np.float32)\n    y_batch = np.arange(0, NUM_CLASSES, step=1/n, dtype=np.float32).astype(np.int64)\n    non_empty_frame_idxs_batch = np.zeros([NUM_CLASSES*n, INPUT_SIZE], dtype=np.float32)\n    \n    # Dictionary mapping ordinally encoded sign to corresponding sample indices\n    CLASS2IDXS = {}\n    for i in range(NUM_CLASSES):\n        CLASS2IDXS[i] = np.argwhere(y == i).squeeze().astype(np.int32)\n            \n    while True:\n        # Fill batch arrays\n        for i in range(NUM_CLASSES):\n            idxs = np.random.choice(CLASS2IDXS[i], n)\n            X_batch[i*n:(i+1)*n] = X[idxs]\n            non_empty_frame_idxs_batch[i*n:(i+1)*n] = NON_EMPTY_FRAME_IDXS[idxs]\n        \n        yield { 'frames': X_batch, 'non_empty_frame_idxs': non_empty_frame_idxs_batch }, y_batch","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:02:52.084342Z","iopub.execute_input":"2023-11-22T08:02:52.084645Z","iopub.status.idle":"2023-11-22T08:02:52.094678Z","shell.execute_reply.started":"2023-11-22T08:02:52.084616Z","shell.execute_reply":"2023-11-22T08:02:52.093693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dummy_dataset = get_train_batch_all_signs(X_train, y_train, NON_EMPTY_FRAME_IDXS_TRAIN)\nX_batch, y_batch = next(dummy_dataset)\n\nfor k, v in X_batch.items():\n    print(f'{k} shape: {v.shape}, dtype: {v.dtype}')\n\n# Batch shape/dtype\nprint(f'y_batch shape: {y_batch.shape}, dtype: {y_batch.dtype}')\n# Verify each batch contains each sign exactly N times\ndisplay(pd.Series(y_batch).value_counts().to_frame('Counts'))","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:02:52.096063Z","iopub.execute_input":"2023-11-22T08:02:52.097034Z","iopub.status.idle":"2023-11-22T08:02:52.179174Z","shell.execute_reply.started":"2023-11-22T08:02:52.097002Z","shell.execute_reply":"2023-11-22T08:02:52.178099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Config","metadata":{}},{"cell_type":"code","source":"# Epsilon value for layer normalisation\nLAYER_NORM_EPS = 1e-6\n\n# Dense layer units for landmarks\nLIPS_UNITS = 384\nHANDS_UNITS = 384\nPOSE_UNITS = 384\n# final embedding and transformer embedding size\nUNITS = 512\n\n# Transformer\nNUM_BLOCKS = 2\nMLP_RATIO = 2\n\n# Dropout\nEMBEDDING_DROPOUT = 0.00\nMLP_DROPOUT_RATIO = 0.30\nCLASSIFIER_DROPOUT_RATIO = 0.10\n\n# Initiailizers\nINIT_HE_UNIFORM = tf.keras.initializers.he_uniform\nINIT_GLOROT_UNIFORM = tf.keras.initializers.glorot_uniform\nINIT_ZEROS = tf.keras.initializers.constant(0.0)\n# Activations\nGELU = tf.keras.activations.gelu\n\nprint(f'UNITS: {UNITS}')","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:02:52.180436Z","iopub.execute_input":"2023-11-22T08:02:52.180747Z","iopub.status.idle":"2023-11-22T08:02:52.188355Z","shell.execute_reply.started":"2023-11-22T08:02:52.180718Z","shell.execute_reply":"2023-11-22T08:02:52.187377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transformer\n\nNeed to implement transformer from scratch as TFLite does not support the native TF implementation of MultiHeadAttention.","metadata":{}},{"cell_type":"code","source":"# based on: https://stackoverflow.com/questions/67342988/verifying-the-implementation-of-multihead-attention-in-transformer\n# replaced softmax with softmax layer to support masked softmax\ndef scaled_dot_product(q,k,v, softmax, attention_mask):\n    #calculates Q . K(transpose)\n    qkt = tf.matmul(q,k,transpose_b=True)\n    #caculates scaling factor\n    dk = tf.math.sqrt(tf.cast(q.shape[-1],dtype=tf.float32))\n    scaled_qkt = qkt/dk\n    softmax = softmax(scaled_qkt, mask=attention_mask)\n    \n    z = tf.matmul(softmax,v)\n    #shape: (m,Tx,depth), same shape as q,k,v\n    return z\n\nclass MultiHeadAttention(tf.keras.layers.Layer):\n    def __init__(self,d_model,num_of_heads):\n        super(MultiHeadAttention,self).__init__()\n        self.d_model = d_model\n        self.num_of_heads = num_of_heads\n        self.depth = d_model//num_of_heads\n        self.wq = [tf.keras.layers.Dense(self.depth) for i in range(num_of_heads)]\n        self.wk = [tf.keras.layers.Dense(self.depth) for i in range(num_of_heads)]\n        self.wv = [tf.keras.layers.Dense(self.depth) for i in range(num_of_heads)]\n        self.wo = tf.keras.layers.Dense(d_model)\n        self.softmax = tf.keras.layers.Softmax()\n        \n    def call(self,x, attention_mask):\n        \n        multi_attn = []\n        for i in range(self.num_of_heads):\n            Q = self.wq[i](x)\n            K = self.wk[i](x)\n            V = self.wv[i](x)\n            multi_attn.append(scaled_dot_product(Q,K,V, self.softmax, attention_mask))\n            \n        multi_head = tf.concat(multi_attn,axis=-1)\n        multi_head_attention = self.wo(multi_head)\n        return multi_head_attention","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:02:52.189964Z","iopub.execute_input":"2023-11-22T08:02:52.190577Z","iopub.status.idle":"2023-11-22T08:02:52.204908Z","shell.execute_reply.started":"2023-11-22T08:02:52.190542Z","shell.execute_reply":"2023-11-22T08:02:52.203912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Full Transformer\nclass Transformer(tf.keras.Model):\n    def __init__(self, num_blocks):\n        super(Transformer, self).__init__(name='transformer')\n        self.num_blocks = num_blocks\n    \n    def build(self, input_shape):\n        self.ln_1s = []\n        self.mhas = []\n        self.ln_2s = []\n        self.mlps = []\n        # Make Transformer Blocks\n        for i in range(self.num_blocks):\n            # Multi Head Attention\n            self.mhas.append(MultiHeadAttention(UNITS, 8))\n            # Multi Layer Perception\n            self.mlps.append(tf.keras.Sequential([\n                tf.keras.layers.Dense(UNITS * MLP_RATIO, activation=GELU, kernel_initializer=INIT_GLOROT_UNIFORM),\n                tf.keras.layers.Dropout(MLP_DROPOUT_RATIO),\n                tf.keras.layers.Dense(UNITS, kernel_initializer=INIT_HE_UNIFORM),\n            ]))\n        \n    def call(self, x, attention_mask):\n        # Iterate input over transformer blocks\n        for mha, mlp in zip(self.mhas, self.mlps):\n            x = x + mha(x, attention_mask)\n            x = x + mlp(x)\n    \n        return x","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:02:52.206391Z","iopub.execute_input":"2023-11-22T08:02:52.206991Z","iopub.status.idle":"2023-11-22T08:02:52.218948Z","shell.execute_reply.started":"2023-11-22T08:02:52.206951Z","shell.execute_reply":"2023-11-22T08:02:52.218014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Landmark Embedding","metadata":{}},{"cell_type":"code","source":"class LandmarkEmbedding(tf.keras.Model):\n    def __init__(self, units, name):\n        super(LandmarkEmbedding, self).__init__(name=f'{name}_embedding')\n        self.units = units\n        \n    def build(self, input_shape):\n        # Embedding for missing landmark in frame, initizlied with zeros\n        self.empty_embedding = self.add_weight(\n            name=f'{self.name}_empty_embedding',\n            shape=[self.units],\n            initializer=INIT_ZEROS,\n        )\n        # Embedding\n        self.dense = tf.keras.Sequential([\n            tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_1', use_bias=False, kernel_initializer=INIT_GLOROT_UNIFORM),\n            tf.keras.layers.Activation(GELU),\n            tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_2', use_bias=False, kernel_initializer=INIT_HE_UNIFORM),\n        ], name=f'{self.name}_dense')\n\n    def call(self, x):\n        return tf.where(\n                # Checks whether landmark is missing in frame\n                tf.reduce_sum(x, axis=2, keepdims=True) == 0,\n                # If so, the empty embedding is used\n                self.empty_embedding,\n                # Otherwise the landmark data is embedded\n                self.dense(x),\n            )","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:02:52.220523Z","iopub.execute_input":"2023-11-22T08:02:52.22099Z","iopub.status.idle":"2023-11-22T08:02:52.231417Z","shell.execute_reply.started":"2023-11-22T08:02:52.220951Z","shell.execute_reply":"2023-11-22T08:02:52.230507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Embedding","metadata":{}},{"cell_type":"code","source":"class Embedding(tf.keras.Model):\n    def __init__(self):\n        super(Embedding, self).__init__()\n        \n    def get_diffs(self, l):\n        S = l.shape[2]\n        other = tf.expand_dims(l, 3)\n        other = tf.repeat(other, S, axis=3)\n        other = tf.transpose(other, [0,1,3,2])\n        diffs = tf.expand_dims(l, 3) - other\n        diffs = tf.reshape(diffs, [-1, INPUT_SIZE, S*S])\n        return diffs\n\n    def build(self, input_shape):\n        # Positional Embedding, initialized with zeros\n        self.positional_embedding = tf.keras.layers.Embedding(INPUT_SIZE+1, UNITS, embeddings_initializer=INIT_ZEROS)\n        # Embedding layer for Landmarks\n        self.lips_embedding = LandmarkEmbedding(LIPS_UNITS, 'lips')\n        self.left_hand_embedding = LandmarkEmbedding(HANDS_UNITS, 'left_hand')\n        self.pose_embedding = LandmarkEmbedding(POSE_UNITS, 'pose')\n        # Landmark Weights\n        self.landmark_weights = tf.Variable(tf.zeros([3], dtype=tf.float32), name='landmark_weights')\n        # Fully Connected Layers for combined landmarks\n        self.fc = tf.keras.Sequential([\n            tf.keras.layers.Dense(UNITS, name='fully_connected_1', use_bias=False, kernel_initializer=INIT_GLOROT_UNIFORM),\n            tf.keras.layers.Activation(GELU),\n            tf.keras.layers.Dense(UNITS, name='fully_connected_2', use_bias=False, kernel_initializer=INIT_HE_UNIFORM),\n        ], name='fc')\n\n\n    def call(self, lips0, left_hand0, pose0, non_empty_frame_idxs, training=False):\n        # Lips\n        lips_embedding = self.lips_embedding(lips0)\n        # Left Hand\n        left_hand_embedding = self.left_hand_embedding(left_hand0)\n        # Pose\n        pose_embedding = self.pose_embedding(pose0)\n        # Merge Embeddings of all landmarks with mean pooling\n        x = tf.stack((\n            lips_embedding, left_hand_embedding, pose_embedding,\n        ), axis=3)\n        x = x * tf.nn.softmax(self.landmark_weights)\n        x = tf.reduce_sum(x, axis=3)\n        # Fully Connected Layers\n        x = self.fc(x)\n        # Add Positional Embedding\n        max_frame_idxs = tf.clip_by_value(\n                tf.reduce_max(non_empty_frame_idxs, axis=1, keepdims=True),\n                1,\n                np.PINF,\n            )\n        normalised_non_empty_frame_idxs = tf.where(\n            tf.math.equal(non_empty_frame_idxs, -1.0),\n            INPUT_SIZE,\n            tf.cast(\n                non_empty_frame_idxs / max_frame_idxs * INPUT_SIZE,\n                tf.int32,\n            ),\n        )\n        x = x + self.positional_embedding(normalised_non_empty_frame_idxs)\n        \n        return x","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:02:52.232971Z","iopub.execute_input":"2023-11-22T08:02:52.233256Z","iopub.status.idle":"2023-11-22T08:02:52.249666Z","shell.execute_reply.started":"2023-11-22T08:02:52.233229Z","shell.execute_reply":"2023-11-22T08:02:52.248718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Augmentation","metadata":{}},{"cell_type":"code","source":"# Not used, adds random X/y translation to input on samples level\nclass Augmentation(tf.keras.layers.Layer):\n    def __init__(self, noise_std):\n        super(Augmentation, self).__init__()\n        self.noise_std = noise_std\n    \n    def add_noise(self, t):\n        B = tf.shape(t)[0]\n        return tf.where(\n            t == 0.0,\n            0.0,\n            t + tf.random.normal([B,1,1,tf.shape(t)[3]], 0, self.noise_std),\n        )\n    \n    def call(self, lips0, left_hand0, pose0, training=False):\n        if training:\n            # Lips\n            lips0 = self.add_noise(lips0)\n            # Left Hand\n            left_hand0 = self.add_noise(left_hand0)\n            # Pose\n            pose0 = self.add_noise(pose0)\n        \n        return lips0, left_hand0, pose0","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:02:52.250757Z","iopub.execute_input":"2023-11-22T08:02:52.25104Z","iopub.status.idle":"2023-11-22T08:02:52.262373Z","shell.execute_reply.started":"2023-11-22T08:02:52.251013Z","shell.execute_reply":"2023-11-22T08:02:52.261475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Sparse Categorical Crossentropy With Label Smoothing","metadata":{}},{"cell_type":"code","source":"# source:: https://stackoverflow.com/questions/60689185/label-smoothing-for-sparse-categorical-crossentropy\ndef scce_with_ls(y_true, y_pred):\n    # One Hot Encode Sparsely Encoded Target Sign\n    y_true = tf.cast(y_true, tf.int32)\n    y_true = tf.one_hot(y_true, NUM_CLASSES, axis=1)\n    y_true = tf.squeeze(y_true, axis=2)\n    # Categorical Crossentropy with native label smoothing support\n    return tf.keras.losses.categorical_crossentropy(y_true, y_pred, label_smoothing=0.25)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:02:52.263417Z","iopub.execute_input":"2023-11-22T08:02:52.263742Z","iopub.status.idle":"2023-11-22T08:02:52.274896Z","shell.execute_reply.started":"2023-11-22T08:02:52.263713Z","shell.execute_reply":"2023-11-22T08:02:52.273783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"def get_model():\n    # Inputs\n    frames = tf.keras.layers.Input([INPUT_SIZE, N_COLS, N_DIMS], dtype=tf.float32, name='frames')\n    non_empty_frame_idxs = tf.keras.layers.Input([INPUT_SIZE], dtype=tf.float32, name='non_empty_frame_idxs')\n    # Padding Mask\n    mask0 = tf.cast(tf.math.not_equal(non_empty_frame_idxs, -1), tf.float32)\n    mask0 = tf.expand_dims(mask0, axis=2)\n    # Random Frame Masking\n    mask = tf.where(\n        (tf.random.uniform(tf.shape(mask0)) > 0.25) & tf.math.not_equal(mask0, 0.0),\n        1.0,\n        0.0,\n    )\n    # Correct Samples Which are all masked now...\n    mask = tf.where(\n        tf.math.equal(tf.reduce_sum(mask, axis=[1,2], keepdims=True), 0.0),\n        mask0,\n        mask,\n    )\n    \n    \n    \"\"\"\n        left_hand: 468:489\n        pose: 489:522\n        right_hand: 522:543\n    \"\"\"\n    x = frames\n    x = tf.slice(x, [0,0,0,0], [-1,INPUT_SIZE, N_COLS, 2])\n    # LIPS\n    lips = tf.slice(x, [0,0,LIPS_START,0], [-1,INPUT_SIZE, 40, 2])\n    lips = tf.where(\n            tf.math.equal(lips, 0.0),\n            0.0,\n            (lips - LIPS_MEAN) / LIPS_STD,\n        )\n    # LEFT HAND\n    left_hand = tf.slice(x, [0,0,40,0], [-1,INPUT_SIZE, 21, 2])\n    left_hand = tf.where(\n            tf.math.equal(left_hand, 0.0),\n            0.0,\n            (left_hand - LEFT_HANDS_MEAN) / LEFT_HANDS_STD,\n        )\n    # POSE\n    pose = tf.slice(x, [0,0,61,0], [-1,INPUT_SIZE, 5, 2])\n    pose = tf.where(\n            tf.math.equal(pose, 0.0),\n            0.0,\n            (pose - POSE_MEAN) / POSE_STD,\n        )\n    \n    # Flatten\n    lips = tf.reshape(lips, [-1, INPUT_SIZE, 40*2])\n    left_hand = tf.reshape(left_hand, [-1, INPUT_SIZE, 21*2])\n    pose = tf.reshape(pose, [-1, INPUT_SIZE, 5*2])\n        \n    # Embedding\n    x = Embedding()(lips, left_hand, pose, non_empty_frame_idxs)\n    \n    # Encoder Transformer Blocks\n    x = Transformer(NUM_BLOCKS)(x, mask)\n    \n    # Pooling\n    x = tf.reduce_sum(x * mask, axis=1) / tf.reduce_sum(mask, axis=1)\n    # Classifier Dropout\n    x = tf.keras.layers.Dropout(CLASSIFIER_DROPOUT_RATIO)(x)\n    # Classification Layer\n    x = tf.keras.layers.Dense(NUM_CLASSES, activation=tf.keras.activations.softmax, kernel_initializer=INIT_GLOROT_UNIFORM)(x)\n    \n    outputs = x\n    \n    # Create Tensorflow Model\n    model = tf.keras.models.Model(inputs=[frames, non_empty_frame_idxs], outputs=outputs)\n    \n    # Sparse Categorical Cross Entropy With Label Smoothing\n    loss = scce_with_ls\n    \n    # Adam Optimizer with weight decay\n    optimizer = tfa.optimizers.AdamW(learning_rate=1e-3, weight_decay=1e-5, clipnorm=1.0)\n    \n    # TopK Metrics\n    metrics = [\n        tf.keras.metrics.SparseCategoricalAccuracy(name='acc'),\n        tf.keras.metrics.SparseTopKCategoricalAccuracy(k=5, name='top_5_acc'),\n        tf.keras.metrics.SparseTopKCategoricalAccuracy(k=10, name='top_10_acc'),\n    ]\n    \n    model.compile(loss=loss, optimizer=optimizer, metrics=metrics)\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:02:52.276537Z","iopub.execute_input":"2023-11-22T08:02:52.276835Z","iopub.status.idle":"2023-11-22T08:02:52.297751Z","shell.execute_reply.started":"2023-11-22T08:02:52.276808Z","shell.execute_reply":"2023-11-22T08:02:52.296713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.backend.clear_session()\n\nmodel = get_model()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:02:52.298979Z","iopub.execute_input":"2023-11-22T08:02:52.299292Z","iopub.status.idle":"2023-11-22T08:02:54.568873Z","shell.execute_reply.started":"2023-11-22T08:02:52.299256Z","shell.execute_reply":"2023-11-22T08:02:54.567997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot model summary\nmodel.summary(expand_nested=True)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:02:54.570257Z","iopub.execute_input":"2023-11-22T08:02:54.570661Z","iopub.status.idle":"2023-11-22T08:02:54.756419Z","shell.execute_reply.started":"2023-11-22T08:02:54.570618Z","shell.execute_reply":"2023-11-22T08:02:54.755262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.utils.plot_model(model, show_shapes=True, show_dtype=True, show_layer_names=True, expand_nested=True, show_layer_activations=True)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:02:54.758234Z","iopub.execute_input":"2023-11-22T08:02:54.758732Z","iopub.status.idle":"2023-11-22T08:02:55.996263Z","shell.execute_reply.started":"2023-11-22T08:02:54.758688Z","shell.execute_reply":"2023-11-22T08:02:55.995063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# No NaN Predictions","metadata":{}},{"cell_type":"code","source":"if not PREPROCESS_DATA and TRAIN_MODEL:\n    y_pred = model.predict_on_batch(X_batch).flatten()\n\n    print(f'# NaN Values In Prediction: {np.isnan(y_pred).sum()}')","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:02:55.998337Z","iopub.execute_input":"2023-11-22T08:02:55.999249Z","iopub.status.idle":"2023-11-22T08:02:59.297771Z","shell.execute_reply.started":"2023-11-22T08:02:55.999203Z","shell.execute_reply":"2023-11-22T08:02:59.296469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Weight Initialization","metadata":{}},{"cell_type":"code","source":"if not PREPROCESS_DATA and TRAIN_MODEL:\n    plt.figure(figsize=(12,5))\n    plt.title(f'Softmax Output Initialized Model | µ={y_pred.mean():.3f}, σ={y_pred.std():.3f}', pad=25)\n    pd.Series(y_pred).plot(kind='hist', bins=128, label='Class Probability')\n    plt.xlim(0, max(y_pred) * 1.1)\n    plt.vlines([1 / NUM_CLASSES], 0, plt.ylim()[1], color='red', label=f'Random Guessing Baseline 1/NUM_CLASSES={1 / NUM_CLASSES:.3f}')\n    plt.grid()\n    plt.legend()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:02:59.299154Z","iopub.execute_input":"2023-11-22T08:02:59.299539Z","iopub.status.idle":"2023-11-22T08:02:59.925273Z","shell.execute_reply.started":"2023-11-22T08:02:59.299501Z","shell.execute_reply":"2023-11-22T08:02:59.923824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Learning Rate Scheduler","metadata":{}},{"cell_type":"code","source":"def lrfn(current_step, num_warmup_steps, lr_max, num_cycles=0.50, num_training_steps=N_EPOCHS):\n    \n    if current_step < num_warmup_steps:\n        if WARMUP_METHOD == 'log':\n            return lr_max * 0.10 ** (num_warmup_steps - current_step)\n        else:\n            return lr_max * 2 ** -(num_warmup_steps - current_step)\n    else:\n        progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))\n\n        return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))) * lr_max","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:02:59.927349Z","iopub.execute_input":"2023-11-22T08:02:59.927774Z","iopub.status.idle":"2023-11-22T08:02:59.935399Z","shell.execute_reply.started":"2023-11-22T08:02:59.927741Z","shell.execute_reply":"2023-11-22T08:02:59.934407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_lr_schedule(lr_schedule, epochs):\n    fig = plt.figure(figsize=(20, 10))\n    plt.plot([None] + lr_schedule + [None])\n    # X Labels\n    x = np.arange(1, epochs + 1)\n    x_axis_labels = [i if epochs <= 40 or i % 5 == 0 or i == 1 else None for i in range(1, epochs + 1)]\n    plt.xlim([1, epochs])\n    plt.xticks(x, x_axis_labels) # set tick step to 1 and let x axis start at 1\n    \n    # Increase y-limit for better readability\n    plt.ylim([0, max(lr_schedule) * 1.1])\n    \n    # Title\n    schedule_info = f'start: {lr_schedule[0]:.1E}, max: {max(lr_schedule):.1E}, final: {lr_schedule[-1]:.1E}'\n    plt.title(f'Step Learning Rate Schedule, {schedule_info}', size=18, pad=12)\n    \n    # Plot Learning Rates\n    for x, val in enumerate(lr_schedule):\n        if epochs <= 40 or x % 5 == 0 or x is epochs - 1:\n            if x < len(lr_schedule) - 1:\n                if lr_schedule[x - 1] < val:\n                    ha = 'right'\n                else:\n                    ha = 'left'\n            elif x == 0:\n                ha = 'right'\n            else:\n                ha = 'left'\n            plt.plot(x + 1, val, 'o', color='black');\n            offset_y = (max(lr_schedule) - min(lr_schedule)) * 0.02\n            plt.annotate(f'{val:.1E}', xy=(x + 1, val + offset_y), size=12, ha=ha)\n    \n    plt.xlabel('Epoch', size=16, labelpad=5)\n    plt.ylabel('Learning Rate', size=16, labelpad=5)\n    plt.grid()\n    plt.show()\n\n# Learning rate for encoder\nLR_SCHEDULE = [lrfn(step, num_warmup_steps=N_WARMUP_EPOCHS, lr_max=LR_MAX, num_cycles=0.50) for step in range(N_EPOCHS)]\n# Plot Learning Rate Schedule\nplot_lr_schedule(LR_SCHEDULE, epochs=N_EPOCHS)\n# Learning Rate Callback\nlr_callback = tf.keras.callbacks.LearningRateScheduler(lambda step: LR_SCHEDULE[step], verbose=1)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:02:59.936824Z","iopub.execute_input":"2023-11-22T08:02:59.937106Z","iopub.status.idle":"2023-11-22T08:03:00.749422Z","shell.execute_reply.started":"2023-11-22T08:02:59.937078Z","shell.execute_reply":"2023-11-22T08:03:00.748355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Weight Decay Callback","metadata":{}},{"cell_type":"code","source":"# Custom callback to update weight decay with learning rate\nclass WeightDecayCallback(tf.keras.callbacks.Callback):\n    def __init__(self, wd_ratio=WD_RATIO):\n        self.step_counter = 0\n        self.wd_ratio = wd_ratio\n    \n    def on_epoch_begin(self, epoch, logs=None):\n        model.optimizer.weight_decay = model.optimizer.learning_rate * self.wd_ratio\n        print(f'learning rate: {model.optimizer.learning_rate.numpy():.2e}, weight decay: {model.optimizer.weight_decay.numpy():.2e}')","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:03:00.750898Z","iopub.execute_input":"2023-11-22T08:03:00.75129Z","iopub.status.idle":"2023-11-22T08:03:00.758963Z","shell.execute_reply.started":"2023-11-22T08:03:00.751252Z","shell.execute_reply":"2023-11-22T08:03:00.757934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Performance Benchmark","metadata":{}},{"cell_type":"code","source":"%%timeit -n 100\nif TRAIN_MODEL:\n    # Verify model prediction is <<<100ms\n    model.predict_on_batch({ 'frames': X_train[:1], 'non_empty_frame_idxs': NON_EMPTY_FRAME_IDXS_TRAIN[:1] })\n    pass","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:03:00.760435Z","iopub.execute_input":"2023-11-22T08:03:00.761003Z","iopub.status.idle":"2023-11-22T08:03:13.004649Z","shell.execute_reply.started":"2023-11-22T08:03:00.760962Z","shell.execute_reply":"2023-11-22T08:03:13.00351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"if USE_VAL:\n    # Verify Validation Dataset Covers All Signs\n    print(f'# Unique Signs in Validation Set: {pd.Series(y_val).nunique()}')\n    # Value Counts\n    display(pd.Series(y_val).value_counts().to_frame('Count').iloc[[1,2,3,-3,-2,-1]])","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:03:13.005818Z","iopub.execute_input":"2023-11-22T08:03:13.006117Z","iopub.status.idle":"2023-11-22T08:03:13.012061Z","shell.execute_reply.started":"2023-11-22T08:03:13.006088Z","shell.execute_reply":"2023-11-22T08:03:13.010895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluate Initialzied Model","metadata":{}},{"cell_type":"code","source":"# Sanity Check\nif TRAIN_MODEL and USE_VAL:\n    _ = model.evaluate(*validation_data, verbose=2)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:03:13.013347Z","iopub.execute_input":"2023-11-22T08:03:13.013696Z","iopub.status.idle":"2023-11-22T08:03:13.023512Z","shell.execute_reply.started":"2023-11-22T08:03:13.013666Z","shell.execute_reply":"2023-11-22T08:03:13.022634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"if TRAIN_MODEL:\n    # Clear all models in GPU\n    tf.keras.backend.clear_session()\n\n    # Get new fresh model\n    model = get_model()\n    \n    # Sanity Check\n    model.summary()\n\n    # Actual Training\n    history = model.fit(\n            x=get_train_batch_all_signs(X_train, y_train, NON_EMPTY_FRAME_IDXS_TRAIN),\n            steps_per_epoch=len(X_train) // (NUM_CLASSES * BATCH_ALL_SIGNS_N),\n            epochs=N_EPOCHS,\n            # Only used for validation data since training data is a generator\n            batch_size=BATCH_SIZE,\n            validation_data=validation_data,\n            callbacks=[\n                lr_callback,\n                WeightDecayCallback(),\n            ],\n            verbose = VERBOSE,\n        )","metadata":{"execution":{"iopub.status.busy":"2023-11-22T08:03:13.024674Z","iopub.execute_input":"2023-11-22T08:03:13.025001Z","iopub.status.idle":"2023-11-22T09:26:47.047027Z","shell.execute_reply.started":"2023-11-22T08:03:13.024964Z","shell.execute_reply":"2023-11-22T09:26:47.045912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save Model Weights\nmodel.save_weights('model.h5')","metadata":{"execution":{"iopub.status.busy":"2023-11-22T09:26:47.048672Z","iopub.execute_input":"2023-11-22T09:26:47.048999Z","iopub.status.idle":"2023-11-22T09:26:47.200689Z","shell.execute_reply.started":"2023-11-22T09:26:47.048966Z","shell.execute_reply":"2023-11-22T09:26:47.199501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if USE_VAL:\n    # Validation Predictions\n    y_val_pred = model.predict({ 'frames': X_val, 'non_empty_frame_idxs': NON_EMPTY_FRAME_IDXS_VAL }, verbose=2).argmax(axis=1)\n    # Label\n    labels = [ORD2SIGN.get(i).replace(' ', '_') for i in range(NUM_CLASSES)]","metadata":{"execution":{"iopub.status.busy":"2023-11-22T09:26:47.202005Z","iopub.execute_input":"2023-11-22T09:26:47.202301Z","iopub.status.idle":"2023-11-22T09:26:47.207904Z","shell.execute_reply.started":"2023-11-22T09:26:47.202267Z","shell.execute_reply":"2023-11-22T09:26:47.206893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Landmark Attention Weights","metadata":{}},{"cell_type":"code","source":"# Landmark Weights\nfor w in model.get_layer('embedding').weights:\n    if 'landmark_weights' in w.name:\n        weights = scipy.special.softmax(w)\n\nlandmarks = ['lips_embedding', 'left_hand_embedding', 'pose_embedding']\n\nfor w, lm in zip(weights, landmarks):\n    print(f'{lm} weight: {(w*100):.1f}%')","metadata":{"execution":{"iopub.status.busy":"2023-11-22T09:26:47.209159Z","iopub.execute_input":"2023-11-22T09:26:47.209504Z","iopub.status.idle":"2023-11-22T09:26:47.222595Z","shell.execute_reply.started":"2023-11-22T09:26:47.209463Z","shell.execute_reply":"2023-11-22T09:26:47.221493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Classification Report","metadata":{}},{"cell_type":"code","source":"def print_classification_report():\n    # Classification report for all signs\n    classification_report = sklearn.metrics.classification_report(\n            y_val,\n            y_val_pred,\n            target_names=labels,\n            output_dict=True,\n        )\n    # Round Data for better readability\n    classification_report = pd.DataFrame(classification_report).T\n    classification_report = classification_report.round(2)\n    classification_report = classification_report.astype({\n            'support': np.uint16,\n        })\n    # Add signs\n    classification_report['sign'] = [e if e in SIGN2ORD else -1 for e in classification_report.index]\n    classification_report['sign_ord'] = classification_report['sign'].apply(SIGN2ORD.get).fillna(-1).astype(np.int16)\n    # Sort on F1-score\n    classification_report = pd.concat((\n        classification_report.head(NUM_CLASSES).sort_values('f1-score', ascending=False),\n        classification_report.tail(3),\n    ))\n\n    pd.options.display.max_rows = 999\n    display(classification_report)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T09:26:47.224009Z","iopub.execute_input":"2023-11-22T09:26:47.224534Z","iopub.status.idle":"2023-11-22T09:26:47.23363Z","shell.execute_reply.started":"2023-11-22T09:26:47.224493Z","shell.execute_reply":"2023-11-22T09:26:47.232613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if USE_VAL:\n    print_classification_report()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T09:26:47.234754Z","iopub.execute_input":"2023-11-22T09:26:47.235133Z","iopub.status.idle":"2023-11-22T09:26:47.243126Z","shell.execute_reply.started":"2023-11-22T09:26:47.235093Z","shell.execute_reply":"2023-11-22T09:26:47.24208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training History","metadata":{}},{"cell_type":"code","source":"def plot_history_metric(metric, f_best=np.argmax, ylim=None, yscale=None, yticks=None):\n    plt.figure(figsize=(20, 10))\n    \n    values = history.history[metric]\n    N_EPOCHS = len(values)\n    val = 'val' in ''.join(history.history.keys())\n    # Epoch Ticks\n    if N_EPOCHS <= 20:\n        x = np.arange(1, N_EPOCHS + 1)\n    else:\n        x = [1, 5] + [10 + 5 * idx for idx in range((N_EPOCHS - 10) // 5 + 1)]\n\n    x_ticks = np.arange(1, N_EPOCHS+1)\n\n    # Validation\n    if val:\n        val_values = history.history[f'val_{metric}']\n        val_argmin = f_best(val_values)\n        plt.plot(x_ticks, val_values, label=f'val')\n\n    # summarize history for accuracy\n    plt.plot(x_ticks, values, label=f'train')\n    argmin = f_best(values)\n    plt.scatter(argmin + 1, values[argmin], color='red', s=75, marker='o', label=f'train_best')\n    if val:\n        plt.scatter(val_argmin + 1, val_values[val_argmin], color='purple', s=75, marker='o', label=f'val_best')\n\n    plt.title(f'Model {metric}', fontsize=24, pad=10)\n    plt.ylabel(metric, fontsize=20, labelpad=10)\n\n    if ylim:\n        plt.ylim(ylim)\n\n    if yscale is not None:\n        plt.yscale(yscale)\n        \n    if yticks is not None:\n        plt.yticks(yticks, fontsize=16)\n\n    plt.xlabel('epoch', fontsize=20, labelpad=10)        \n    plt.tick_params(axis='x', labelsize=8)\n    plt.xticks(x, fontsize=16) # set tick step to 1 and let x axis start at 1\n    plt.yticks(fontsize=16)\n    \n    plt.legend(prop={'size': 10})\n    plt.grid()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T09:26:47.2447Z","iopub.execute_input":"2023-11-22T09:26:47.245062Z","iopub.status.idle":"2023-11-22T09:26:47.259716Z","shell.execute_reply.started":"2023-11-22T09:26:47.245024Z","shell.execute_reply":"2023-11-22T09:26:47.258622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if TRAIN_MODEL:\n    plot_history_metric('loss', f_best=np.argmin)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T09:26:47.260772Z","iopub.execute_input":"2023-11-22T09:26:47.26107Z","iopub.status.idle":"2023-11-22T09:26:47.664102Z","shell.execute_reply.started":"2023-11-22T09:26:47.261042Z","shell.execute_reply":"2023-11-22T09:26:47.662969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if TRAIN_MODEL:\n    plot_history_metric('acc', ylim=[0,1], yticks=np.arange(0.0, 1.1, 0.1))","metadata":{"execution":{"iopub.status.busy":"2023-11-22T09:26:47.665391Z","iopub.execute_input":"2023-11-22T09:26:47.666036Z","iopub.status.idle":"2023-11-22T09:26:48.06807Z","shell.execute_reply.started":"2023-11-22T09:26:47.666001Z","shell.execute_reply":"2023-11-22T09:26:48.067053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if TRAIN_MODEL:\n    plot_history_metric('top_5_acc', ylim=[0,1], yticks=np.arange(0.0, 1.1, 0.1))","metadata":{"execution":{"iopub.status.busy":"2023-11-22T09:26:48.069591Z","iopub.execute_input":"2023-11-22T09:26:48.070346Z","iopub.status.idle":"2023-11-22T09:26:48.488149Z","shell.execute_reply.started":"2023-11-22T09:26:48.070297Z","shell.execute_reply":"2023-11-22T09:26:48.487095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if TRAIN_MODEL:\n    plot_history_metric('top_10_acc', ylim=[0,1], yticks=np.arange(0.0, 1.1, 0.1))","metadata":{"execution":{"iopub.status.busy":"2023-11-22T09:26:48.489486Z","iopub.execute_input":"2023-11-22T09:26:48.489803Z","iopub.status.idle":"2023-11-22T09:26:48.888125Z","shell.execute_reply.started":"2023-11-22T09:26:48.489773Z","shell.execute_reply":"2023-11-22T09:26:48.887086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission\n\nSubmission code loosley based on [this notebook](https://www.kaggle.com/code/dschettler8845/gislr-learn-eda-baseline#baseline) by [Darien Schettler\n](https://www.kaggle.com/dschettler8845)","metadata":{}},{"cell_type":"code","source":"# TFLite model for submission\nclass TFLiteModel(tf.Module):\n    def __init__(self, model):\n        super(TFLiteModel, self).__init__()\n\n        # Load the feature generation and main models\n        self.preprocess_layer = preprocess_layer\n        self.model = model\n    \n    @tf.function(input_signature=[tf.TensorSpec(shape=[None, N_ROWS, N_DIMS], dtype=tf.float32, name='inputs')])\n    def __call__(self, inputs):\n        # Preprocess Data\n        x, non_empty_frame_idxs = self.preprocess_layer(inputs)\n        # Add Batch Dimension\n        x = tf.expand_dims(x, axis=0)\n        non_empty_frame_idxs = tf.expand_dims(non_empty_frame_idxs, axis=0)\n        # Make Prediction\n        outputs = self.model({ 'frames': x, 'non_empty_frame_idxs': non_empty_frame_idxs })\n        # Squeeze Output 1x250 -> 250\n        outputs = tf.squeeze(outputs, axis=0)\n\n        # Return a dictionary with the output tensor\n        return {'outputs': outputs}\n\n# Define TF Lite Model\ntflite_keras_model = TFLiteModel(model)\n\n# Sanity Check\ndemo_raw_data = load_relevant_data_subset(train['file_path'].values[5])\nprint(f'demo_raw_data shape: {demo_raw_data.shape}, dtype: {demo_raw_data.dtype}')\ndemo_output = tflite_keras_model(demo_raw_data)[\"outputs\"]\nprint(f'demo_output shape: {demo_output.shape}, dtype: {demo_output.dtype}')\ndemo_prediction = demo_output.numpy().argmax()\nprint(f'demo_prediction: {demo_prediction}, correct: {train.iloc[0][\"sign_ord\"]}')","metadata":{"execution":{"iopub.status.busy":"2023-11-22T09:26:48.88983Z","iopub.execute_input":"2023-11-22T09:26:48.890268Z","iopub.status.idle":"2023-11-22T09:26:51.555062Z","shell.execute_reply.started":"2023-11-22T09:26:48.890223Z","shell.execute_reply":"2023-11-22T09:26:51.55394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create Model Converter\nkeras_model_converter = tf.lite.TFLiteConverter.from_keras_model(tflite_keras_model)\n# Convert Model\ntflite_model = keras_model_converter.convert()\n# Write Model\nwith open('/kaggle/working/model.tflite', 'wb') as f:\n    f.write(tflite_model)\n    \n# Zip Model\n!zip submission.zip /kaggle/working/model.tflite","metadata":{"execution":{"iopub.status.busy":"2023-11-22T09:26:51.556318Z","iopub.execute_input":"2023-11-22T09:26:51.556659Z","iopub.status.idle":"2023-11-22T09:27:35.505233Z","shell.execute_reply.started":"2023-11-22T09:26:51.556627Z","shell.execute_reply":"2023-11-22T09:27:35.50396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Verify TFLite model can be loaded and used for prediction\n!pip install tflite-runtime\nimport tflite_runtime.interpreter as tflite\n\ninterpreter = tflite.Interpreter(\"/kaggle/working/model.tflite\")\nfound_signatures = list(interpreter.get_signature_list().keys())\nprediction_fn = interpreter.get_signature_runner(\"serving_default\")\n\noutput = prediction_fn(inputs=demo_raw_data)\nsign = output['outputs'].argmax()\n\nprint(\"PRED : \", ORD2SIGN.get(sign), f'[{sign}]')\nprint(\"TRUE : \", train.sign.values[0], f'[{train.sign_ord.values[0]}]')","metadata":{"execution":{"iopub.status.busy":"2023-11-22T09:27:35.507152Z","iopub.execute_input":"2023-11-22T09:27:35.507613Z","iopub.status.idle":"2023-11-22T09:27:48.121607Z","shell.execute_reply.started":"2023-11-22T09:27:35.507564Z","shell.execute_reply":"2023-11-22T09:27:48.120244Z"},"trusted":true},"execution_count":null,"outputs":[]}]}