{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":46105,"databundleVersionId":5087314,"sourceType":"competition"},{"sourceId":5315518,"sourceType":"datasetVersion","datasetId":3036481}],"dockerImageVersionId":31260,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport matplotlib.pyplot as plt \nimport matplotlib as mpl\nimport seaborn as sns\n\nfrom tqdm.notebook import tqdm\nfrom sklearn.model_selection import train_test_split, GroupShuffleSplit \n\nimport glob, sys, os, math, gc, sklearn, scipy \nfrom scipy.stats import iqr\n\nprint(f'Tensorflow V{tf.__version__}')\nprint(f'Keras V{tf.keras.__version__}')\nprint(f'Python V{sys.version}') ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-21T14:29:24.786941Z","iopub.execute_input":"2026-01-21T14:29:24.787483Z","iopub.status.idle":"2026-01-21T14:29:40.791693Z","shell.execute_reply.started":"2026-01-21T14:29:24.787452Z","shell.execute_reply":"2026-01-21T14:29:40.790951Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tf.config.run_functions_eagerly(True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T14:29:40.793172Z","iopub.execute_input":"2026-01-21T14:29:40.793621Z","iopub.status.idle":"2026-01-21T14:29:40.797118Z","shell.execute_reply.started":"2026-01-21T14:29:40.793595Z","shell.execute_reply":"2026-01-21T14:29:40.796491Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# MatplotLib Global Settings\nmpl.rcParams.update(mpl.rcParamsDefault)\nmpl.rcParams['xtick.labelsize'] = 16\nmpl.rcParams['ytick.labelsize'] = 16\nmpl.rcParams['axes.labelsize'] = 18\nmpl.rcParams['axes.titlesize'] = 24 ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T14:29:40.798105Z","iopub.execute_input":"2026-01-21T14:29:40.798379Z","iopub.status.idle":"2026-01-21T14:29:40.814789Z","shell.execute_reply.started":"2026-01-21T14:29:40.798350Z","shell.execute_reply":"2026-01-21T14:29:40.813916Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# If True, processing data from scratch\n# If False, loads preprocessed data\nPREPROCESS_DATA = True  # Set to True if first run with aug\nTRAIN_MODEL = True\n# True: use 10% of participants as validation set\n# False: use all data for training -> gives better LB result\nUSE_VAL = True  # Enable for early stopping\n\nN_ROWS = 543\nN_DIMS = 3\nDIM_NAMES = ['x', 'y', 'z']\nSEED = 42\nNUM_CLASSES = 250\nIS_INTERACTIVE = os.environ['KAGGLE_KERNEL_RUN_TYPE'] == 'Interactive'\nVERBOSE = 1 if IS_INTERACTIVE else 2\n\nINPUT_SIZE = 64\n\nBATCH_ALL_SIGNS_N = 4\nBATCH_SIZE = 64 \nN_EPOCHS = 50 # 100\nLR_MAX = 1e-3\nN_WARMUP_EPOCHS = 0\nWD_RATIO = 0.05\nMASK_VAL = 4237","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T14:29:40.816511Z","iopub.execute_input":"2026-01-21T14:29:40.817035Z","iopub.status.idle":"2026-01-21T14:29:40.831536Z","shell.execute_reply.started":"2026-01-21T14:29:40.817000Z","shell.execute_reply":"2026-01-21T14:29:40.830852Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Prints Shape and Dtype For List Of Variables\ndef print_shape_dtype(l, names):\n    for e, n in zip(l, names):\n        print(f'{n} shape: {e.shape}, dtype: {e.dtype}') ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T14:29:40.832483Z","iopub.execute_input":"2026-01-21T14:29:40.832763Z","iopub.status.idle":"2026-01-21T14:29:40.845733Z","shell.execute_reply.started":"2026-01-21T14:29:40.832741Z","shell.execute_reply":"2026-01-21T14:29:40.845101Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Read Training Data\n#if IS_INTERACTIVE or not PREPROCESS_DATA:\n    #train = pd.read_csv('/kaggle/input/asl-signs/train.csv').sample(int(5e3), random_state=SEED)\n#else:\ntrain = pd.read_csv('/kaggle/input/asl-signs/train.csv')\n\nN_SAMPLES = len(train)\nprint(f'N_SAMPLES: {N_SAMPLES}') ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T14:29:40.846735Z","iopub.execute_input":"2026-01-21T14:29:40.847069Z","iopub.status.idle":"2026-01-21T14:29:41.077207Z","shell.execute_reply.started":"2026-01-21T14:29:40.847037Z","shell.execute_reply":"2026-01-21T14:29:41.076406Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Get complete file path to file\ndef get_file_path(path):\n    return f'/kaggle/input/asl-signs/{path}'\n\ntrain['file_path'] = train['path'].apply(get_file_path) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T14:29:41.078242Z","iopub.execute_input":"2026-01-21T14:29:41.078511Z","iopub.status.idle":"2026-01-21T14:29:41.112637Z","shell.execute_reply.started":"2026-01-21T14:29:41.078486Z","shell.execute_reply":"2026-01-21T14:29:41.112036Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Ordinally Encode Sign","metadata":{}},{"cell_type":"code","source":"# Add ordinally Encoded Sign (assign number to each sign name)\ntrain['sign_ord'] = train['sign'].astype('category').cat.codes\n\n# Dictionaries to translate sign to ordinal encoded sign\nSIGN2ORD = train[['sign', 'sign_ord']].set_index('sign').squeeze().to_dict()\nORD2SIGN = train[['sign_ord', 'sign']].set_index('sign_ord').squeeze().to_dict() ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T14:29:41.113554Z","iopub.execute_input":"2026-01-21T14:29:41.113807Z","iopub.status.idle":"2026-01-21T14:29:41.217401Z","shell.execute_reply.started":"2026-01-21T14:29:41.113786Z","shell.execute_reply":"2026-01-21T14:29:41.216685Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"display(train.sample(n=5)) \ndisplay(train.info()) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T14:29:41.218358Z","iopub.execute_input":"2026-01-21T14:29:41.218676Z","iopub.status.idle":"2026-01-21T14:29:41.271169Z","shell.execute_reply.started":"2026-01-21T14:29:41.218651Z","shell.execute_reply":"2026-01-21T14:29:41.270386Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"N = int(1e3) if (IS_INTERACTIVE or not PREPROCESS_DATA) else int(10e3)\nN_UNIQUE_FRAMES = np.zeros(N, dtype=np.uint16)\nN_MISSING_FRAMES = np.zeros(N, dtype=np.uint16)\nMAX_FRAME = np.zeros(N, dtype=np.uint16)\n\nPERCENTILES = [0.01, 0.05, 0.25, 0.50, 0.75, 0.95, 0.99, 0.999]\n\nfor idx, file_path in enumerate(tqdm(train['file_path'].sample(N, random_state=SEED))):\n    df = pd.read_parquet(file_path)\n    N_UNIQUE_FRAMES[idx] = df['frame'].nunique()\n    N_MISSING_FRAMES[idx] = (df['frame'].max() - df['frame'].min()) - df['frame'].nunique() + 1\n    MAX_FRAME[idx] = df['frame'].max() ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T14:29:41.273811Z","iopub.execute_input":"2026-01-21T14:29:41.274127Z","iopub.status.idle":"2026-01-21T14:30:11.015794Z","shell.execute_reply.started":"2026-01-21T14:29:41.274100Z","shell.execute_reply":"2026-01-21T14:30:11.014937Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Number of unique frames in each video\ndisplay(pd.Series(N_UNIQUE_FRAMES).describe(percentiles=PERCENTILES).to_frame('N_UNIQUE_FRAMES'))\n\nplt.figure(figsize=(15,8))\nplt.title('Number of Unique Frames', size=24)\npd.Series(N_UNIQUE_FRAMES).plot(kind='hist', bins=128)\nplt.grid()\nxlim = math.ceil(plt.xlim()[1])\nplt.xlim(0, xlim)\nplt.xticks(np.arange(0, xlim+25, 25))\nplt.show() ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T14:30:11.016767Z","iopub.execute_input":"2026-01-21T14:30:11.017072Z","iopub.status.idle":"2026-01-21T14:30:11.429910Z","shell.execute_reply.started":"2026-01-21T14:30:11.017047Z","shell.execute_reply":"2026-01-21T14:30:11.429278Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Number of missing frames, consecutive frames with missing intermediate frame, i.e. 1,2,4,5 but 3 is missing\ndisplay(pd.Series(N_MISSING_FRAMES).describe(percentiles=PERCENTILES).to_frame('N_MISSING_FRAMES'))\n\nplt.figure(figsize=(15,8))\nplt.title('Number of Missing Frames', size=24)\npd.Series(N_MISSING_FRAMES).plot(kind='hist', bins=128)\nplt.grid()\nplt.xlim(0, math.ceil(plt.xlim()[1]))\nplt.show() ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T14:30:11.430818Z","iopub.execute_input":"2026-01-21T14:30:11.431346Z","iopub.status.idle":"2026-01-21T14:30:11.700696Z","shell.execute_reply.started":"2026-01-21T14:30:11.431321Z","shell.execute_reply":"2026-01-21T14:30:11.700037Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Maximum frame number\ndisplay(pd.Series(MAX_FRAME).describe(percentiles=PERCENTILES).to_frame('MAX_FRAME'))\n\nplt.figure(figsize=(15,8))\nplt.title('Maximum Frames Index', size=24)\npd.Series(MAX_FRAME).plot(kind='hist', bins=128)\nplt.grid() \nplt.xlim(0, math.ceil(plt.xlim()[1]))\nplt.show() ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T14:30:11.701613Z","iopub.execute_input":"2026-01-21T14:30:11.702106Z","iopub.status.idle":"2026-01-21T14:30:11.974168Z","shell.execute_reply.started":"2026-01-21T14:30:11.702079Z","shell.execute_reply":"2026-01-21T14:30:11.973415Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Landmark Indices for Lips, Hands, and Pose","metadata":{}},{"cell_type":"code","source":"USE_TYPES = ['left_hand', 'pose', 'right_hand']\nSTART_IDX = 468\nLIPS_IDXS0 = np.array([\n        61, 185, 40, 39, 37, 0, 267, 269, 270, 409,\n        291, 146, 91, 181, 84, 17, 314, 405, 321, 375,\n        78, 191, 80, 81, 82, 13, 312, 311, 310, 415,\n        95, 88, 178, 87, 14, 317, 402, 318, 324, 308,\n    ])\n# Landmark indices in original data\nLEFT_HAND_IDXS0 = np.arange(468,489)\nRIGHT_HAND_IDXS0 = np.arange(522,543)\nLEFT_POSE_IDXS0 = np.array([502, 504, 506, 508, 510])\nRIGHT_POSE_IDXS0 = np.array([503, 505, 507, 509, 511])\nLANDMARK_IDXS_LEFT_DOMINANT0 = np.concatenate((LIPS_IDXS0, LEFT_HAND_IDXS0, LEFT_POSE_IDXS0))\nLANDMARK_IDXS_RIGHT_DOMINANT0 = np.concatenate((LIPS_IDXS0, RIGHT_HAND_IDXS0, RIGHT_POSE_IDXS0))\nHAND_IDXS0 = np.concatenate((LEFT_HAND_IDXS0, RIGHT_HAND_IDXS0), axis=0)\nN_COLS = LANDMARK_IDXS_LEFT_DOMINANT0.size\n# Landmark indices in processed data\nLIPS_IDXS = np.argwhere(np.isin(LANDMARK_IDXS_LEFT_DOMINANT0, LIPS_IDXS0)).squeeze()\nLEFT_HAND_IDXS = np.argwhere(np.isin(LANDMARK_IDXS_LEFT_DOMINANT0, LEFT_HAND_IDXS0)).squeeze()\nRIGHT_HAND_IDXS = np.argwhere(np.isin(LANDMARK_IDXS_LEFT_DOMINANT0, RIGHT_HAND_IDXS0)).squeeze()\nHAND_IDXS = np.argwhere(np.isin(LANDMARK_IDXS_LEFT_DOMINANT0, HAND_IDXS0)).squeeze()\nPOSE_IDXS = np.argwhere(np.isin(LANDMARK_IDXS_LEFT_DOMINANT0, LEFT_POSE_IDXS0)).squeeze()\n\nprint(f'# HAND_IDXS: {len(HAND_IDXS)}, N_COLS: {N_COLS}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T14:30:11.975128Z","iopub.execute_input":"2026-01-21T14:30:11.975404Z","iopub.status.idle":"2026-01-21T14:30:11.984320Z","shell.execute_reply.started":"2026-01-21T14:30:11.975374Z","shell.execute_reply":"2026-01-21T14:30:11.983504Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"LIPS_START = 0\nLEFT_HAND_START = LIPS_IDXS.size\nRIGHT_HAND_START = LEFT_HAND_START + LEFT_HAND_IDXS.size\nPOSE_START = RIGHT_HAND_START + RIGHT_HAND_IDXS.size\n\nprint(f'LIPS_START: {LIPS_START}, LEFT_HAND_START: {LEFT_HAND_START}, RIGHT_HAND_START: {RIGHT_HAND_START}, POSE_START: {POSE_START}') ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T14:30:11.985256Z","iopub.execute_input":"2026-01-21T14:30:11.985491Z","iopub.status.idle":"2026-01-21T14:30:11.997662Z","shell.execute_reply.started":"2026-01-21T14:30:11.985471Z","shell.execute_reply":"2026-01-21T14:30:11.996931Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Process data Tensorflow","metadata":{}},{"cell_type":"code","source":"ROWS_PER_FRAME = 543  # number of landmarks per frame\n\ndef load_relevant_data_subset(pq_path):\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T14:30:11.998690Z","iopub.execute_input":"2026-01-21T14:30:11.999162Z","iopub.status.idle":"2026-01-21T14:30:12.011991Z","shell.execute_reply.started":"2026-01-21T14:30:11.999139Z","shell.execute_reply":"2026-01-21T14:30:12.011167Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class PreprocessLayer(tf.keras.layers.Layer):\n    def __init__(self):\n        super(PreprocessLayer, self).__init__()\n        normalisation_correction = tf.constant([\n                    # Add 0.50 to left hand (original right hand) and substract 0.50 of right hand (original left hand)\n                    [0] * len(LIPS_IDXS) + [0.50] * len(LEFT_HAND_IDXS) + [0.50] * len(POSE_IDXS),\n                    # Y coordinates stay intact\n                    [0] * len(LANDMARK_IDXS_LEFT_DOMINANT0),\n                    # Z coordinates stay intact\n                    [0] * len(LANDMARK_IDXS_LEFT_DOMINANT0),\n                ],\n                dtype=tf.float32,\n            )\n        self.normalisation_correction = tf.transpose(normalisation_correction, [1,0])\n        \n    def pad_edge(self, t, repeats, side):\n        if side == 'LEFT':\n            return tf.concat((tf.repeat(t[:1], repeats=repeats, axis=0), t), axis=0)\n        elif side == 'RIGHT':\n            return tf.concat((t, tf.repeat(t[-1:], repeats=repeats, axis=0)), axis=0)\n    \n    @tf.function(\n        input_signature=(tf.TensorSpec(shape=[None,N_ROWS,N_DIMS], dtype=tf.float32),),\n    )\n    def call(self, data0):\n        # Number of Frames in Video\n        N_FRAMES0 = tf.shape(data0)[0]\n        \n        # Find dominant hand by comparing summed absolute coordinates\n        left_hand_sum = tf.math.reduce_sum(tf.where(tf.math.is_nan(tf.gather(data0, LEFT_HAND_IDXS0, axis=1)), 0, 1))\n        right_hand_sum = tf.math.reduce_sum(tf.where(tf.math.is_nan(tf.gather(data0, RIGHT_HAND_IDXS0, axis=1)), 0, 1))\n        left_dominant = left_hand_sum >= right_hand_sum\n        \n        # Count non-NaN Hand values in each frame for the dominant hand\n        if left_dominant:\n            frames_hands_non_nan_sum = tf.math.reduce_sum(\n                    tf.where(tf.math.is_nan(tf.gather(data0, LEFT_HAND_IDXS0, axis=1)), 0, 1),\n                    axis=[1, 2],\n                )\n        else:\n            frames_hands_non_nan_sum = tf.math.reduce_sum(\n                    tf.where(tf.math.is_nan(tf.gather(data0, RIGHT_HAND_IDXS0, axis=1)), 0, 1),\n                    axis=[1, 2],\n                )\n        \n        # Find frames indices with coordinates of the dominant hand\n        non_empty_frames_idxs = tf.where(frames_hands_non_nan_sum > 0)\n        non_empty_frames_idxs = tf.squeeze(non_empty_frames_idxs, axis=1)\n        # Filter frames\n        data = tf.gather(data0, non_empty_frames_idxs, axis=0)\n        \n        # Cast Indices in float32 to be compatible with Tensorflow Lite\n        non_empty_frames_idxs = tf.cast(non_empty_frames_idxs, tf.float32)\n        # Normalize to start with 0\n        non_empty_frames_idxs -= tf.reduce_min(non_empty_frames_idxs)\n        \n        # Number of Frames in Filtered Video\n        N_FRAMES = tf.shape(data)[0]\n        \n        # Gather Relevant Landmark Columns\n        if left_dominant:\n            data = tf.gather(data, LANDMARK_IDXS_LEFT_DOMINANT0, axis=1)\n        else:\n            data = tf.gather(data, LANDMARK_IDXS_RIGHT_DOMINANT0, axis=1)\n            data = (\n                    self.normalisation_correction + (\n                        (data - self.normalisation_correction) * tf.where(self.normalisation_correction != 0, -1.0, 1.0))\n                )\n        \n        # Video fits in INPUT_SIZE\n        if N_FRAMES < INPUT_SIZE:\n            # Pad With -1 to indicate padding\n            non_empty_frames_idxs = tf.pad(non_empty_frames_idxs, [[0, INPUT_SIZE-N_FRAMES]], constant_values=-1)\n            # Pad Data With Zeros\n            data = tf.pad(data, [[0, INPUT_SIZE-N_FRAMES], [0,0], [0,0]], constant_values=0)\n            # Fill NaN Values With 0\n            data = tf.where(tf.math.is_nan(data), 0.0, data)\n            return data, non_empty_frames_idxs\n        # Video needs to be downsampled to INPUT_SIZE\n        else:\n            # Repeat\n            if N_FRAMES < INPUT_SIZE**2:\n                repeats = tf.math.floordiv(INPUT_SIZE * INPUT_SIZE, N_FRAMES0)\n                data = tf.repeat(data, repeats=repeats, axis=0)\n                non_empty_frames_idxs = tf.repeat(non_empty_frames_idxs, repeats=repeats, axis=0)\n\n            # Pad To Multiple Of Input Size\n            pool_size = tf.math.floordiv(len(data), INPUT_SIZE)\n            if tf.math.mod(len(data), INPUT_SIZE) > 0:\n                pool_size += 1\n\n            if pool_size == 1:\n                pad_size = (pool_size * INPUT_SIZE) - len(data)\n            else:\n                pad_size = (pool_size * INPUT_SIZE) % len(data)\n\n            # Pad Start/End with Start/End value\n            pad_left = tf.math.floordiv(pad_size, 2) + tf.math.floordiv(INPUT_SIZE, 2)\n            pad_right = tf.math.floordiv(pad_size, 2) + tf.math.floordiv(INPUT_SIZE, 2)\n            if tf.math.mod(pad_size, 2) > 0:\n                pad_right += 1\n\n            # Pad By Concatenating Left/Right Edge Values\n            data = self.pad_edge(data, pad_left, 'LEFT')\n            data = self.pad_edge(data, pad_right, 'RIGHT')\n\n            # Pad Non Empty Frame Indices\n            non_empty_frames_idxs = self.pad_edge(non_empty_frames_idxs, pad_left, 'LEFT')\n            non_empty_frames_idxs = self.pad_edge(non_empty_frames_idxs, pad_right, 'RIGHT')\n\n            # Reshape to Mean Pool\n            data = tf.reshape(data, [INPUT_SIZE, -1, N_COLS, N_DIMS])\n            non_empty_frames_idxs = tf.reshape(non_empty_frames_idxs, [INPUT_SIZE, -1])\n\n            # Mean Pool\n            data = tf.experimental.numpy.nanmean(data, axis=1)\n            non_empty_frames_idxs = tf.experimental.numpy.nanmean(non_empty_frames_idxs, axis=1)\n\n            # Fill NaN Values With 0\n            data = tf.where(tf.math.is_nan(data), 0.0, data)\n            \n            return data, non_empty_frames_idxs\n    \npreprocess_layer = PreprocessLayer() ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T14:30:12.013054Z","iopub.execute_input":"2026-01-21T14:30:12.013327Z","iopub.status.idle":"2026-01-21T14:30:12.842287Z","shell.execute_reply.started":"2026-01-21T14:30:12.013295Z","shell.execute_reply":"2026-01-21T14:30:12.841578Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\n    face: 0:468\n    left_hand: 468:489\n    pose: 489:522\n    right_hand: 522:544\n        \n\"\"\"\ndef get_data(file_path):\n    # Load Raw Data\n    data = load_relevant_data_subset(file_path)\n    # Process Data Using Tensorflow\n    data = preprocess_layer(data)\n    \n    return data ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T14:30:12.843159Z","iopub.execute_input":"2026-01-21T14:30:12.843442Z","iopub.status.idle":"2026-01-21T14:30:12.847615Z","shell.execute_reply.started":"2026-01-21T14:30:12.843418Z","shell.execute_reply":"2026-01-21T14:30:12.847003Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def preprocess_data():\n    print('Preprocessing Data with 3-Way Split (Train/Val/Test)...')\n    \n    # First Split: 90% Train, 10% Temp\n    from sklearn.model_selection import GroupShuffleSplit\n    splitter_1 = GroupShuffleSplit(test_size=0.10, n_splits=2, random_state=SEED)\n    PARTICIPANT_IDS = train['participant_id'].values\n    train_idxs, temp_idxs = next(splitter_1.split(train, train['sign'], groups=PARTICIPANT_IDS))\n    \n    train_df = train.iloc[train_idxs].reset_index(drop=True)\n    temp_df = train.iloc[temp_idxs].reset_index(drop=True)\n    \n    # Second Split: Split Temp 50/50 into Val/Test (resulting in 5% Val, 5% Test of total)\n    splitter_2 = GroupShuffleSplit(test_size=0.50, n_splits=2, random_state=SEED)\n    TEMP_PARTICIPANT_IDS = temp_df['participant_id'].values\n    val_idxs, test_idxs = next(splitter_2.split(temp_df, temp_df['sign'], groups=TEMP_PARTICIPANT_IDS))\n    \n    val_df = temp_df.iloc[val_idxs].reset_index(drop=True)\n    test_df = temp_df.iloc[test_idxs].reset_index(drop=True)\n    \n    print(f'Train Samples: {len(train_df)}, Val Samples: {len(val_df)}, Test Samples: {len(test_df)}')\n    \n    # Determine Shape from First Sample\n    print('Determining data shape...')\n    first_path = train_df.iloc[0]['file_path']\n    first_data, first_non_empty = get_data(first_path)\n    first_data = first_data.numpy() if hasattr(first_data, 'numpy') else np.array(first_data)\n    row_dim = INPUT_SIZE\n    col_dim = first_data.shape[1]\n    dim_dim = N_DIMS\n    \n    print(f'Target Shape: (N, {row_dim}, {col_dim}, {dim_dim})')\n    \n    # Create Memmaps\n    import os\n    from numpy.lib.format import open_memmap\n    \n    # Define helper to create memmaps\n    def create_mmap(name, shape, dtype):\n        if os.path.exists(name): os.remove(name)\n        return open_memmap(name, mode='w+', dtype=dtype, shape=shape)\n    \n    X_train = create_mmap('X_train.npy', (len(train_df), row_dim, col_dim, dim_dim), np.float32)\n    y_train = create_mmap('y_train.npy', (len(train_df),), np.int32)\n    ne_train = create_mmap('NON_EMPTY_FRAME_IDXS_TRAIN.npy', (len(train_df), row_dim), np.float32)\n    \n    X_val = create_mmap('X_val.npy', (len(val_df), row_dim, col_dim, dim_dim), np.float32)\n    y_val = create_mmap('y_val.npy', (len(val_df),), np.int32)\n    ne_val = create_mmap('NON_EMPTY_FRAME_IDXS_VAL.npy', (len(val_df), row_dim), np.float32)\n    \n    X_test = create_mmap('X_test.npy', (len(test_df), row_dim, col_dim, dim_dim), np.float32)\n    y_test = create_mmap('y_test.npy', (len(test_df),), np.int32)\n    ne_test = create_mmap('NON_EMPTY_FRAME_IDXS_TEST.npy', (len(test_df), row_dim), np.float32)\n    \n    # Processing Helper\n    def process_and_save(df, X_mmap, y_mmap, ne_mmap, desc):\n        for i, row in tqdm(df.iterrows(), total=len(df), desc=desc):\n            data, non_empty_from_preprocess = get_data(row['file_path'])\n            data = data.numpy() if hasattr(data, 'numpy') else np.array(data)\n            if len(data) < INPUT_SIZE:\n                data = np.pad(data, ((0, INPUT_SIZE - len(data)), (0, 0), (0, 0)), mode='constant', constant_values=0)\n                non_empty = np.array([1]*len(data) + [0]*(INPUT_SIZE-len(data)), dtype=np.float32)\n            else:\n                idxs = np.floor(np.linspace(0, len(data)-1, INPUT_SIZE)).astype(int)\n                data = data[idxs]\n                non_empty = np.ones(INPUT_SIZE, dtype=np.float32)\n            \n            if np.isnan(data).sum() > 0:\n                data = np.nan_to_num(data)\n\n            X_mmap[i] = data\n            y_mmap[i] = row['sign_ord']\n            ne_mmap[i] = non_empty\n            \n            if i % 1000 == 0: X_mmap.flush()\n        X_mmap.flush(); y_mmap.flush(); ne_mmap.flush()\n    \n    # Run Processing\n    process_and_save(train_df, X_train, y_train, ne_train, 'Processing Train')\n    process_and_save(val_df, X_val, y_val, ne_val, 'Processing Val')\n    process_and_save(test_df, X_test, y_test, ne_test, 'Processing Test')\n    \n    print('Preprocessing Complete. Data saved to disk.')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T14:30:12.848385Z","iopub.execute_input":"2026-01-21T14:30:12.848707Z","iopub.status.idle":"2026-01-21T14:30:12.863505Z","shell.execute_reply.started":"2026-01-21T14:30:12.848687Z","shell.execute_reply":"2026-01-21T14:30:12.862762Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Preprocess All Data From Scratch\nif PREPROCESS_DATA:\n    preprocess_data()\n    ROOT_DIR = '.'\nelse:\n    ROOT_DIR = '/kaggle/input/gislr-dataset-public'\n    \n# Load Data\nif USE_VAL:\n    # Load Train\n    X_train = np.load(f'{ROOT_DIR}/X_train.npy', mmap_mode='r')\n    y_train = np.load(f'{ROOT_DIR}/y_train.npy', mmap_mode='r')\n    NON_EMPTY_FRAME_IDXS_TRAIN = np.load(f'{ROOT_DIR}/NON_EMPTY_FRAME_IDXS_TRAIN.npy', mmap_mode='r')\n    # Load Val\n    X_val = np.load(f'{ROOT_DIR}/X_val.npy', mmap_mode='r') \n    X_test = np.load(f'{ROOT_DIR}/X_test.npy', mmap_mode='r')\n    y_test = np.load(f'{ROOT_DIR}/y_test.npy', mmap_mode='r')\n    NON_EMPTY_FRAME_IDXS_TEST = np.load(f'{ROOT_DIR}/NON_EMPTY_FRAME_IDXS_TEST.npy', mmap_mode='r')\n    y_val = np.load(f'{ROOT_DIR}/y_val.npy', mmap_mode='r')\n    NON_EMPTY_FRAME_IDXS_VAL = np.load(f'{ROOT_DIR}/NON_EMPTY_FRAME_IDXS_VAL.npy', mmap_mode='r')\n    # Define validation Data\n    y_val_oh = tf.one_hot(y_val, NUM_CLASSES)\n    validation_data = ({ 'frames': X_val, 'non_empty_frame_idxs': NON_EMPTY_FRAME_IDXS_VAL }, y_val_oh)\nelse:\n    X_train = np.load(f'{ROOT_DIR}/X.npy', mmap_mode='r')\n    y_train = np.load(f'{ROOT_DIR}/y.npy', mmap_mode='r')\n    NON_EMPTY_FRAME_IDXS_TRAIN = np.load(f'{ROOT_DIR}/NON_EMPTY_FRAME_IDXS.npy', mmap_mode='r')\n    validation_data = None\n\n# Train \nprint_shape_dtype([X_train, y_train, NON_EMPTY_FRAME_IDXS_TRAIN], ['X_train', 'y_train', 'NON_EMPTY_FRAME_IDXS_TRAIN'])\n# Val\nif USE_VAL:\n    print_shape_dtype([X_val, y_val, NON_EMPTY_FRAME_IDXS_VAL], ['X_val', 'y_val', 'NON_EMPTY_FRAME_IDXS_VAL'])\n# Sanity Check\nprint(f'# NaN Values X_train: {np.isnan(X_train).sum()}') ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T14:30:12.864493Z","iopub.execute_input":"2026-01-21T14:30:12.865007Z","iopub.status.idle":"2026-01-21T15:20:04.916130Z","shell.execute_reply.started":"2026-01-21T14:30:12.864977Z","shell.execute_reply":"2026-01-21T15:20:04.915294Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Class Count\ndisplay(pd.Series(y_train).value_counts().to_frame('Class Count').iloc[[0,1,2,3,4, -5,-4,-3,-2,-1]]) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:20:04.917282Z","iopub.execute_input":"2026-01-21T15:20:04.917511Z","iopub.status.idle":"2026-01-21T15:20:04.936616Z","shell.execute_reply.started":"2026-01-21T15:20:04.917489Z","shell.execute_reply":"2026-01-21T15:20:04.935674Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Number of Frames","metadata":{}},{"cell_type":"code","source":"# Vast majority of samples fits has less than 32 non empty frames\nN_EMPTY_FRAMES = (NON_EMPTY_FRAME_IDXS_TRAIN != -1).sum(axis=1) \nN_EMPTY_FRAMES_WATERFALL = []\nfor n in tqdm(range(1,INPUT_SIZE+1)):\n    N_EMPTY_FRAMES_WATERFALL.append(sum(N_EMPTY_FRAMES >= n) / len(NON_EMPTY_FRAME_IDXS_TRAIN) * 100)\n\nplt.figure(figsize=(18,10))\nplt.title('Waterfall Plot For Number Of Non Empty Frames')\npd.Series(N_EMPTY_FRAMES_WATERFALL).plot(kind='bar')\nplt.grid(axis='y')\nplt.xticks(np.arange(INPUT_SIZE), np.arange(1, INPUT_SIZE+1))\nplt.xlabel('Number of Non Empty Frames', size=16)\nplt.yticks(np.arange(0, 100+10, 10))\nplt.ylim(0, 100)\nplt.ylabel('Percentage of Samples With At Least N Non Empty Frames', size=16)\nplt.show() ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:20:04.938443Z","iopub.execute_input":"2026-01-21T15:20:04.938786Z","iopub.status.idle":"2026-01-21T15:20:06.015291Z","shell.execute_reply.started":"2026-01-21T15:20:04.938761Z","shell.execute_reply":"2026-01-21T15:20:06.014596Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Percentage of Frames Filled","metadata":{}},{"cell_type":"code","source":"# Percentage of frames filled, this is the maximum fill percentage of each landmark\nP_DATA_FILLED = (NON_EMPTY_FRAME_IDXS_TRAIN != -1).sum() / NON_EMPTY_FRAME_IDXS_TRAIN.size * 100\nprint(f'P_DATA_FILLED: {P_DATA_FILLED:.2f}%') ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:20:06.016340Z","iopub.execute_input":"2026-01-21T15:20:06.017042Z","iopub.status.idle":"2026-01-21T15:20:06.026021Z","shell.execute_reply.started":"2026-01-21T15:20:06.017014Z","shell.execute_reply":"2026-01-21T15:20:06.025442Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Stats - Lips","metadata":{}},{"cell_type":"code","source":"# Percentage of Lips Measurements\nP_LEFT_LIPS_MEASUREMENTS = (X_train[:,:,LIPS_IDXS] != 0).sum() / X_train[:,:,LIPS_IDXS].size / P_DATA_FILLED * 1e4\nprint(f'P_LEFT_LIPS_MEASUREMENTS: {P_LEFT_LIPS_MEASUREMENTS:.2f}%') ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:20:06.027035Z","iopub.execute_input":"2026-01-21T15:20:06.027471Z","iopub.status.idle":"2026-01-21T15:20:22.270999Z","shell.execute_reply.started":"2026-01-21T15:20:06.027448Z","shell.execute_reply":"2026-01-21T15:20:22.270312Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_lips_median_iqr():\n    # LIPS\n    LIPS_MEDIAN_X = np.zeros([LIPS_IDXS.size], dtype=np.float32)\n    LIPS_MEDIAN_Y = np.zeros([LIPS_IDXS.size], dtype=np.float32)\n    LIPS_IQR_X = np.zeros([LIPS_IDXS.size], dtype=np.float32)\n    LIPS_IQR_Y = np.zeros([LIPS_IDXS.size], dtype=np.float32)\n\n    fig, axes = plt.subplots(3, 1, figsize=(15, N_DIMS*6))\n\n    for col, ll in enumerate(tqdm( np.transpose(X_train[:,:,LIPS_IDXS], [2,3,0,1]).reshape([LIPS_IDXS.size, N_DIMS, -1]) )):\n        for dim, l in enumerate(ll):\n            v = l[np.nonzero(l)]\n            if dim == 0: # X\n                LIPS_MEDIAN_X[col] = np.median(v)\n                LIPS_IQR_X[col] = iqr(v)\n            if dim == 1: # Y\n                LIPS_MEDIAN_Y[col] = np.median(v)\n                LIPS_IQR_Y[col] = iqr(v)\n\n            axes[dim].boxplot(v, notch=False, showfliers=False, positions=[col], whis=[5,95])\n\n    for ax, dim_name in zip(axes, DIM_NAMES):\n        ax.set_title(f'Lips {dim_name.upper()} Dimension', size=24)\n        ax.tick_params(axis='x', labelsize=8)\n        ax.grid(axis='y')\n\n    plt.subplots_adjust(hspace=0.50)\n    plt.show()\n\n    LIPS_MEDIAN = np.array([LIPS_MEDIAN_X, LIPS_MEDIAN_Y]).T\n    LIPS_IQR = np.array([LIPS_IQR_X, LIPS_IQR_Y]).T\n    \n    return LIPS_MEDIAN, LIPS_IQR\n\nLIPS_MEDIAN, LIPS_IQR = get_lips_median_iqr() ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:20:22.272087Z","iopub.execute_input":"2026-01-21T15:20:22.272380Z","iopub.status.idle":"2026-01-21T15:20:42.341394Z","shell.execute_reply.started":"2026-01-21T15:20:22.272343Z","shell.execute_reply":"2026-01-21T15:20:42.340701Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Stats - Hands","metadata":{}},{"cell_type":"code","source":"# Verify Normalised to Left Hand Dominant\nP_LEFT_HAND_MEASUREMENTS = (X_train[:,:,LEFT_HAND_IDXS] != 0).sum() / X_train[:,:,LEFT_HAND_IDXS].size / P_DATA_FILLED * 1e4\n# P_RIGHT_HAND_MEASUREMENTS = (X_train[:,:,RIGHT_HAND_IDXS] != 0).sum() / X_train[:,:,RIGHT_HAND_IDXS].size / P_DATA_FILLED * 1e4\nprint(f'P_LEFT_HAND_MEASUREMENTS: {P_LEFT_HAND_MEASUREMENTS:.2f}%') ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:20:42.342274Z","iopub.execute_input":"2026-01-21T15:20:42.342578Z","iopub.status.idle":"2026-01-21T15:20:50.864950Z","shell.execute_reply.started":"2026-01-21T15:20:42.342556Z","shell.execute_reply":"2026-01-21T15:20:50.864243Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_left_right_hand_median_iqr():\n    # LEFT HAND\n    LEFT_HANDS_MEDIAN_X = np.zeros([LEFT_HAND_IDXS.size], dtype=np.float32)\n    LEFT_HANDS_MEDIAN_Y = np.zeros([LEFT_HAND_IDXS.size], dtype=np.float32)\n    LEFT_HANDS_IQR_X = np.zeros([LEFT_HAND_IDXS.size], dtype=np.float32)\n    LEFT_HANDS_IQR_Y = np.zeros([LEFT_HAND_IDXS.size], dtype=np.float32)\n\n    fig, axes = plt.subplots(3, 1, figsize=(15, N_DIMS*6))\n\n    for col, ll in enumerate(tqdm( np.transpose(X_train[:,:,LEFT_HAND_IDXS], [2,3,0,1]).reshape([LEFT_HAND_IDXS.size, N_DIMS, -1]) )):\n        for dim, l in enumerate(ll):\n            v = l[np.nonzero(l)]\n            if dim == 0: # X\n                LEFT_HANDS_MEDIAN_X[col] = np.median(v)\n                LEFT_HANDS_IQR_X[col] = iqr(v)\n            if dim == 1: # Y\n                LEFT_HANDS_MEDIAN_Y[col] = np.median(v)\n                LEFT_HANDS_IQR_Y[col] = iqr(v)\n            # Plot\n            axes[dim].boxplot(v, notch=False, showfliers=False, positions=[col], whis=[5,95])\n\n    for ax, dim_name in zip(axes, DIM_NAMES):\n        ax.set_title(f'Hands {dim_name.upper()} Dimension', size=24)\n        ax.tick_params(axis='x', labelsize=8)\n        ax.grid(axis='y')\n\n    plt.subplots_adjust(hspace=0.50)\n    plt.show()\n\n    LEFT_HANDS_MEDIAN = np.array([LEFT_HANDS_MEDIAN_X, LEFT_HANDS_MEDIAN_Y]).T\n    LEFT_HANDS_IQR = np.array([LEFT_HANDS_IQR_X, LEFT_HANDS_IQR_Y]).T\n    \n    return LEFT_HANDS_MEDIAN, LEFT_HANDS_IQR\n\nLEFT_HANDS_MEDIAN, LEFT_HANDS_IQR = get_left_right_hand_median_iqr() ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:20:50.865953Z","iopub.execute_input":"2026-01-21T15:20:50.866253Z","iopub.status.idle":"2026-01-21T15:21:02.506422Z","shell.execute_reply.started":"2026-01-21T15:20:50.866230Z","shell.execute_reply":"2026-01-21T15:21:02.505781Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Stats - Pose","metadata":{}},{"cell_type":"code","source":"# Percentage of Lips Measurements\nP_POSE_MEASUREMENTS = (X_train[:,:,POSE_IDXS] != 0).sum() / X_train[:,:,POSE_IDXS].size / P_DATA_FILLED * 1e4\nprint(f'P_POSE_MEASUREMENTS: {P_POSE_MEASUREMENTS:.2f}%') ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:21:02.510487Z","iopub.execute_input":"2026-01-21T15:21:02.510743Z","iopub.status.idle":"2026-01-21T15:21:04.531686Z","shell.execute_reply.started":"2026-01-21T15:21:02.510720Z","shell.execute_reply":"2026-01-21T15:21:04.530816Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_pose_median_iqr():\n    # POSE\n    POSE_MEDIAN_X = np.zeros([POSE_IDXS.size], dtype=np.float32)\n    POSE_MEDIAN_Y = np.zeros([POSE_IDXS.size], dtype=np.float32)\n    POSE_IQR_X = np.zeros([POSE_IDXS.size], dtype=np.float32)\n    POSE_IQR_Y = np.zeros([POSE_IDXS.size], dtype=np.float32)\n\n    fig, axes = plt.subplots(3, 1, figsize=(15, N_DIMS*6))\n\n    for col, ll in enumerate(tqdm( np.transpose(X_train[:,:,POSE_IDXS], [2,3,0,1]).reshape([POSE_IDXS.size, N_DIMS, -1]) )):\n        for dim, l in enumerate(ll):\n            v = l[np.nonzero(l)]\n            if dim == 0: # X\n                POSE_MEDIAN_X[col] = np.median(v)\n                POSE_IQR_X[col] = iqr(v)\n            if dim == 1: # Y\n                POSE_MEDIAN_Y[col] = np.median(v)\n                POSE_IQR_Y[col] = iqr(v)\n\n            axes[dim].boxplot(v, notch=False, showfliers=False, positions=[col], whis=[5,95])\n\n    for ax, dim_name in zip(axes, DIM_NAMES):\n        ax.set_title(f'Pose {dim_name.upper()} Dimension', size=24)\n        ax.tick_params(axis='x', labelsize=8)\n        ax.grid(axis='y')\n\n    plt.subplots_adjust(hspace=0.50)\n    plt.show()\n\n    POSE_MEDIAN = np.array([POSE_MEDIAN_X, POSE_MEDIAN_Y]).T\n    POSE_IQR = np.array([POSE_IQR_X, POSE_IQR_Y]).T\n    \n    return POSE_MEDIAN, POSE_IQR\n\nPOSE_MEDIAN, POSE_IQR = get_pose_median_iqr() ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:21:04.532657Z","iopub.execute_input":"2026-01-21T15:21:04.532912Z","iopub.status.idle":"2026-01-21T15:21:07.485216Z","shell.execute_reply.started":"2026-01-21T15:21:04.532862Z","shell.execute_reply":"2026-01-21T15:21:07.484471Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Samples","metadata":{}},{"cell_type":"code","source":"# Custom sampler to get a batch containing N times all signs\ndef get_train_batch_all_signs(X, y, NON_EMPTY_FRAME_IDXS, n=BATCH_ALL_SIGNS_N, mixup_alpha=0.4):\n    # Dictionary mapping ordinally encoded sign to corresponding sample indices\n    CLASS2IDXS = {}\n    for i in range(NUM_CLASSES):\n        CLASS2IDXS[i] = np.argwhere(y == i).squeeze().astype(np.int32)\n            \n    while True:\n        # Arrays to store batch\n        X_batch = np.zeros([NUM_CLASSES*n, INPUT_SIZE, N_COLS, N_DIMS], dtype=np.float32)\n        y_batch = np.zeros([NUM_CLASSES*n, NUM_CLASSES], dtype=np.float32)\n        non_empty_frame_idxs_batch = np.zeros([NUM_CLASSES*n, INPUT_SIZE], dtype=np.float32)\n\n        # Fill batch arrays\n        for i in range(NUM_CLASSES):\n            idxs = np.random.choice(CLASS2IDXS[i], n)\n            X_batch[i*n:(i+1)*n] = X[idxs]\n            non_empty_frame_idxs_batch[i*n:(i+1)*n] = NON_EMPTY_FRAME_IDXS[idxs]\n            y_batch[i*n:(i+1)*n, i] = 1.0\n        \n        # Add augmentation (reduced aggressiveness)\n        for b in range(X_batch.shape[0]):\n            # Geometric Augmentation\n            def augment_points(points):\n                if len(points) == 0:\n                    return points\n                centroid = np.mean(points, axis=0)\n                points_centered = points - centroid\n                angle = np.random.uniform(-15, 15) * np.pi / 180\n                rot_matrix = np.array([[np.cos(angle), -np.sin(angle)], \n                                       [np.sin(angle), np.cos(angle)]])\n                points_rot = points_centered @ rot_matrix.T\n                scale = np.random.uniform(0.9, 1.1)  # Reduced from 0.85-1.15\n                points_scaled = points_rot * scale\n                trans = np.random.uniform(-0.05, 0.05, 2)  # Reduced from -0.1 to 0.1\n                points_trans = points_scaled + trans\n                points_aug = points_trans + centroid\n                points_aug += np.random.normal(0, 0.01, points_aug.shape)  # Reduced noise\n                return points_aug\n            \n            # Process each landmark group\n            for start_idx, num_points, group_idxs in [\n                (LIPS_START, LIPS_IDXS.size, LIPS_IDXS),\n                (LEFT_HAND_START, LEFT_HAND_IDXS.size, LEFT_HAND_IDXS),\n                (POSE_START, POSE_IDXS.size, POSE_IDXS)\n            ]:\n                group_xy = X_batch[b, :, group_idxs, :2]\n                mask = np.any(group_xy != 0, axis=-1)\n                if np.any(mask):\n                    frame_idxs, point_idxs = np.where(mask)\n                    non_zero_coords = group_xy[frame_idxs, point_idxs, :]\n                    augmented_coords = augment_points(non_zero_coords)\n                    group_xy[frame_idxs, point_idxs, :] = augmented_coords\n                X_batch[b, :, group_idxs, :2] = group_xy\n            \n            # Feature Masking - REDUCED from 30% to 15%\n            if np.random.rand() < 0.15:\n                drop_mask = np.random.rand(N_COLS) < np.random.uniform(0.05, 0.15)\n                X_batch[b, :, drop_mask, :] = 0\n\n            # Time Masking - REDUCED from 50% to 30%\n            if np.random.rand() < 0.3:\n                len_mask = np.random.randint(2, 8)  # Reduced from 2-12\n                start_mask = np.random.randint(0, INPUT_SIZE - len_mask)\n                X_batch[b, start_mask:start_mask+len_mask, :, :] = 0\n                non_empty_frame_idxs_batch[b, start_mask:start_mask+len_mask] = -1\n\n        # MixUp - REDUCED from 70% to 30%\n        if mixup_alpha > 0 and np.random.rand() < 0.3:\n            lam = np.random.beta(mixup_alpha, mixup_alpha)\n            perm_indices = np.random.permutation(X_batch.shape[0])\n            \n            X_batch = lam * X_batch + (1 - lam) * X_batch[perm_indices]\n            y_batch = lam * y_batch + (1 - lam) * y_batch[perm_indices]\n            \n            if lam < 0.5:\n                non_empty_frame_idxs_batch = non_empty_frame_idxs_batch[perm_indices]\n        \n        yield { 'frames': X_batch, 'non_empty_frame_idxs': non_empty_frame_idxs_batch }, y_batch","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:21:07.486355Z","iopub.execute_input":"2026-01-21T15:21:07.486798Z","iopub.status.idle":"2026-01-21T15:21:07.501697Z","shell.execute_reply.started":"2026-01-21T15:21:07.486771Z","shell.execute_reply":"2026-01-21T15:21:07.500707Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dummy_dataset = get_train_batch_all_signs(X_train, y_train, NON_EMPTY_FRAME_IDXS_TRAIN)\nX_batch, y_batch = next(dummy_dataset)\n\nfor k, v in X_batch.items():\n    print(f'{k} shape: {v.shape}, dtype: {v.dtype}')\n\n# Batch shape/dtype\nprint(f'y_batch shape: {y_batch.shape}, dtype: {y_batch.dtype}')\n# Verify each batch contains each sign exactly N times\ndisplay(pd.Series(np.argmax(y_batch, axis=1)).value_counts().to_frame('Counts')) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:21:07.502701Z","iopub.execute_input":"2026-01-21T15:21:07.502992Z","iopub.status.idle":"2026-01-21T15:21:08.297572Z","shell.execute_reply.started":"2026-01-21T15:21:07.502968Z","shell.execute_reply":"2026-01-21T15:21:08.296714Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model Config","metadata":{}},{"cell_type":"code","source":"# Epsilon value for layer normalisation\nLAYER_NORM_EPS = 1e-6\n\n# Dense layer units for landmarks\nLIPS_UNITS = 192 # 384 or 128\nHANDS_UNITS = 192 # 384\nPOSE_UNITS = 192 # 384\n# final embedding and transformer embedding size\nUNITS = 320 # 512 or 256\n\n# Transformer\nNUM_BLOCKS = 2\nMLP_RATIO = 2\n\n# Dropout\nEMBEDDING_DROPOUT = 0.20 #0.00\nMLP_DROPOUT_RATIO = 0.40 #0.30\nCLASSIFIER_DROPOUT_RATIO = 0.50 #0.10\n\n# Initiailizers\nINIT_HE_UNIFORM = tf.keras.initializers.he_uniform\nINIT_GLOROT_UNIFORM = tf.keras.initializers.glorot_uniform\nINIT_ZEROS = tf.keras.initializers.constant(0.0)\n# Activations\nGELU = tf.keras.activations.gelu","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:21:08.298783Z","iopub.execute_input":"2026-01-21T15:21:08.299201Z","iopub.status.idle":"2026-01-21T15:21:08.304425Z","shell.execute_reply.started":"2026-01-21T15:21:08.299173Z","shell.execute_reply":"2026-01-21T15:21:08.303763Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def scaled_dot_product(q,k,v, softmax, attention_mask):\n    #calculates Q . K(transpose)\n    qkt = tf.matmul(q,k,transpose_b=True)\n    # calculates scaling factor\n    dk = tf.math.sqrt(tf.cast(q.shape[-1],dtype=tf.float32))\n    scaled_qkt = qkt/dk\n    softmax = softmax(scaled_qkt, mask=attention_mask)\n    \n    z = tf.matmul(softmax,v)\n    #shape: (m,Tx,depth), same shape as q,k,v\n    return z\n\nclass MultiHeadAttention(tf.keras.layers.Layer):\n    def __init__(self,d_model,num_of_heads):\n        super(MultiHeadAttention,self).__init__()\n        self.d_model = d_model\n        self.num_of_heads = num_of_heads\n        self.depth = d_model//num_of_heads\n        self.wq = [tf.keras.layers.Dense(self.depth) for i in range(num_of_heads)]\n        self.wk = [tf.keras.layers.Dense(self.depth) for i in range(num_of_heads)]\n        self.wv = [tf.keras.layers.Dense(self.depth) for i in range(num_of_heads)]\n        self.wo = tf.keras.layers.Dense(d_model)\n        self.softmax = tf.keras.layers.Softmax()\n        \n    def call(self,x, attention_mask):\n        \n        multi_attn = []\n        for i in range(self.num_of_heads):\n            Q = self.wq[i](x)\n            K = self.wk[i](x)\n            V = self.wv[i](x)\n            multi_attn.append(scaled_dot_product(Q,K,V, self.softmax, attention_mask))\n            \n        multi_head = tf.concat(multi_attn,axis=-1)\n        multi_head_attention = self.wo(multi_head)\n        return multi_head_attention ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:21:08.306270Z","iopub.execute_input":"2026-01-21T15:21:08.306588Z","iopub.status.idle":"2026-01-21T15:21:08.322503Z","shell.execute_reply.started":"2026-01-21T15:21:08.306553Z","shell.execute_reply":"2026-01-21T15:21:08.321702Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Full Transformer\nclass Transformer(tf.keras.Model):\n    def __init__(self, num_blocks):\n        super(Transformer, self).__init__(name='transformer')\n        self.num_blocks = num_blocks\n    \n    def build(self, input_shape):\n        self.ln_1s = []\n        self.mhas = []\n        self.ln_2s = []\n        self.mlps = []\n        # Make Transformer Blocks\n        for i in range(self.num_blocks):\n            # Multi-Head Attention\n            self.mhas.append(MultiHeadAttention(UNITS, 8))\n            # Multi Layer Perception\n            self.mlps.append(tf.keras.Sequential([\n                tf.keras.layers.Dense(UNITS * MLP_RATIO, activation=GELU, kernel_initializer=INIT_GLOROT_UNIFORM),\n                tf.keras.layers.Dropout(MLP_DROPOUT_RATIO),\n                tf.keras.layers.Dense(UNITS, kernel_initializer=INIT_HE_UNIFORM),\n            ]))\n        \n    def call(self, x, attention_mask):\n        # Iterate input over transformer blocks\n        for mha, mlp in zip(self.mhas, self.mlps):\n            x = x + mha(x, attention_mask)\n            x = x + mlp(x)\n    \n        return x ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:21:08.323461Z","iopub.execute_input":"2026-01-21T15:21:08.323773Z","iopub.status.idle":"2026-01-21T15:21:08.341099Z","shell.execute_reply.started":"2026-01-21T15:21:08.323752Z","shell.execute_reply":"2026-01-21T15:21:08.340338Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Landmark Embedding","metadata":{}},{"cell_type":"code","source":"class LandmarkEmbedding(tf.keras.Model):\n    def __init__(self, units, name):\n        super(LandmarkEmbedding, self).__init__(name=f'{name}_embedding')\n        self.units = units\n        \n    def build(self, input_shape):\n        # Embedding for missing landmark in frame, initialized with zeros\n        self.empty_embedding = self.add_weight(\n            name=f'{self.name}_empty_embedding',\n            shape=[self.units],\n            initializer=INIT_ZEROS,\n        )\n        # Embedding\n        self.dense = tf.keras.Sequential([\n            tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_1', use_bias=False, kernel_initializer=INIT_GLOROT_UNIFORM),\n            tf.keras.layers.Activation(GELU),\n            tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_2', use_bias=False, kernel_initializer=INIT_HE_UNIFORM),\n        ], name=f'{self.name}_dense')\n\n    def call(self, x):\n        return tf.where(\n                # Checks whether landmark is missing in frame\n                tf.reduce_sum(x, axis=2, keepdims=True) == 0,\n                # If so, the empty embedding is used\n                self.empty_embedding,\n                # Otherwise the landmark data is embedded\n                self.dense(x),\n            )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:21:08.342232Z","iopub.execute_input":"2026-01-21T15:21:08.342747Z","iopub.status.idle":"2026-01-21T15:21:08.355520Z","shell.execute_reply.started":"2026-01-21T15:21:08.342724Z","shell.execute_reply":"2026-01-21T15:21:08.354951Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Embedding(tf.keras.Model):\n    def __init__(self):\n        super(Embedding, self).__init__()\n        \n    def get_diffs(self, l):\n        S = l.shape[2]\n        other = tf.expand_dims(l, 3)\n        other = tf.repeat(other, S, axis=3)\n        other = tf.transpose(other, [0,1,3,2])\n        diffs = tf.expand_dims(l, 3) - other\n        diffs = tf.reshape(diffs, [-1, INPUT_SIZE, S*S])\n        return diffs\n\n    def build(self, input_shape):\n        # Positional Embedding, initialized with zeros\n        self.positional_embedding = tf.keras.layers.Embedding(INPUT_SIZE+1, UNITS, embeddings_initializer=INIT_ZEROS)\n        # Embedding layer for Landmarks\n        self.lips_embedding = LandmarkEmbedding(LIPS_UNITS, 'lips')\n        self.left_hand_embedding = LandmarkEmbedding(HANDS_UNITS, 'left_hand')\n        self.pose_embedding = LandmarkEmbedding(POSE_UNITS, 'pose')\n        # Landmark Weights\n        self.landmark_weights = tf.Variable(tf.zeros([3], dtype=tf.float32), name='landmark_weights')\n        # Fully Connected Layers for combined landmarks\n        self.fc = tf.keras.Sequential([\n            tf.keras.layers.Dense(UNITS, name='fully_connected_1', use_bias=False, kernel_initializer=INIT_GLOROT_UNIFORM),\n            tf.keras.layers.Activation(GELU),\n            tf.keras.layers.Dense(UNITS, name='fully_connected_2', use_bias=False, kernel_initializer=INIT_HE_UNIFORM),\n        ], name='fc')\n\n\n    def call(self, lips0, left_hand0, pose0, non_empty_frame_idxs, training=False):\n        # Lips\n        lips_embedding = self.lips_embedding(lips0)\n        # Left Hand\n        left_hand_embedding = self.left_hand_embedding(left_hand0)\n        # Pose\n        pose_embedding = self.pose_embedding(pose0)\n        # Merge Embeddings of all landmarks with mean pooling\n        x = tf.stack((\n            lips_embedding, left_hand_embedding, pose_embedding,\n        ), axis=3)\n        x = x * tf.nn.softmax(self.landmark_weights)\n        x = tf.reduce_sum(x, axis=3)\n        # Fully Connected Layers\n        x = self.fc(x)\n        # Add Positional Embedding\n        max_frame_idxs = tf.clip_by_value(\n                tf.reduce_max(non_empty_frame_idxs, axis=1, keepdims=True),\n                1,\n                np.inf, #PINF\n            )\n        normalised_non_empty_frame_idxs = tf.where(\n            tf.math.equal(non_empty_frame_idxs, -1.0),\n            INPUT_SIZE,\n            tf.cast(\n                non_empty_frame_idxs / max_frame_idxs * INPUT_SIZE,\n                tf.int32,\n            ),\n        )\n        x = x + self.positional_embedding(normalised_non_empty_frame_idxs)\n        \n        return x","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:21:08.356502Z","iopub.execute_input":"2026-01-21T15:21:08.356813Z","iopub.status.idle":"2026-01-21T15:21:08.372079Z","shell.execute_reply.started":"2026-01-21T15:21:08.356791Z","shell.execute_reply":"2026-01-21T15:21:08.371301Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Categorical Cross Entropy with label Smoothing","metadata":{}},{"cell_type":"code","source":"def cce_with_ls(y_true, y_pred):\n    # Handle sparse labels\n    if len(y_true.shape) == 1 or (len(y_true.shape) == 2 and y_true.shape[1] == 1):\n        y_true = tf.one_hot(tf.cast(tf.reshape(y_true, [-1]), tf.int32), NUM_CLASSES)\n        \n    return tf.keras.losses.categorical_crossentropy(y_true, y_pred, label_smoothing=0.25)\n\ndef get_motion_features(t):\n    # t: (Batch, Frames, Landmarks*2)\n    shape = tf.shape(t)\n    B, T = shape[0], shape[1]\n    L = shape[2] // 2\n    \n    t_reshaped = tf.reshape(t, [B, T, L, 2])\n    \n    # Calculate dx\n    t_prev = tf.concat([t_reshaped[:, :1], t_reshaped[:, :-1]], axis=1)\n    dx = t_reshaped - t_prev\n    \n    # Calculate dx2\n    dx_prev = tf.concat([dx[:, :1], dx[:, :-1]], axis=1)\n    dx2 = dx - dx_prev\n    \n    out = tf.concat([t_reshaped, dx, dx2], axis=-1)\n    out = tf.reshape(out, [B, T, L*6])\n    \n    # Mask out if original was 0\n    mask = tf.cast(tf.not_equal(t_reshaped, 0.0), tf.float32)\n    mask = tf.reduce_max(mask, axis=-1, keepdims=True)\n    out_reshaped = tf.reshape(out, [B, T, L, 6])\n    out_reshaped = out_reshaped * mask\n    out = tf.reshape(out_reshaped, [B, T, L*6])\n    \n    return out\n\ndef get_spatial_features(t):\n    # t: (Batch, Frames, Landmarks*2)\n    shape = tf.shape(t)\n    B, T = shape[0], shape[1]\n    L = shape[2] // 2\n    \n    t_reshaped = tf.reshape(t, [B, T, L, 2])\n    \n    # Pairwise differences: (B, T, L, L, 2)\n    r1 = tf.expand_dims(t_reshaped, 3)\n    r2 = tf.expand_dims(t_reshaped, 2)\n    diffs = r1 - r2 \n    \n    # Distances (Euclidean) -> (B, T, L, L)\n    dists = tf.norm(diffs + 1e-6, axis=-1)\n    \n    # Flatten\n    out = tf.reshape(dists, [B, T, L*L])\n    \n    # Masking (if original point was 0)\n    mask = tf.cast(tf.not_equal(t_reshaped, 0.0), tf.float32) # (B, T, L, 2)\n    mask = tf.reduce_max(mask, axis=-1) # (B, T, L)\n    \n    # Pairwise mask\n    m1 = tf.expand_dims(mask, 3)\n    m2 = tf.expand_dims(mask, 2)\n    pairwise_mask = m1 * m2 # (B, T, L, L)\n    \n    out = out * tf.reshape(pairwise_mask, [B, T, L*L])\n    \n    return out\n\n\nclass Augmentation(tf.keras.layers.Layer):\n    def __init__(self, **kwargs):\n        super().__init__(**kwargs)\n\n    def call(self, lips, left_hand, pose):\n        # Expects inputs of shape (B, T, L, 2)\n        # Random Rotation\n        if True: # Always apply random rotation if this layer is active (usually strictly during training)\n            # Random angle between -10 and 10 degrees\n            theta = tf.random.uniform((tf.shape(lips)[0],), minval=-0.2, maxval=0.2)\n            cos_t = tf.cos(theta)\n            sin_t = tf.sin(theta)\n            # Rotation Matrix (B, 2, 2)\n            # [[cos, -sin], [sin, cos]]\n            rot_mat = tf.stack([cos_t, -sin_t, sin_t, cos_t], axis=1)\n            rot_mat = tf.reshape(rot_mat, (-1, 2, 2))\n            \n            # Apply Rotation\n            # lips: (B, T, 40, 2) -> flatten to (B, T*40, 2) for matmul then reshape back?\n            # Easier: einsum 'btlz,bzo->btlo'\n            lips = tf.einsum('btlz,bzo->btlo', lips, rot_mat)\n            left_hand = tf.einsum('btlz,bzo->btlo', left_hand, rot_mat)\n            pose = tf.einsum('btlz,bzo->btlo', pose, rot_mat)\n        \n        # Random Scaling\n        if True:\n            # Scale between 0.8 and 1.2\n            scale = tf.random.uniform((tf.shape(lips)[0], 1, 1, 1), minval=0.8, maxval=1.2)\n            lips = lips * scale\n            left_hand = left_hand * scale\n            pose = pose * scale\n            \n        # Random Translation (Shift)\n        if True:\n            # Shift by +/- 0.1 (coordinate space is roughly normalized to std via later layers, but here raw)\n            shift = tf.random.uniform((tf.shape(lips)[0], 1, 1, 2), minval=-0.1, maxval=0.1)\n            lips = lips + shift\n            left_hand = left_hand + shift\n            pose = pose + shift\n\n        return lips, left_hand, pose","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:21:08.373101Z","iopub.execute_input":"2026-01-21T15:21:08.373440Z","iopub.status.idle":"2026-01-21T15:21:08.391384Z","shell.execute_reply.started":"2026-01-21T15:21:08.373418Z","shell.execute_reply":"2026-01-21T15:21:08.390640Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class ECA(tf.keras.layers.Layer):\n    def __init__(self, kernel_size=5, **kwargs):\n        super().__init__(**kwargs)\n        self.supports_masking = True\n        self.kernel_size = kernel_size\n        self.conv = tf.keras.layers.Conv1D(1, kernel_size=kernel_size, strides=1, padding=\"same\", use_bias=False)\n\n    def call(self, inputs, mask=None):\n        nn = tf.keras.layers.GlobalAveragePooling1D(keepdims=True)(inputs)\n        nn = self.conv(nn)\n        nn = tf.keras.activations.sigmoid(nn)\n        return inputs * nn\n\nclass LateDropout(tf.keras.layers.Layer):\n    def __init__(self, rate, noise_shape=None, start_step=0, **kwargs):\n        super().__init__(**kwargs)\n        self.supports_masking = True\n        self.rate = rate\n        self.noise_shape = noise_shape\n        self.start_step = start_step\n        self.dropout = tf.keras.layers.Dropout(rate, noise_shape=noise_shape)\n\n    def build(self, input_shape):\n        super().build(input_shape)\n        agg = tf.VariableAggregation.ONLY_FIRST_REPLICA\n        self._train_counter = tf.Variable(0, dtype=\"int64\", aggregation=agg, trainable=False)\n\n    def call(self, inputs, training=False):\n        if training is None:\n            training = tf.keras.backend.learning_phase()\n        \n        def train_fn():\n            self._train_counter.assign_add(1)\n            return tf.cond(\n                self._train_counter > self.start_step,\n                lambda: self.dropout(inputs, training=True),\n                lambda: inputs\n            )\n            \n        # Handle both boolean and tensor 'training' flag\n        return tf.cond(\n            tf.cast(training, tf.bool),\n            train_fn,\n            lambda: inputs\n        ) \n            \n\nclass CausalDWConv1D(tf.keras.layers.Layer):\n    def __init__(self, kernel_size=17, dilation_rate=1, use_bias=False, depthwise_initializer='glorot_uniform', name='', **kwargs):\n        super().__init__(name=name,**kwargs)\n        self.causal_pad = tf.keras.layers.ZeroPadding1D((dilation_rate*(kernel_size-1),0),name=name + '_pad')\n        self.dw_conv = tf.keras.layers.DepthwiseConv1D(\n                            kernel_size,\n                            strides=1,\n                            dilation_rate=dilation_rate,\n                            padding='valid',\n                            use_bias=use_bias,\n                            depthwise_initializer=depthwise_initializer,\n                            name=name + '_dwconv'\n                        )\n        self.supports_masking = True\n        \n    def call(self, inputs):\n        x = self.causal_pad(inputs)\n        x = self.dw_conv(x)\n        return x\n\ndef Conv1DBlock(channel_size, kernel_size, dilation_rate=1, drop_rate=0.0, expand_ratio=2, se_ratio=0.25, activation='swish', name=None):\n    '''\n    Efficient Conv1D Block: Expansion -> Depthwise Conv -> Squeeze-Excitation -> Projection\n    Similar to MobileNetV3 / EfficientNet block but 1D.\n    '''\n    if name is None:\n        name = str(tf.keras.backend.get_uid(\"mbblock\"))\n    \n    # Expansion\n    def apply(inputs):\n        channels_in = tf.keras.backend.int_shape(inputs)[-1]\n        channels_expand = channels_in * expand_ratio\n        \n        x = inputs\n        \n        # Expansion\n        if expand_ratio != 1:\n            x = tf.keras.layers.Dense(channels_expand, use_bias=False, activation=activation, name=name + '_expand')(x)\n            x = tf.keras.layers.BatchNormalization(momentum=0.95, name=name + '_expand_bn')(x)\n        \n        # Depthwise Conv\n        x = tf.keras.layers.DepthwiseConv1D(kernel_size, strides=1, dilation_rate=dilation_rate, padding='same', use_bias=False, name=name + '_dwconv')(x)\n        x = tf.keras.layers.BatchNormalization(momentum=0.95, name=name + '_dwconv_bn')(x)\n        x = tf.keras.layers.Activation(activation)(x) \n        \n        return x\n    return apply\n\nclass MultiHeadSelfAttention(tf.keras.layers.Layer):\n    def __init__(self, dim=256, num_heads=4, dropout=0, **kwargs):\n        super().__init__(**kwargs)\n        self.dim = dim\n        self.scale = self.dim ** -0.5\n        self.num_heads = num_heads\n        self.head_dim = dim // num_heads\n        self.qkv = tf.keras.layers.Dense(dim * 3, use_bias=False)\n        self.drop1 = tf.keras.layers.Dropout(dropout)\n        self.proj = tf.keras.layers.Dense(dim, use_bias=False)\n        self.supports_masking = True\n\n    def call(self, inputs, mask=None):\n        if mask is not None:\n            # mask has shape (B, T)\n            # needs to be (B, 1, 1, T)\n            mask = mask[:, None, None, :]\n            \n        qkv = self.qkv(inputs)\n        qkv = tf.keras.layers.Permute((2, 1, 3))(tf.keras.layers.Reshape((-1, self.num_heads, self.head_dim * 3))(qkv))\n        q, k, v = tf.split(qkv, [self.head_dim, self.head_dim, self.head_dim], axis=-1)\n        \n        attn = tf.matmul(q, k, transpose_b=True) * self.scale\n        \n        if mask is not None:\n            attn = attn + (1 - tf.cast(mask, attn.dtype)) * -1e9\n\n        attn = tf.nn.softmax(attn, axis=-1)\n        attn = self.drop1(attn)\n\n        x = attn @ v\n        x = tf.keras.layers.Reshape((-1, self.dim))(tf.keras.layers.Permute((2, 1, 3))(x))\n        x = self.proj(x)\n        return x\n\ndef TransformerBlock(dim=256, num_heads=4, expand=4, attn_dropout=0.2, drop_rate=0.2, activation='swish'):\n    def apply(inputs):\n        x = inputs\n        # Norm\n        x = tf.keras.layers.BatchNormalization(momentum=0.95)(x)\n        # SA\n        x = MultiHeadSelfAttention(dim=dim,num_heads=num_heads,dropout=attn_dropout)(x)\n        x = tf.keras.layers.Dropout(drop_rate, noise_shape=(None,1,1))(x)\n        x = tf.keras.layers.Add()([inputs, x])\n        attn_out = x\n\n        # FFN\n        x = tf.keras.layers.BatchNormalization(momentum=0.95)(x)\n        x = tf.keras.layers.Dense(dim*expand, use_bias=False, activation=activation)(x)\n        x = tf.keras.layers.Dense(dim, use_bias=False)(x)\n        x = tf.keras.layers.Dropout(drop_rate, noise_shape=(None,1,1))(x)\n        x = tf.keras.layers.Add()([attn_out, x])\n        return x\n    return apply\n\ndef Conv1DBlock(dim, ksize, drop_rate=0.0, activation='swish'):\n    def apply(inputs):\n        x = inputs\n        x = tf.keras.layers.BatchNormalization(momentum=0.95)(x)\n        x = tf.keras.layers.Dense(dim, use_bias=False, activation=activation)(x)\n        x = tf.keras.layers.DepthwiseConv1D(ksize, strides=1, padding='same', use_bias=False)(x)\n        x = tf.keras.layers.BatchNormalization(momentum=0.95)(x)\n        x = tf.keras.layers.Activation(activation)(x)\n        x = tf.keras.layers.Dense(dim, use_bias=False)(x)\n        x = tf.keras.layers.Dropout(drop_rate, noise_shape=(None,1,1))(x)\n        x = tf.keras.layers.Add()([inputs, x])\n        return x\n    return apply ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:21:08.392374Z","iopub.execute_input":"2026-01-21T15:21:08.392673Z","iopub.status.idle":"2026-01-21T15:21:08.418806Z","shell.execute_reply.started":"2026-01-21T15:21:08.392645Z","shell.execute_reply":"2026-01-21T15:21:08.418043Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Adversarial Weight Perturbation (AWP)\nclass AWP(tf.keras.Model):\n    def __init__(self, core_model, delta=0.2, eps=1e-6, start_step=0, **kwargs):\n        super(AWP, self).__init__(**kwargs)\n        self.core_model = core_model\n        self.delta = delta\n        self.eps = eps\n        self.start_step = start_step\n        self.step_counter = tf.Variable(0, trainable=False, dtype=tf.int64)\n\n    def call(self, inputs, training=None):\n        return self.core_model(inputs, training=training)\n\n    def train_step(self, data):\n        self.step_counter.assign_add(1)\n        x, y = data\n\n        # Calculate gradients on original weights\n        with tf.GradientTape() as tape:\n            y_pred = self.core_model(x, training=True)\n            loss = self.compiled_loss(y, y_pred, regularization_losses=self.core_model.losses)\n        \n        trainable_vars = self.core_model.trainable_variables\n        gradients = tape.gradient(loss, trainable_vars)\n\n        # Perturb weights (using tf.cond for graph compatibility)\n        should_perturb = self.step_counter >= self.start_step\n        \n        def perturb_weights():\n            for v, g in zip(trainable_vars, gradients):\n                if g is not None:\n                    # Handle IndexedSlices (sparse gradients from Embeddings)\n                    if isinstance(g, tf.IndexedSlices):\n                        g = tf.convert_to_tensor(g)\n                    \n                    norm = tf.norm(g)\n                    perturbation = self.delta * g / (norm + self.eps)\n                    v.assign_add(perturbation)\n            return tf.constant(0)  # Dummy return\n        \n        def no_op():\n            return tf.constant(0)\n        \n        tf.cond(should_perturb, perturb_weights, no_op)\n\n        # Calculate gradients on perturbed weights\n        with tf.GradientTape() as tape:\n             y_pred = self.core_model(x, training=True)\n             loss_Mp = self.compiled_loss(y, y_pred, regularization_losses=self.core_model.losses)\n             \n        gradients_Mp = tape.gradient(loss_Mp, trainable_vars)\n        \n        # Restore weights (using tf.cond)\n        def restore_weights():\n            for v, g in zip(trainable_vars, gradients):\n                if g is not None:\n                    # Handle IndexedSlices (sparse gradients from Embeddings)\n                    if isinstance(g, tf.IndexedSlices):\n                        g = tf.convert_to_tensor(g)\n                        \n                    norm = tf.norm(g)\n                    perturbation = self.delta * g / (norm + self.eps)\n                    v.assign_sub(perturbation)\n            return tf.constant(0)\n        \n        tf.cond(should_perturb, restore_weights, no_op)\n\n        # Update\n        self.optimizer.apply_gradients(zip(gradients_Mp, trainable_vars))\n        \n        self.compiled_metrics.update_state(y, y_pred)\n        return {m.name: m.result() for m in self.metrics}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:21:08.419720Z","iopub.execute_input":"2026-01-21T15:21:08.420049Z","iopub.status.idle":"2026-01-21T15:21:08.435948Z","shell.execute_reply.started":"2026-01-21T15:21:08.420018Z","shell.execute_reply":"2026-01-21T15:21:08.435366Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_model(max_len=INPUT_SIZE, dim=256, ksize=17, dropout_step=0):\n    # Inputs\n    frames = tf.keras.layers.Input([INPUT_SIZE, N_COLS, N_DIMS], dtype=tf.float32, name='frames')\n    non_empty_frame_idxs = tf.keras.layers.Input([INPUT_SIZE], dtype=tf.float32, name='non_empty_frame_idxs')\n\n    # Padding Mask\n    mask0 = tf.keras.layers.Lambda(lambda x: tf.cast(tf.math.not_equal(x, -1), tf.float32), name='mask0')(non_empty_frame_idxs)\n    mask0_expanded = tf.keras.layers.Lambda(lambda x: tf.expand_dims(x, axis=2), name='mask0_expanded')(mask0)\n\n    # Random Frame Masking\n    def create_mask(x):\n        mask = tf.where(\n            (tf.random.uniform(tf.shape(x[0])) > 0.25) & tf.math.not_equal(x[0], 0.0),\n            1.0, 0.0,\n        )\n        mask = tf.where(\n            tf.math.equal(tf.reduce_sum(mask, axis=[1,2], keepdims=True), 0.0),\n            x[0], mask,\n        )\n        return mask\n    \n    mask = tf.keras.layers.Lambda(create_mask, name='create_mask')([mask0_expanded, mask0_expanded])\n\n    # Slicing the XY coordinates\n    x = tf.keras.layers.Lambda(lambda t: tf.slice(t, [0,0,0,0], [-1,INPUT_SIZE, N_COLS, 2]), name='slice_xy')(frames)\n    \n    lips_raw = tf.keras.layers.Lambda(lambda t: tf.slice(t, [0,0,LIPS_START,0], [-1,INPUT_SIZE, 40, 2]), name='slice_lips_raw')(x)\n    left_hand_raw = tf.keras.layers.Lambda(lambda t: tf.slice(t, [0,0,40,0], [-1,INPUT_SIZE, 21, 2]), name='slice_left_hand_raw')(x)\n    pose_raw = tf.keras.layers.Lambda(lambda t: tf.slice(t, [0,0,61,0], [-1,INPUT_SIZE, 5, 2]), name='slice_pose_raw')(x)\n\n    # --- AUGMENTATION (Applied to RAW coordinates) ---\n    aug = Augmentation()\n    lips_raw, left_hand_raw, pose_raw = aug(lips_raw, left_hand_raw, pose_raw)\n\n    # LIPS Processing\n    lips = tf.keras.layers.Lambda(lambda t: tf.where(tf.math.equal(t, 0.0), 0.0, (t - LIPS_MEDIAN) / LIPS_IQR), name='normalize_lips')(lips_raw)\n    lips = tf.keras.layers.Reshape((INPUT_SIZE, 40*2), name='reshape_lips')(lips)\n    lips = tf.keras.layers.Lambda(get_motion_features, name='motion_lips')(lips)\n\n    # LEFT HAND Processing\n    left_hand = tf.keras.layers.Lambda(lambda t: tf.where(tf.math.equal(t, 0.0), 0.0, (t - LEFT_HANDS_MEDIAN) / LEFT_HANDS_IQR), name='normalize_left_hand')(left_hand_raw)\n    left_hand = tf.keras.layers.Reshape((INPUT_SIZE, 21*2), name='reshape_left_hand')(left_hand)\n    left_hand_motion = tf.keras.layers.Lambda(get_motion_features, name='motion_left_hand')(left_hand)\n    left_hand_spatial = tf.keras.layers.Lambda(get_spatial_features, name='spatial_left_hand')(left_hand)\n    left_hand = tf.keras.layers.Concatenate(axis=-1, name='concat_left_hand')([left_hand_motion, left_hand_spatial])\n\n    # POSE Processing\n    pose = tf.keras.layers.Lambda(lambda t: tf.where(tf.math.equal(t, 0.0), 0.0, (t - POSE_MEDIAN) / POSE_IQR), name='normalize_pose')(pose_raw)\n    pose = tf.keras.layers.Reshape((INPUT_SIZE, 5*2), name='reshape_pose')(pose)\n    pose_motion = tf.keras.layers.Lambda(get_motion_features, name='motion_pose')(pose)\n    pose_spatial = tf.keras.layers.Lambda(get_spatial_features, name='spatial_pose')(pose)\n    pose = tf.keras.layers.Concatenate(axis=-1, name='concat_pose')([pose_motion, pose_spatial])\n    \n    # Embedding\n    x = Embedding()(lips, left_hand, pose, non_empty_frame_idxs)\n    \n    # === SQUEEZEFORMER TRUNK ===\n    # Stem\n    x = tf.keras.layers.Dense(dim, use_bias=False, name='stem_conv')(x)\n    x = tf.keras.layers.BatchNormalization(momentum=0.95, name='stem_bn')(x)\n    \n    # Block 1 - Progressive Stochastic Depth (0.0 -> 0.2)\n    dpr = [x for x in tf.linspace(0.0, 0.2, 8)]\n\n    x = Conv1DBlock(dim, ksize, drop_rate=dpr[0])(x)\n    x = Conv1DBlock(dim, ksize, drop_rate=dpr[1])(x)\n    x = Conv1DBlock(dim, ksize, drop_rate=dpr[2])(x)\n    x = TransformerBlock(dim, expand=2, drop_rate=dpr[3])(x)\n\n    # Block 2\n    x = Conv1DBlock(dim, ksize, drop_rate=dpr[4])(x)\n    x = Conv1DBlock(dim, ksize, drop_rate=dpr[5])(x)\n    x = Conv1DBlock(dim, ksize, drop_rate=dpr[6])(x)\n    x = TransformerBlock(dim, expand=2, drop_rate=dpr[7])(x)\n\n    # Top\n    x = tf.keras.layers.Dense(dim*2, activation=None, name='top_conv')(x)\n    x = tf.keras.layers.GlobalAveragePooling1D()(x)\n    x = tf.keras.layers.Dropout(0.4)(x)  # Slightly lower dropout\n    x = tf.keras.layers.Dense(NUM_CLASSES, name='classifier')(x)\n    \n    outputs = x\n    model = tf.keras.models.Model(inputs=[frames, non_empty_frame_idxs], outputs=outputs)\n    \n    # Optimized loss and metrics\n    loss = tf.keras.losses.CategoricalCrossentropy(from_logits=True, label_smoothing=0.05)\n    \n    # Top-5 accuracy metric\n    top5_acc = tf.keras.metrics.TopKCategoricalAccuracy(k=5, name='top5_acc')\n    \n    optimizer = tf.keras.optimizers.AdamW(learning_rate=1e-3, weight_decay=1e-5, clipnorm=1.0)\n    model.compile(loss=loss, optimizer=optimizer, metrics=['acc', top5_acc])\n    return model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:21:08.436846Z","iopub.execute_input":"2026-01-21T15:21:08.437160Z","iopub.status.idle":"2026-01-21T15:21:08.456388Z","shell.execute_reply.started":"2026-01-21T15:21:08.437135Z","shell.execute_reply":"2026-01-21T15:21:08.455793Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tf.keras.backend.clear_session()\n\nmodel = get_model() ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:21:08.457186Z","iopub.execute_input":"2026-01-21T15:21:08.457460Z","iopub.status.idle":"2026-01-21T15:21:10.677442Z","shell.execute_reply.started":"2026-01-21T15:21:08.457439Z","shell.execute_reply":"2026-01-21T15:21:10.676735Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.summary(expand_nested =True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:21:10.678296Z","iopub.execute_input":"2026-01-21T15:21:10.678533Z","iopub.status.idle":"2026-01-21T15:21:10.774520Z","shell.execute_reply.started":"2026-01-21T15:21:10.678501Z","shell.execute_reply":"2026-01-21T15:21:10.773947Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# tf.keras.utils.plot_model(model, show_shapes=True, show_dtype=True, show_layer_names=True, expand_nested=True, show_layer_activations=True) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:21:10.775384Z","iopub.execute_input":"2026-01-21T15:21:10.775664Z","iopub.status.idle":"2026-01-21T15:21:10.779476Z","shell.execute_reply.started":"2026-01-21T15:21:10.775629Z","shell.execute_reply":"2026-01-21T15:21:10.778919Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# No NaN Predictions","metadata":{}},{"cell_type":"code","source":"if not PREPROCESS_DATA and TRAIN_MODEL:\n    y_pred = model.predict_on_batch(X_batch).flatten()\n\n    print(f'# NaN Values In Prediction: {np.isnan(y_pred).sum()}') ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:21:10.780300Z","iopub.execute_input":"2026-01-21T15:21:10.780579Z","iopub.status.idle":"2026-01-21T15:21:10.791713Z","shell.execute_reply.started":"2026-01-21T15:21:10.780548Z","shell.execute_reply":"2026-01-21T15:21:10.791041Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Weight Initialization","metadata":{}},{"cell_type":"code","source":"if not PREPROCESS_DATA and TRAIN_MODEL:\n    plt.figure(figsize=(12,5))\n    plt.title(f'Softmax Output Initialized Model | µ={y_pred.mean():.3f}, σ={y_pred.std():.3f}', pad=25)\n    pd.Series(y_pred).plot(kind='hist', bins=128, label='Class Probability')\n    plt.xlim(0, max(y_pred) * 1.1)\n    plt.vlines([1 / NUM_CLASSES], 0, plt.ylim()[1], color='red', label=f'Random Guessing Baseline 1/NUM_CLASSES={1 / NUM_CLASSES:.3f}')\n    plt.grid()\n    plt.legend()\n    plt.show() ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:21:10.792655Z","iopub.execute_input":"2026-01-21T15:21:10.792868Z","iopub.status.idle":"2026-01-21T15:21:10.805014Z","shell.execute_reply.started":"2026-01-21T15:21:10.792839Z","shell.execute_reply":"2026-01-21T15:21:10.804342Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def lrfn(current_step, num_warmup_steps, lr_max, num_cycles=0.50, num_training_steps=N_EPOCHS):\n    \n    if current_step < num_warmup_steps:\n        if WARMUP_METHOD == 'log':\n            return lr_max * 0.10 ** (num_warmup_steps - current_step)\n        else:\n            return lr_max * 2 ** -(num_warmup_steps - current_step)\n    else:\n        progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))\n\n        return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))) * lr_max ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:21:10.805860Z","iopub.execute_input":"2026-01-21T15:21:10.806158Z","iopub.status.idle":"2026-01-21T15:21:10.818117Z","shell.execute_reply.started":"2026-01-21T15:21:10.806130Z","shell.execute_reply":"2026-01-21T15:21:10.817435Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def plot_lr_schedule(lr_schedule, epochs):\n    fig = plt.figure(figsize=(20, 10))\n    plt.plot([None] + lr_schedule + [None])\n    # X Labels\n    x = np.arange(1, epochs + 1)\n    x_axis_labels = [i if epochs <= 40 or i % 5 == 0 or i == 1 else None for i in range(1, epochs + 1)]\n    plt.xlim([1, epochs])\n    plt.xticks(x, x_axis_labels) # set tick step to 1 and let x axis start at 1\n    \n    plt.ylim([0, max(lr_schedule) * 1.1])\n    \n    # Title\n    schedule_info = f'start: {lr_schedule[0]:.1E}, max: {max(lr_schedule):.1E}, final: {lr_schedule[-1]:.1E}'\n    plt.title(f'Step Learning Rate Schedule, {schedule_info}', size=18, pad=12)\n    \n    # Plot Learning Rates\n    for x, val in enumerate(lr_schedule):\n        if epochs <= 40 or x % 5 == 0 or x is epochs - 1:\n            if x < len(lr_schedule) - 1:\n                if lr_schedule[x - 1] < val:\n                    ha = 'right'\n                else:\n                    ha = 'left'\n            elif x == 0:\n                ha = 'right'\n            else:\n                ha = 'left'\n            plt.plot(x + 1, val, 'o', color='black');\n            offset_y = (max(lr_schedule) - min(lr_schedule)) * 0.02\n            plt.annotate(f'{val:.1E}', xy=(x + 1, val + offset_y), size=12, ha=ha)\n    \n    plt.xlabel('Epoch', size=16, labelpad=5)\n    plt.ylabel('Learning Rate', size=16, labelpad=5)\n    plt.grid()\n    plt.show()\n\n# Learning rate for encoder\nLR_SCHEDULE = [lrfn(step, num_warmup_steps=N_WARMUP_EPOCHS, lr_max=LR_MAX, num_cycles=0.50) for step in range(N_EPOCHS)]\n# Plot Learning Rate Schedule\nplot_lr_schedule(LR_SCHEDULE, epochs=N_EPOCHS)\n# Learning Rate Callback\nlr_callback = tf.keras.callbacks.LearningRateScheduler(lambda step: LR_SCHEDULE[step], verbose=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:21:10.819148Z","iopub.execute_input":"2026-01-21T15:21:10.819434Z","iopub.status.idle":"2026-01-21T15:21:11.166084Z","shell.execute_reply.started":"2026-01-21T15:21:10.819402Z","shell.execute_reply":"2026-01-21T15:21:11.165403Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Weight Decay Callback","metadata":{}},{"cell_type":"code","source":"# Custom callback to update weight decay with learning rate\nclass WeightDecayCallback(tf.keras.callbacks.Callback):\n    def __init__(self, wd_ratio=WD_RATIO):\n        self.step_counter = 0\n        self.wd_ratio = wd_ratio\n    \n    def on_epoch_begin(self, epoch, logs=None):\n        model.optimizer.weight_decay = model.optimizer.learning_rate * self.wd_ratio\n        print(f'learning rate: {model.optimizer.learning_rate.numpy():.2e}, weight decay: {model.optimizer.weight_decay.numpy():.2e}') ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:21:11.167076Z","iopub.execute_input":"2026-01-21T15:21:11.167472Z","iopub.status.idle":"2026-01-21T15:21:11.172118Z","shell.execute_reply.started":"2026-01-21T15:21:11.167447Z","shell.execute_reply":"2026-01-21T15:21:11.171297Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Performance Benchmark","metadata":{}},{"cell_type":"code","source":"# Custom callback to update weight decay with learning rate\nclass WeightDecayCallback(tf.keras.callbacks.Callback):\n    def __init__(self, wd_ratio=WD_RATIO):\n        self.step_counter = 0\n        self.wd_ratio = wd_ratio\n    \n    def on_epoch_begin(self, epoch, logs=None):\n        model.optimizer.weight_decay = model.optimizer.learning_rate * self.wd_ratio\n        print(f'learning rate: {model.optimizer.learning_rate.numpy():.2e}, weight decay: {model.optimizer.weight_decay.numpy():.2e}') ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:21:11.172892Z","iopub.execute_input":"2026-01-21T15:21:11.173145Z","iopub.status.idle":"2026-01-21T15:21:11.189590Z","shell.execute_reply.started":"2026-01-21T15:21:11.173123Z","shell.execute_reply":"2026-01-21T15:21:11.188767Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"if USE_VAL:\n    # Verify val dataset covers all signs\n    print(f'# Unique Signs in Validation Set: {pd.Series(y_val).nunique()}')\n    # Value Counts\n    display(pd.Series(y_val).value_counts().to_frame('Count').iloc[[1,2,3,-3,-2,-1]]) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:21:11.190551Z","iopub.execute_input":"2026-01-21T15:21:11.191112Z","iopub.status.idle":"2026-01-21T15:21:11.207554Z","shell.execute_reply.started":"2026-01-21T15:21:11.191079Z","shell.execute_reply":"2026-01-21T15:21:11.206904Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Sanity Check\nif TRAIN_MODEL and USE_VAL:\n    _ = model.evaluate(*validation_data, verbose=2) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:21:11.208446Z","iopub.execute_input":"2026-01-21T15:21:11.208725Z","iopub.status.idle":"2026-01-21T15:21:35.781757Z","shell.execute_reply.started":"2026-01-21T15:21:11.208702Z","shell.execute_reply":"2026-01-21T15:21:35.780936Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Ensure validation labels are one-hot for CCE\\n\nif TRAIN_MODEL and USE_VAL:\n    try:\n        # Assuming validation_data is tuple (X, y)\\n\n        if isinstance(validation_data, tuple) or isinstance(validation_data, list):\n            val_inputs, val_labels = validation_data\n            if len(val_labels.shape) == 1:\n                print('Converting validation labels to One-Hot')\n                val_labels_oh = tf.one_hot(val_labels, NUM_CLASSES)\n                validation_data = (val_inputs, val_labels_oh)\n    except Exception as e:\n        print(f'Warning: Could not auto-convert validation labels: {e}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:21:35.783499Z","iopub.execute_input":"2026-01-21T15:21:35.783758Z","iopub.status.idle":"2026-01-21T15:21:35.788439Z","shell.execute_reply.started":"2026-01-21T15:21:35.783735Z","shell.execute_reply":"2026-01-21T15:21:35.787626Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if TRAIN_MODEL:\n    tf.keras.backend.clear_session()\n\n    steps_per_epoch = len(X_train) // (NUM_CLASSES * BATCH_ALL_SIGNS_N)\n    \n    # Create model with optimized dimension\n    core_model = get_model(dim=256)  # Increased capacity\n    core_model.summary()\n\n    model = core_model\n\n    early_stopping = tf.keras.callbacks.EarlyStopping(\n        monitor='val_loss', \n        patience=15,  # Increased patience\n        restore_best_weights=True, \n        verbose=1\n    )\n\n    history = model.fit(\n        x=get_train_batch_all_signs(X_train, y_train, NON_EMPTY_FRAME_IDXS_TRAIN),\n        steps_per_epoch=steps_per_epoch,\n        epochs=N_EPOCHS,\n        validation_data=validation_data,\n        callbacks=[\n            early_stopping,\n            lr_callback,\n            WeightDecayCallback(),\n        ],\n        verbose=2,\n    ) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:21:35.789493Z","iopub.execute_input":"2026-01-21T15:21:35.789768Z","iopub.status.idle":"2026-01-21T17:43:44.838306Z","shell.execute_reply.started":"2026-01-21T15:21:35.789744Z","shell.execute_reply":"2026-01-21T17:43:44.837419Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save Model Weights\nmodel.save_weights('model.weights.h5') ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T17:43:44.839524Z","iopub.execute_input":"2026-01-21T17:43:44.839852Z","iopub.status.idle":"2026-01-21T17:43:45.276807Z","shell.execute_reply.started":"2026-01-21T17:43:44.839796Z","shell.execute_reply":"2026-01-21T17:43:45.276149Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if USE_VAL:\n    # val dataset predictions\n    y_val_pred = model.predict({ 'frames': X_val, 'non_empty_frame_idxs': NON_EMPTY_FRAME_IDXS_VAL }, verbose=2).argmax(axis=1)\n    # Label\n    labels = [ORD2SIGN.get(i).replace(' ', '_') for i in range(NUM_CLASSES)]  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T17:43:45.277787Z","iopub.execute_input":"2026-01-21T17:43:45.278064Z","iopub.status.idle":"2026-01-21T17:44:07.342045Z","shell.execute_reply.started":"2026-01-21T17:43:45.278040Z","shell.execute_reply":"2026-01-21T17:44:07.341172Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Test set Evaluation\nif X_test is not None and y_test is not None:\n    print(\"=\" * 50)\n    print(\"EVALUATING ON HOLDOUT TEST SET\")\n    print(\"=\" * 50)\n    \n    # Prepare test data in the same format\n    X_test_batch = np.zeros([len(y_test), INPUT_SIZE, N_COLS, N_DIMS], dtype=np.float32)\n    for i in range(len(y_test)):\n        X_test_batch[i] = X_test[i]\n    \n    # One-hot encode test labels\n    y_test_onehot = np.zeros([len(y_test), NUM_CLASSES], dtype=np.float32)\n    for i, label in enumerate(y_test):\n        y_test_onehot[i, int(label)] = 1.0\n    \n    test_data = (\n        {'frames': X_test_batch, 'non_empty_frame_idxs': NON_EMPTY_FRAME_IDXS_TEST},\n        y_test_onehot\n    )\n    \n    # Evaluate\n    results = model.evaluate(test_data[0], test_data[1], verbose=1)\n    \n    print(\"\\n\" + \"=\" * 50)\n    print(f\"TEST LOSS:     {results[0]:.4f}\")\n    print(f\"TEST ACCURACY: {results[1]*100:.2f}%\")\n    print(\"=\" * 50)\n    \n    # Optional: Get predictions for confusion matrix\n    predictions = model.predict(test_data[0], verbose=1)\n    y_pred = np.argmax(predictions, axis=1)\n    y_true = np.argmax(y_test_onehot, axis=1)\n    \n    # Top-5 accuracy\n    top5_correct = 0\n    for i in range(len(y_true)):\n        top5_preds = np.argsort(predictions[i])[-5:]\n        if y_true[i] in top5_preds:\n            top5_correct += 1\n    top5_acc = top5_correct / len(y_true)\n    print(f\"TOP-5 ACCURACY: {top5_acc*100:.2f}%\")\nelse:\n    print(\"Test set not available. Run preprocessing with PREPROCESS_DATA=True to generate.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T17:44:07.343257Z","iopub.execute_input":"2026-01-21T17:44:07.343569Z","iopub.status.idle":"2026-01-21T17:45:43.284744Z","shell.execute_reply.started":"2026-01-21T17:44:07.343545Z","shell.execute_reply":"2026-01-21T17:45:43.284036Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Landmark Attention Weights","metadata":{}},{"cell_type":"code","source":"embedding_layer = None\nfor layer in model.layers:\n    if 'embedding' in layer.name:\n        embedding_layer = layer\n        break\n\nif embedding_layer is None:\n    raise RuntimeError(\"Embedding layer not found in the model.\")\n\nlandmark_weights_var = getattr(embedding_layer, 'landmark_weights', None)\n\nif landmark_weights_var is None:\n    raise RuntimeError(\"'landmark_weights' attribute not found in the embedding layer.\")\n\n# Convert to numpy and apply softmax for readable weights\nimport scipy\n\nweights = scipy.special.softmax(landmark_weights_var.numpy())\n\n# Print weights\nlandmarks = ['lips_embedding', 'left_hand_embedding', 'pose_embedding']\n\nfor w, lm in zip(weights, landmarks):\n    print(f'{lm} weight: {(w * 100):.1f}%') ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T17:45:43.285741Z","iopub.execute_input":"2026-01-21T17:45:43.286119Z","iopub.status.idle":"2026-01-21T17:45:43.293719Z","shell.execute_reply.started":"2026-01-21T17:45:43.286092Z","shell.execute_reply":"2026-01-21T17:45:43.292961Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Classification Report","metadata":{}},{"cell_type":"code","source":"def print_classification_report():\n    # Classification report for all signs\n    classification_report = sklearn.metrics.classification_report(\n            y_val,\n            y_val_pred,\n            target_names=labels,\n            output_dict=True,\n        )\n    # Round Data for better readability\n    classification_report = pd.DataFrame(classification_report).T\n    classification_report = classification_report.round(2)\n    classification_report = classification_report.astype({\n            'support': np.uint16,\n        })\n    # Add signs\n    classification_report['sign'] = [e if e in SIGN2ORD else -1 for e in classification_report.index]\n    classification_report['sign_ord'] = classification_report['sign'].apply(SIGN2ORD.get).fillna(-1).astype(np.int16)\n    # Sort on F1-score\n    classification_report = pd.concat((\n        classification_report.head(NUM_CLASSES).sort_values('f1-score', ascending=False),\n        classification_report.tail(3),\n    ))\n\n    pd.options.display.max_rows = 999\n    display(classification_report)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T17:45:43.294572Z","iopub.execute_input":"2026-01-21T17:45:43.294867Z","iopub.status.idle":"2026-01-21T17:45:43.310675Z","shell.execute_reply.started":"2026-01-21T17:45:43.294830Z","shell.execute_reply":"2026-01-21T17:45:43.309791Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if USE_VAL:\n    print_classification_report() ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T17:45:43.311778Z","iopub.execute_input":"2026-01-21T17:45:43.312530Z","iopub.status.idle":"2026-01-21T17:45:43.403682Z","shell.execute_reply.started":"2026-01-21T17:45:43.312503Z","shell.execute_reply":"2026-01-21T17:45:43.402960Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training History","metadata":{}},{"cell_type":"code","source":"def plot_history_metric(metric, f_best=np.argmax, ylim=None, yscale=None, yticks=None):\n    plt.figure(figsize=(20, 10))\n    \n    values = history.history[metric]\n    N_EPOCHS = len(values)\n    val = 'val' in ''.join(history.history.keys())\n    # Epoch ticks\n    if N_EPOCHS <= 20:\n        x = np.arange(1, N_EPOCHS + 1)\n    else:\n        x = [1, 5] + [10 + 5 * idx for idx in range((N_EPOCHS - 10) // 5 + 1)]\n\n    x_ticks = np.arange(1, N_EPOCHS+1)\n\n    # Validation\n    if val:\n        val_values = history.history[f'val_{metric}']\n        val_argmin = f_best(val_values)\n        plt.plot(x_ticks, val_values, label=f'val')\n\n    # summarize history for accuracy\n    plt.plot(x_ticks, values, label=f'train')\n    argmin = f_best(values)\n    plt.scatter(argmin + 1, values[argmin], color='red', s=75, marker='o', label=f'train_best')\n    if val:\n        plt.scatter(val_argmin + 1, val_values[val_argmin], color='purple', s=75, marker='o', label=f'val_best')\n\n    plt.title(f'Model {metric}', fontsize=24, pad=10)\n    plt.ylabel(metric, fontsize=20, labelpad=10)\n\n    if ylim:\n        plt.ylim(ylim)\n\n    if yscale is not None:\n        plt.yscale(yscale)\n        \n    if yticks is not None:\n        plt.yticks(yticks, fontsize=16)\n\n    plt.xlabel('epoch', fontsize=20, labelpad=10)        \n    plt.tick_params(axis='x', labelsize=8)\n    plt.xticks(x, fontsize=16) # set tick step to 1 and let x axis start at 1\n    plt.yticks(fontsize=16)\n    \n    plt.legend(prop={'size': 10})\n    plt.grid()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T17:45:43.404778Z","iopub.execute_input":"2026-01-21T17:45:43.405122Z","iopub.status.idle":"2026-01-21T17:45:43.415042Z","shell.execute_reply.started":"2026-01-21T17:45:43.405097Z","shell.execute_reply":"2026-01-21T17:45:43.414282Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if TRAIN_MODEL:\n    plot_history_metric('loss', f_best=np.argmin) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T17:45:43.416341Z","iopub.execute_input":"2026-01-21T17:45:43.416585Z","iopub.status.idle":"2026-01-21T17:45:43.747008Z","shell.execute_reply.started":"2026-01-21T17:45:43.416561Z","shell.execute_reply":"2026-01-21T17:45:43.746203Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if TRAIN_MODEL:\n    plot_history_metric('acc', ylim=[0,1], yticks=np.arange(0.0, 1.1, 0.1)) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T17:45:43.748142Z","iopub.execute_input":"2026-01-21T17:45:43.748569Z","iopub.status.idle":"2026-01-21T17:45:44.060281Z","shell.execute_reply.started":"2026-01-21T17:45:43.748539Z","shell.execute_reply":"2026-01-21T17:45:44.059481Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# TFLite Model Conversion","metadata":{}},{"cell_type":"code","source":"class TFLiteModel(tf.Module):\n    def __init__(self, model):\n        super(TFLiteModel, self).__init__()\n        self.model = model\n        \n        # Initialize Normalization Correction (Copied from PreprocessLayer)\n        normalisation_correction = tf.constant([\n                    # Add 0.50 to left hand (original right hand) and substract 0.50 of right hand (original left hand)\n                    [0] * len(LIPS_IDXS) + [0.50] * len(LEFT_HAND_IDXS) + [0.50] * len(POSE_IDXS),\n                    # Y coordinates stay intact\n                    [0] * len(LANDMARK_IDXS_LEFT_DOMINANT0),\n                    # Z coordinates stay intact\n                    [0] * len(LANDMARK_IDXS_LEFT_DOMINANT0),\n                ],\n                dtype=tf.float32,\n            )\n        self.normalisation_correction = tf.transpose(normalisation_correction, [1,0])\n\n    def pad_edge(self, t, repeats, side):\n        if side == 'LEFT':\n            return tf.concat((tf.repeat(t[:1], repeats=repeats, axis=0), t), axis=0)\n        elif side == 'RIGHT':\n            return tf.concat((t, tf.repeat(t[-1:], repeats=repeats, axis=0)), axis=0)\n\n    @tf.function(input_signature=[tf.TensorSpec(shape=[None, 543, 3], dtype=tf.float32, name='inputs')])\n    def __call__(self, data0):\n        # Number of Frames in Video\n        N_FRAMES0 = tf.shape(data0)[0]\n        \n        # Find dominant hand by comparing summed absolute coordinates\n        left_hand_sum = tf.math.reduce_sum(tf.where(tf.math.is_nan(tf.gather(data0, LEFT_HAND_IDXS0, axis=1)), 0, 1))\n        right_hand_sum = tf.math.reduce_sum(tf.where(tf.math.is_nan(tf.gather(data0, RIGHT_HAND_IDXS0, axis=1)), 0, 1))\n        left_dominant = left_hand_sum >= right_hand_sum\n        \n        # Count non-NaN Hand values in each frame for the dominant hand\n        if left_dominant:\n            frames_hands_non_nan_sum = tf.math.reduce_sum(\n                    tf.where(tf.math.is_nan(tf.gather(data0, LEFT_HAND_IDXS0, axis=1)), 0, 1),\n                    axis=[1, 2],\n                )\n        else:\n            frames_hands_non_nan_sum = tf.math.reduce_sum(\n                    tf.where(tf.math.is_nan(tf.gather(data0, RIGHT_HAND_IDXS0, axis=1)), 0, 1),\n                    axis=[1, 2],\n                )\n        \n        # Find frames indices with coordinates of the dominant hand\n        non_empty_frames_idxs = tf.where(frames_hands_non_nan_sum > 0)\n        non_empty_frames_idxs = tf.squeeze(non_empty_frames_idxs, axis=1)\n        # Filter frames\n        data = tf.gather(data0, non_empty_frames_idxs, axis=0)\n        \n        # Cast Indices in float32 to be compatible with Tensorflow Lite\n        non_empty_frames_idxs = tf.cast(non_empty_frames_idxs, tf.float32)\n        # Normalize to start with 0\n        non_empty_frames_idxs -= tf.reduce_min(non_empty_frames_idxs)\n        \n        # Number of Frames in Filtered Video\n        N_FRAMES = tf.shape(data)[0]\n        \n        # Gather Relevant Landmark Columns\n        if left_dominant:\n            data = tf.gather(data, LANDMARK_IDXS_LEFT_DOMINANT0, axis=1)\n        else:\n            data = tf.gather(data, LANDMARK_IDXS_RIGHT_DOMINANT0, axis=1)\n            data = (\n                    self.normalisation_correction + (\n                        (data - self.normalisation_correction) * tf.where(self.normalisation_correction != 0, -1.0, 1.0))\n                )\n        \n        # Video fits in INPUT_SIZE\n        if N_FRAMES < INPUT_SIZE:\n            # Pad With -1 to indicate padding\n            non_empty_frames_idxs = tf.pad(non_empty_frames_idxs, [[0, INPUT_SIZE-N_FRAMES]], constant_values=-1)\n            # Pad Data With Zeros\n            data = tf.pad(data, [[0, INPUT_SIZE-N_FRAMES], [0,0], [0,0]], constant_values=0)\n            # Fill NaN Values With 0\n            data = tf.where(tf.math.is_nan(data), 0.0, data)\n        # Video needs to be downsampled to INPUT_SIZE\n        else:\n            # Repeat\n            if N_FRAMES < INPUT_SIZE**2:\n                repeats = tf.math.floordiv(INPUT_SIZE * INPUT_SIZE, N_FRAMES0)\n                data = tf.repeat(data, repeats=repeats, axis=0)\n                non_empty_frames_idxs = tf.repeat(non_empty_frames_idxs, repeats=repeats, axis=0)\n\n            # Pad To Multiple Of Input Size\n            pool_size = tf.math.floordiv(len(data), INPUT_SIZE)\n            if tf.math.mod(len(data), INPUT_SIZE) > 0:\n                pool_size += 1\n\n            if pool_size == 1:\n                pad_size = (pool_size * INPUT_SIZE) - len(data)\n            else:\n                pad_size = (pool_size * INPUT_SIZE) % len(data)\n\n            # Pad Start/End with Start/End value\n            pad_left = tf.math.floordiv(pad_size, 2) + tf.math.floordiv(INPUT_SIZE, 2)\n            pad_right = tf.math.floordiv(pad_size, 2) + tf.math.floordiv(INPUT_SIZE, 2)\n            if tf.math.mod(pad_size, 2) > 0:\n                pad_right += 1\n\n            # Pad By Concatenating Left/Right Edge Values\n            data = self.pad_edge(data, pad_left, 'LEFT')\n            data = self.pad_edge(data, pad_right, 'RIGHT')\n\n            # Pad Non Empty Frame Indices\n            non_empty_frames_idxs = self.pad_edge(non_empty_frames_idxs, pad_left, 'LEFT')\n            non_empty_frames_idxs = self.pad_edge(non_empty_frames_idxs, pad_right, 'RIGHT')\n\n            # Reshape to Mean Pool\n            data = tf.reshape(data, [INPUT_SIZE, -1, N_COLS, N_DIMS])\n            non_empty_frames_idxs = tf.reshape(non_empty_frames_idxs, [INPUT_SIZE, -1])\n\n            # Mean Pool\n            data = tf.math.divide_no_nan(tf.reduce_sum(tf.where(tf.math.is_nan(data), 0.0, data), axis=1), tf.reduce_sum(tf.cast(tf.math.is_finite(data), tf.float32), axis=1))\n            non_empty_frames_idxs = tf.math.divide_no_nan(tf.reduce_sum(tf.where(tf.math.is_nan(non_empty_frames_idxs), 0.0, non_empty_frames_idxs), axis=1), tf.reduce_sum(tf.cast(tf.math.is_finite(non_empty_frames_idxs), tf.float32), axis=1))\n\n            # Fill NaN Values With 0\n            data = tf.where(tf.math.is_nan(data), 0.0, data)\n            \n        # --- Preprocessing Done, now Inference ---\n        x = data\n        \n        # Make Prediction (Add batch dim matching inputs)\n        # Note: x is (64, 66, 3), model needs (Batch, 64, 66, 3)\n        outputs = self.model({'frames':tf.expand_dims(x, axis=0), 'non_empty_frame_idxs': tf.expand_dims(non_empty_frames_idxs, axis=0)})[0]\n\n        # Return Dictionary\n        return {'outputs': outputs}\n\ntflite_keras_model = TFLiteModel(model)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T17:45:44.061352Z","iopub.execute_input":"2026-01-21T17:45:44.061605Z","iopub.status.idle":"2026-01-21T17:45:44.085330Z","shell.execute_reply.started":"2026-01-21T17:45:44.061583Z","shell.execute_reply":"2026-01-21T17:45:44.084452Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import json\n# Get concrete function\nconcrete_func = tflite_keras_model.__call__.get_concrete_function()\n\n# Convert from concrete function\nconverter = tf.lite.TFLiteConverter.from_concrete_functions([concrete_func], tflite_keras_model)\nconverter.optimizations = [tf.lite.Optimize.DEFAULT]\nconverter.target_spec.supported_types = [tf.float16]\ntflite_model = converter.convert()\n\n# Save TFLite Model\nwith open('inference.tflite', 'wb') as f:\n    f.write(tflite_model)\n    \n# Zip for Submission\nimport zipfile\n\ninference_args = {\n    \"selected_columns\": [i for i in range(543)],\n    \"columns\": [\"x\", \"y\", \"z\"]\n}\n\nwith open('inference_args.json', 'w') as f:\n    json.dump(inference_args, f)\n\nwith zipfile.ZipFile('submission.zip', 'w') as z:\n    z.write('inference.tflite')\n    z.write('inference_args.json')\n    \nprint('Submission created: submission.zip')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T17:45:44.086497Z","iopub.execute_input":"2026-01-21T17:45:44.087228Z","iopub.status.idle":"2026-01-21T17:45:50.276618Z","shell.execute_reply.started":"2026-01-21T17:45:44.087190Z","shell.execute_reply":"2026-01-21T17:45:50.275755Z"}},"outputs":[],"execution_count":null}]}