{"cells":[{"cell_type":"markdown","metadata":{},"source":"# 1D CNN-Transformer for Sign Language Recognition - Improved Evaluation\n## Enhanced with Comprehensive Classification Metrics"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport matplotlib.pyplot as plt \nimport matplotlib as mpl\nimport seaborn as sns\n\nfrom tqdm.notebook import tqdm\nfrom sklearn.model_selection import train_test_split, GroupShuffleSplit \n\nimport glob, sys, os, math, gc, sklearn, scipy \nfrom scipy.stats import iqr\n\nprint(f'Tensorflow V{tf.__version__}')\nprint(f'Keras V{tf.keras.__version__}')\nprint(f'Python V{sys.version}')"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Configuration\nPREPROCESS_DATA = True\nTRAIN_MODEL = True\nUSE_VAL = True\n\nN_ROWS = 543\nN_DIMS = 3\nDIM_NAMES = ['x', 'y', 'z']\nSEED = 42\nNUM_CLASSES = 250\nIS_INTERACTIVE = os.environ.get('KAGGLE_KERNEL_RUN_TYPE', 'Interactive') == 'Interactive'\nVERBOSE = 1 if IS_INTERACTIVE else 2\n\nINPUT_SIZE = 64\nBATCH_ALL_SIGNS_N = 4\nBATCH_SIZE = 64 \nN_EPOCHS = 50\nLR_MAX = 1e-3\nN_WARMUP_EPOCHS = 0\nWD_RATIO = 0.05\nMASK_VAL = 4237\n\n# Set seeds for reproducibility\nnp.random.seed(SEED)\ntf.random.set_seed(SEED)"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Load data (placeholder - adjust paths for your Kaggle dataset)\ntry:\n    train = pd.read_csv('/kaggle/input/asl-signs/train.csv')\nexcept:\n    # For local testing, create dummy data\n    print(\"Warning: Could not load data from Kaggle path. Creating dummy data for demonstration.\")\n    train = pd.DataFrame({\n        'path': [f'dummy_{i}.parquet' for i in range(1000)],\n        'sign': [f'sign_{i % 250}' for i in range(1000)],\n        'participant_id': [i % 50 for i in range(1000)]\n    })\n\n# Create sign mappings\nSIGN2ORD = {s: i for i, s in enumerate(train['sign'].unique())}\nORD2SIGN = {i: s for s, i in SIGN2ORD.items()}\ntrain['sign_ord'] = train['sign'].map(SIGN2ORD)\n\nprint(f'Total samples: {len(train)}')\nprint(f'Unique signs: {len(SIGN2ORD)}')"},{"cell_type":"markdown","metadata":{},"source":"## Landmark Indices and Preprocessing"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Landmark indices\nLIPS_IDXS0 = np.array([\n      61, 185, 40, 39, 37, 0, 267, 269, 270, 409,\n      291, 146, 91, 181, 84, 17, 314, 405, 321, 375,\n      78, 191, 80, 81, 82, 13, 312, 311, 310, 415,\n      95, 88, 178, 87, 14, 317, 402, 318, 324, 308,\n  ])\n\nLEFT_HAND_IDXS0 = np.arange(468, 489)\nRIGHT_HAND_IDXS0 = np.arange(522, 543)\n\nPOSE_IDXS0 = np.array([489, 490, 492, 493, 494])\n\nLIPS_IDXS = np.argwhere(np.isin(np.arange(N_ROWS), LIPS_IDXS0)).squeeze()\nLEFT_HAND_IDXS = np.argwhere(np.isin(np.arange(N_ROWS), LEFT_HAND_IDXS0)).squeeze()\nPOSE_IDXS = np.argwhere(np.isin(np.arange(N_ROWS), POSE_IDXS0)).squeeze()\n\nLANDMARK_IDXS_LEFT_DOMINANT0 = np.concatenate((LIPS_IDXS0, LEFT_HAND_IDXS0, POSE_IDXS0))\nLANDMARK_IDXS_RIGHT_DOMINANT0 = np.concatenate((LIPS_IDXS0, RIGHT_HAND_IDXS0, POSE_IDXS0))\n\nN_COLS = LANDMARK_IDXS_LEFT_DOMINANT0.size\n\nLIPS_START = 0\nLEFT_HAND_START = LIPS_IDXS.size\nPOSE_START = LIPS_IDXS.size + LEFT_HAND_IDXS.size\n\nprint(f'Lips: {LIPS_IDXS.size}, Hands: {LEFT_HAND_IDXS.size}, Pose: {POSE_IDXS.size}')\nprint(f'Total landmarks: {N_COLS}')"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Normalization statistics (these should be computed from your training data)\n# Placeholder values - replace with actual statistics from your data\nLIPS_MEDIAN = np.zeros((40, 2))\nLIPS_IQR = np.ones((40, 2))\n\nLEFT_HANDS_MEDIAN = np.zeros((21, 2))\nLEFT_HANDS_IQR = np.ones((21, 2))\n\nPOSE_MEDIAN = np.zeros((5, 2))\nPOSE_IQR = np.ones((5, 2))"},{"cell_type":"markdown","metadata":{},"source":"## Data Loading and Preprocessing\n\nNote: This is a simplified version. In production, you would load actual parquet files."},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"def get_data(file_path):\n    \"\"\"Load and preprocess a single sample\"\"\"\n    # Placeholder - replace with actual parquet loading\n    # In real implementation: data = pd.read_parquet(file_path)\n    # For now, return random data matching expected shape\n    data = np.random.randn(INPUT_SIZE, N_COLS, N_DIMS).astype(np.float32)\n    non_empty = np.ones(INPUT_SIZE, dtype=np.float32)\n    return data, non_empty\n\ndef print_shape_dtype(arrays, names):\n    \"\"\"Helper to print array info\"\"\"\n    for arr, name in zip(arrays, names):\n        print(f'{name}: shape={arr.shape}, dtype={arr.dtype}')"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"def preprocess_data():\n    \"\"\"Preprocess data with 3-way split\"\"\"\n    print('Preprocessing Data with 3-Way Split (Train/Val/Test)...')\n    \n    # First Split: 90% Train, 10% Temp\n    splitter_1 = GroupShuffleSplit(test_size=0.10, n_splits=2, random_state=SEED)\n    PARTICIPANT_IDS = train['participant_id'].values\n    train_idxs, temp_idxs = next(splitter_1.split(train, train['sign'], groups=PARTICIPANT_IDS))\n    \n    train_df = train.iloc[train_idxs].reset_index(drop=True)\n    temp_df = train.iloc[temp_idxs].reset_index(drop=True)\n    \n    # Second Split: Split Temp 50/50 into Val/Test\n    splitter_2 = GroupShuffleSplit(test_size=0.50, n_splits=2, random_state=SEED)\n    TEMP_PARTICIPANT_IDS = temp_df['participant_id'].values\n    val_idxs, test_idxs = next(splitter_2.split(temp_df, temp_df['sign'], groups=TEMP_PARTICIPANT_IDS))\n    \n    val_df = temp_df.iloc[val_idxs].reset_index(drop=True)\n    test_df = temp_df.iloc[test_idxs].reset_index(drop=True)\n    \n    print(f'Train: {len(train_df)}, Val: {len(val_df)}, Test: {len(test_df)}')\n    \n    # Create memory-mapped arrays\n    from numpy.lib.format import open_memmap\n    \n    def create_mmap(name, shape, dtype):\n        if os.path.exists(name): os.remove(name)\n        return open_memmap(name, mode='w+', dtype=dtype, shape=shape)\n    \n    # Create arrays\n    X_train = create_mmap('X_train.npy', (len(train_df), INPUT_SIZE, N_COLS, N_DIMS), np.float32)\n    y_train = create_mmap('y_train.npy', (len(train_df),), np.int32)\n    ne_train = create_mmap('NON_EMPTY_FRAME_IDXS_TRAIN.npy', (len(train_df), INPUT_SIZE), np.float32)\n    \n    X_val = create_mmap('X_val.npy', (len(val_df), INPUT_SIZE, N_COLS, N_DIMS), np.float32)\n    y_val = create_mmap('y_val.npy', (len(val_df),), np.int32)\n    ne_val = create_mmap('NON_EMPTY_FRAME_IDXS_VAL.npy', (len(val_df), INPUT_SIZE), np.float32)\n    \n    X_test = create_mmap('X_test.npy', (len(test_df), INPUT_SIZE, N_COLS, N_DIMS), np.float32)\n    y_test = create_mmap('y_test.npy', (len(test_df),), np.int32)\n    ne_test = create_mmap('NON_EMPTY_FRAME_IDXS_TEST.npy', (len(test_df), INPUT_SIZE), np.float32)\n    \n    # Process data\n    def process_and_save(df, X_mmap, y_mmap, ne_mmap, desc):\n        for i, row in tqdm(df.iterrows(), total=len(df), desc=desc):\n            # Placeholder - in production, use actual file path\n            data, non_empty = get_data(row.get('path', f'dummy_{i}'))\n            X_mmap[i] = data\n            y_mmap[i] = row['sign_ord']\n            ne_mmap[i] = non_empty\n            \n            if i % 100 == 0: \n                X_mmap.flush()\n        X_mmap.flush(); y_mmap.flush(); ne_mmap.flush()\n    \n    process_and_save(train_df, X_train, y_train, ne_train, 'Processing Train')\n    process_and_save(val_df, X_val, y_val, ne_val, 'Processing Val')\n    process_and_save(test_df, X_test, y_test, ne_test, 'Processing Test')\n    \n    print('Preprocessing Complete!')"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Preprocess or load data\nif PREPROCESS_DATA:\n    preprocess_data()\n    ROOT_DIR = '.'\nelse:\n    ROOT_DIR = '/kaggle/input/gislr-dataset-public'\n\n# Load data\nif USE_VAL:\n    X_train = np.load(f'{ROOT_DIR}/X_train.npy', mmap_mode='r')\n    y_train = np.load(f'{ROOT_DIR}/y_train.npy', mmap_mode='r')\n    NON_EMPTY_FRAME_IDXS_TRAIN = np.load(f'{ROOT_DIR}/NON_EMPTY_FRAME_IDXS_TRAIN.npy', mmap_mode='r')\n    \n    X_val = np.load(f'{ROOT_DIR}/X_val.npy', mmap_mode='r') \n    y_val = np.load(f'{ROOT_DIR}/y_val.npy', mmap_mode='r')\n    NON_EMPTY_FRAME_IDXS_VAL = np.load(f'{ROOT_DIR}/NON_EMPTY_FRAME_IDXS_VAL.npy', mmap_mode='r')\n    \n    X_test = np.load(f'{ROOT_DIR}/X_test.npy', mmap_mode='r')\n    y_test = np.load(f'{ROOT_DIR}/y_test.npy', mmap_mode='r')\n    NON_EMPTY_FRAME_IDXS_TEST = np.load(f'{ROOT_DIR}/NON_EMPTY_FRAME_IDXS_TEST.npy', mmap_mode='r')\n    \n    y_val_oh = tf.one_hot(y_val, NUM_CLASSES)\n    validation_data = ({ 'frames': X_val, 'non_empty_frame_idxs': NON_EMPTY_FRAME_IDXS_VAL }, y_val_oh)\n\nprint_shape_dtype([X_train, y_train], ['X_train', 'y_train'])\nif USE_VAL:\n    print_shape_dtype([X_val, y_val, X_test, y_test], ['X_val', 'y_val', 'X_test', 'y_test'])"},{"cell_type":"markdown","metadata":{},"source":"## Model Architecture"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Model hyperparameters\nLAYER_NORM_EPS = 1e-6\nLIPS_UNITS = 192\nHANDS_UNITS = 192\nPOSE_UNITS = 192\nUNITS = 320\nNUM_BLOCKS = 2\nMLP_RATIO = 2\nEMBEDDING_DROPOUT = 0.20\nMLP_DROPOUT_RATIO = 0.40\nCLASSIFIER_DROPOUT_RATIO = 0.50\n\nINIT_HE_UNIFORM = tf.keras.initializers.he_uniform\nINIT_GLOROT_UNIFORM = tf.keras.initializers.glorot_uniform\nINIT_ZEROS = tf.keras.initializers.constant(0.0)\nGELU = tf.keras.activations.gelu"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"def get_motion_features(t):\n    \"\"\"Extract velocity and acceleration features\"\"\"\n    shape = tf.shape(t)\n    B, T = shape[0], shape[1]\n    L = shape[2] // 2\n    \n    t_reshaped = tf.reshape(t, [B, T, L, 2])\n    \n    # Calculate dx (velocity)\n    t_prev = tf.concat([t_reshaped[:, :1], t_reshaped[:, :-1]], axis=1)\n    dx = t_reshaped - t_prev\n    \n    # Calculate dx2 (acceleration)\n    dx_prev = tf.concat([dx[:, :1], dx[:, :-1]], axis=1)\n    dx2 = dx - dx_prev\n    \n    out = tf.concat([t_reshaped, dx, dx2], axis=-1)\n    out = tf.reshape(out, [B, T, L*6])\n    \n    # Mask out if original was 0\n    mask = tf.cast(tf.not_equal(t_reshaped, 0.0), tf.float32)\n    mask = tf.reduce_max(mask, axis=-1, keepdims=True)\n    out_reshaped = tf.reshape(out, [B, T, L, 6])\n    out_reshaped = out_reshaped * mask\n    out = tf.reshape(out_reshaped, [B, T, L*6])\n    \n    return out\n\ndef get_spatial_features(t):\n    \"\"\"Extract pairwise distance features\"\"\"\n    shape = tf.shape(t)\n    B, T = shape[0], shape[1]\n    L = shape[2] // 2\n    \n    t_reshaped = tf.reshape(t, [B, T, L, 2])\n    \n    # Pairwise differences\n    r1 = tf.expand_dims(t_reshaped, 3)\n    r2 = tf.expand_dims(t_reshaped, 2)\n    diffs = r1 - r2 \n    \n    # Distances\n    dists = tf.norm(diffs + 1e-6, axis=-1)\n    out = tf.reshape(dists, [B, T, L*L])\n    \n    # Masking\n    mask = tf.cast(tf.not_equal(t_reshaped, 0.0), tf.float32)\n    mask = tf.reduce_max(mask, axis=-1)\n    m1 = tf.expand_dims(mask, 3)\n    m2 = tf.expand_dims(mask, 2)\n    pairwise_mask = m1 * m2\n    out = out * tf.reshape(pairwise_mask, [B, T, L*L])\n    \n    return out"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"class LandmarkEmbedding(tf.keras.Model):\n    def __init__(self, units, name):\n        super(LandmarkEmbedding, self).__init__(name=f'{name}_embedding')\n        self.units = units\n        \n    def build(self, input_shape):\n        self.empty_embedding = self.add_weight(\n            name=f'{self.name}_empty_embedding',\n            shape=[self.units],\n            initializer=INIT_ZEROS,\n        )\n        self.dense = tf.keras.Sequential([\n            tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_1', \n                                use_bias=False, kernel_initializer=INIT_GLOROT_UNIFORM),\n            tf.keras.layers.Activation(GELU),\n            tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_2', \n                                use_bias=False, kernel_initializer=INIT_HE_UNIFORM),\n        ], name=f'{self.name}_dense')\n\n    def call(self, x):\n        return tf.where(\n            tf.reduce_sum(x, axis=2, keepdims=True) == 0,\n            self.empty_embedding,\n            self.dense(x),\n        )\n\nclass Embedding(tf.keras.Model):\n    def __init__(self):\n        super(Embedding, self).__init__()\n\n    def build(self, input_shape):\n        self.positional_embedding = tf.keras.layers.Embedding(\n            INPUT_SIZE+1, UNITS, embeddings_initializer=INIT_ZEROS\n        )\n        self.lips_embedding = LandmarkEmbedding(LIPS_UNITS, 'lips')\n        self.left_hand_embedding = LandmarkEmbedding(HANDS_UNITS, 'left_hand')\n        self.pose_embedding = LandmarkEmbedding(POSE_UNITS, 'pose')\n        self.landmark_weights = tf.Variable(\n            tf.zeros([3], dtype=tf.float32), name='landmark_weights'\n        )\n        self.fc = tf.keras.Sequential([\n            tf.keras.layers.Dense(UNITS, name='fully_connected_1', \n                                use_bias=False, kernel_initializer=INIT_GLOROT_UNIFORM),\n            tf.keras.layers.Activation(GELU),\n            tf.keras.layers.Dense(UNITS, name='fully_connected_2', \n                                use_bias=False, kernel_initializer=INIT_HE_UNIFORM),\n        ], name='fc')\n\n    def call(self, lips0, left_hand0, pose0, non_empty_frame_idxs, training=False):\n        lips_embedding = self.lips_embedding(lips0)\n        left_hand_embedding = self.left_hand_embedding(left_hand0)\n        pose_embedding = self.pose_embedding(pose0)\n        \n        # Weighted combination\n        x = tf.stack((lips_embedding, left_hand_embedding, pose_embedding), axis=3)\n        x = x * tf.nn.softmax(self.landmark_weights)\n        x = tf.reduce_sum(x, axis=3)\n        x = self.fc(x)\n        \n        # Positional embedding\n        max_frame_idxs = tf.clip_by_value(\n            tf.reduce_max(non_empty_frame_idxs, axis=1, keepdims=True), 1, np.inf\n        )\n        normalised_non_empty_frame_idxs = tf.where(\n            tf.math.equal(non_empty_frame_idxs, -1.0),\n            INPUT_SIZE,\n            tf.cast(non_empty_frame_idxs / max_frame_idxs * INPUT_SIZE, tf.int32),\n        )\n        x = x + self.positional_embedding(normalised_non_empty_frame_idxs)\n        \n        return x"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"class MultiHeadSelfAttention(tf.keras.layers.Layer):\n    def __init__(self, dim=256, num_heads=4, dropout=0, **kwargs):\n        super().__init__(**kwargs)\n        self.dim = dim\n        self.scale = self.dim ** -0.5\n        self.num_heads = num_heads\n        self.head_dim = dim // num_heads\n        self.qkv = tf.keras.layers.Dense(dim * 3, use_bias=False)\n        self.drop1 = tf.keras.layers.Dropout(dropout)\n        self.proj = tf.keras.layers.Dense(dim, use_bias=False)\n        self.supports_masking = True\n\n    def call(self, inputs, mask=None):\n        if mask is not None:\n            mask = mask[:, None, None, :]\n            \n        qkv = self.qkv(inputs)\n        qkv = tf.keras.layers.Permute((2, 1, 3))(\n            tf.keras.layers.Reshape((-1, self.num_heads, self.head_dim * 3))(qkv)\n        )\n        q, k, v = tf.split(qkv, [self.head_dim, self.head_dim, self.head_dim], axis=-1)\n        \n        attn = tf.matmul(q, k, transpose_b=True) * self.scale\n        \n        if mask is not None:\n            attn = attn + (1 - tf.cast(mask, attn.dtype)) * -1e9\n\n        attn = tf.nn.softmax(attn, axis=-1)\n        attn = self.drop1(attn)\n\n        x = attn @ v\n        x = tf.keras.layers.Reshape((-1, self.dim))(\n            tf.keras.layers.Permute((2, 1, 3))(x)\n        )\n        x = self.proj(x)\n        return x\n\ndef TransformerBlock(dim=256, num_heads=4, expand=4, attn_dropout=0.2, drop_rate=0.2, \n                    activation='swish'):\n    def apply(inputs):\n        x = inputs\n        x = tf.keras.layers.BatchNormalization(momentum=0.95)(x)\n        x = MultiHeadSelfAttention(dim=dim, num_heads=num_heads, dropout=attn_dropout)(x)\n        x = tf.keras.layers.Dropout(drop_rate, noise_shape=(None,1,1))(x)\n        x = tf.keras.layers.Add()([inputs, x])\n        attn_out = x\n\n        x = tf.keras.layers.BatchNormalization(momentum=0.95)(x)\n        x = tf.keras.layers.Dense(dim*expand, use_bias=False, activation=activation)(x)\n        x = tf.keras.layers.Dense(dim, use_bias=False)(x)\n        x = tf.keras.layers.Dropout(drop_rate, noise_shape=(None,1,1))(x)\n        x = tf.keras.layers.Add()([attn_out, x])\n        return x\n    return apply\n\ndef Conv1DBlock(dim, ksize, drop_rate=0.0, activation='swish'):\n    def apply(inputs):\n        x = inputs\n        x = tf.keras.layers.BatchNormalization(momentum=0.95)(x)\n        x = tf.keras.layers.Dense(dim, use_bias=False, activation=activation)(x)\n        x = tf.keras.layers.DepthwiseConv1D(\n            ksize, strides=1, padding='same', use_bias=False\n        )(x)\n        x = tf.keras.layers.BatchNormalization(momentum=0.95)(x)\n        x = tf.keras.layers.Activation(activation)(x)\n        x = tf.keras.layers.Dense(dim, use_bias=False)(x)\n        x = tf.keras.layers.Dropout(drop_rate, noise_shape=(None,1,1))(x)\n        x = tf.keras.layers.Add()([inputs, x])\n        return x\n    return apply"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"def get_model(max_len=INPUT_SIZE, dim=256, ksize=17):\n    \"\"\"Build the 1D CNN-Transformer model\"\"\"\n    # Inputs\n    frames = tf.keras.layers.Input(\n        [INPUT_SIZE, N_COLS, N_DIMS], dtype=tf.float32, name='frames'\n    )\n    non_empty_frame_idxs = tf.keras.layers.Input(\n        [INPUT_SIZE], dtype=tf.float32, name='non_empty_frame_idxs'\n    )\n\n    # Padding Mask\n    mask0 = tf.keras.layers.Lambda(\n        lambda x: tf.cast(tf.math.not_equal(x, -1), tf.float32), name='mask0'\n    )(non_empty_frame_idxs)\n\n    # Extract XY coordinates\n    x = tf.keras.layers.Lambda(\n        lambda t: tf.slice(t, [0,0,0,0], [-1,INPUT_SIZE, N_COLS, 2]), name='slice_xy'\n    )(frames)\n    \n    lips_raw = tf.keras.layers.Lambda(\n        lambda t: tf.slice(t, [0,0,LIPS_START,0], [-1,INPUT_SIZE, 40, 2]), name='slice_lips'\n    )(x)\n    left_hand_raw = tf.keras.layers.Lambda(\n        lambda t: tf.slice(t, [0,0,40,0], [-1,INPUT_SIZE, 21, 2]), name='slice_hand'\n    )(x)\n    pose_raw = tf.keras.layers.Lambda(\n        lambda t: tf.slice(t, [0,0,61,0], [-1,INPUT_SIZE, 5, 2]), name='slice_pose'\n    )(x)\n\n    # Process landmarks\n    lips = tf.keras.layers.Lambda(\n        lambda t: tf.where(tf.math.equal(t, 0.0), 0.0, \n                          (t - LIPS_MEDIAN) / LIPS_IQR), name='normalize_lips'\n    )(lips_raw)\n    lips = tf.keras.layers.Reshape((INPUT_SIZE, 40*2), name='reshape_lips')(lips)\n    lips = tf.keras.layers.Lambda(get_motion_features, name='motion_lips')(lips)\n\n    left_hand = tf.keras.layers.Lambda(\n        lambda t: tf.where(tf.math.equal(t, 0.0), 0.0, \n                          (t - LEFT_HANDS_MEDIAN) / LEFT_HANDS_IQR), \n        name='normalize_hand'\n    )(left_hand_raw)\n    left_hand = tf.keras.layers.Reshape((INPUT_SIZE, 21*2), name='reshape_hand')(left_hand)\n    left_hand_motion = tf.keras.layers.Lambda(\n        get_motion_features, name='motion_hand'\n    )(left_hand)\n    left_hand_spatial = tf.keras.layers.Lambda(\n        get_spatial_features, name='spatial_hand'\n    )(left_hand)\n    left_hand = tf.keras.layers.Concatenate(\n        axis=-1, name='concat_hand'\n    )([left_hand_motion, left_hand_spatial])\n\n    pose = tf.keras.layers.Lambda(\n        lambda t: tf.where(tf.math.equal(t, 0.0), 0.0, \n                          (t - POSE_MEDIAN) / POSE_IQR), name='normalize_pose'\n    )(pose_raw)\n    pose = tf.keras.layers.Reshape((INPUT_SIZE, 5*2), name='reshape_pose')(pose)\n    pose_motion = tf.keras.layers.Lambda(\n        get_motion_features, name='motion_pose'\n    )(pose)\n    pose_spatial = tf.keras.layers.Lambda(\n        get_spatial_features, name='spatial_pose'\n    )(pose)\n    pose = tf.keras.layers.Concatenate(\n        axis=-1, name='concat_pose'\n    )([pose_motion, pose_spatial])\n    \n    # Embedding\n    x = Embedding()(lips, left_hand, pose, non_empty_frame_idxs)\n    \n    # Stem\n    x = tf.keras.layers.Dense(dim, use_bias=False, name='stem_conv')(x)\n    x = tf.keras.layers.BatchNormalization(momentum=0.95, name='stem_bn')(x)\n    \n    # Progressive Stochastic Depth\n    dpr = [x for x in tf.linspace(0.0, 0.2, 8)]\n\n    # Block 1\n    x = Conv1DBlock(dim, ksize, drop_rate=dpr[0])(x)\n    x = Conv1DBlock(dim, ksize, drop_rate=dpr[1])(x)\n    x = Conv1DBlock(dim, ksize, drop_rate=dpr[2])(x)\n    x = TransformerBlock(dim, expand=2, drop_rate=dpr[3])(x)\n\n    # Block 2\n    x = Conv1DBlock(dim, ksize, drop_rate=dpr[4])(x)\n    x = Conv1DBlock(dim, ksize, drop_rate=dpr[5])(x)\n    x = Conv1DBlock(dim, ksize, drop_rate=dpr[6])(x)\n    x = TransformerBlock(dim, expand=2, drop_rate=dpr[7])(x)\n\n    # Head\n    x = tf.keras.layers.Dense(dim*2, activation=None, name='top_conv')(x)\n    x = tf.keras.layers.GlobalAveragePooling1D()(x)\n    x = tf.keras.layers.Dropout(0.4)(x)\n    x = tf.keras.layers.Dense(NUM_CLASSES, name='classifier')(x)\n    \n    outputs = x\n    model = tf.keras.models.Model(\n        inputs=[frames, non_empty_frame_idxs], outputs=outputs\n    )\n    \n    # Compile\n    loss = tf.keras.losses.CategoricalCrossentropy(\n        from_logits=True, label_smoothing=0.05\n    )\n    top5_acc = tf.keras.metrics.TopKCategoricalAccuracy(k=5, name='top5_acc')\n    optimizer = tf.keras.optimizers.AdamW(\n        learning_rate=1e-3, weight_decay=1e-5, clipnorm=1.0\n    )\n    \n    model.compile(loss=loss, optimizer=optimizer, metrics=['acc', top5_acc])\n    return model"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Build model\ntf.keras.backend.clear_session()\nmodel = get_model()\nmodel.summary()"},{"cell_type":"markdown","metadata":{},"source":"## Training"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"def get_train_batch_all_signs(X, y, NON_EMPTY_FRAME_IDXS, n=BATCH_ALL_SIGNS_N):\n    \"\"\"Custom batch generator ensuring all classes in each batch\"\"\"\n    CLASS2IDXS = {}\n    for i in range(NUM_CLASSES):\n        CLASS2IDXS[i] = np.argwhere(y == i).squeeze().astype(np.int32)\n            \n    while True:\n        X_batch = np.zeros([NUM_CLASSES*n, INPUT_SIZE, N_COLS, N_DIMS], dtype=np.float32)\n        y_batch = np.zeros([NUM_CLASSES*n, NUM_CLASSES], dtype=np.float32)\n        non_empty_batch = np.zeros([NUM_CLASSES*n, INPUT_SIZE], dtype=np.float32)\n\n        for i in range(NUM_CLASSES):\n            idxs = np.random.choice(CLASS2IDXS[i], n)\n            X_batch[i*n:(i+1)*n] = X[idxs]\n            non_empty_batch[i*n:(i+1)*n] = NON_EMPTY_FRAME_IDXS[idxs]\n            y_batch[i*n:(i+1)*n, i] = 1.0\n        \n        yield { 'frames': X_batch, 'non_empty_frame_idxs': non_empty_batch }, y_batch"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"def lrfn(current_step, num_warmup_steps, lr_max, num_cycles=0.50, \n         num_training_steps=N_EPOCHS):\n    \"\"\"Cosine learning rate schedule\"\"\"\n    if current_step < num_warmup_steps:\n        return lr_max * 2 ** -(num_warmup_steps - current_step)\n    else:\n        progress = float(current_step - num_warmup_steps) / \\\n                   float(max(1, num_training_steps - num_warmup_steps))\n        return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))) * lr_max\n\nLR_SCHEDULE = [\n    lrfn(step, num_warmup_steps=N_WARMUP_EPOCHS, lr_max=LR_MAX, num_cycles=0.50) \n    for step in range(N_EPOCHS)\n]\nlr_callback = tf.keras.callbacks.LearningRateScheduler(\n    lambda step: LR_SCHEDULE[step], verbose=1\n)"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"class WeightDecayCallback(tf.keras.callbacks.Callback):\n    \"\"\"Update weight decay with learning rate\"\"\"\n    def __init__(self, wd_ratio=WD_RATIO):\n        self.wd_ratio = wd_ratio\n    \n    def on_epoch_begin(self, epoch, logs=None):\n        self.model.optimizer.weight_decay = self.model.optimizer.learning_rate * self.wd_ratio\n        print(f'LR: {self.model.optimizer.learning_rate.numpy():.2e}, '\n              f'WD: {self.model.optimizer.weight_decay.numpy():.2e}')"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"if TRAIN_MODEL:\n    tf.keras.backend.clear_session()\n\n    steps_per_epoch = len(X_train) // (NUM_CLASSES * BATCH_ALL_SIGNS_N)\n    \n    model = get_model(dim=256)\n\n    early_stopping = tf.keras.callbacks.EarlyStopping(\n        monitor='val_loss', \n        patience=15,\n        restore_best_weights=True, \n        verbose=1\n    )\n\n    history = model.fit(\n        x=get_train_batch_all_signs(X_train, y_train, NON_EMPTY_FRAME_IDXS_TRAIN),\n        steps_per_epoch=steps_per_epoch,\n        epochs=N_EPOCHS,\n        validation_data=validation_data,\n        callbacks=[early_stopping, lr_callback, WeightDecayCallback()],\n        verbose=2,\n    )\n    \n    # Save weights\n    model.save_weights('model.weights.h5')\n    print('Model training complete!')"},{"cell_type":"markdown","metadata":{},"source":"## 📊 COMPREHENSIVE EVALUATION SECTION\n### Senior ML Engineer's Complete Classification Report"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Generate predictions for test set\nif USE_VAL and X_test is not None:\n    print(\"Generating predictions on test set...\")\n    \n    # Get predictions (logits)\n    test_logits = model.predict(\n        {'frames': X_test, 'non_empty_frame_idxs': NON_EMPTY_FRAME_IDXS_TEST}, \n        verbose=1\n    )\n    \n    # Convert to probabilities\n    test_probs = tf.nn.softmax(test_logits).numpy()\n    \n    # Get top-1 predictions\n    y_pred = np.argmax(test_probs, axis=1)\n    y_true = y_test\n    \n    print(f\"✓ Generated predictions for {len(y_test)} test samples\")"},{"cell_type":"markdown","metadata":{},"source":"### 1️⃣ Top-1 vs Top-5 Accuracy Comparison"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"def calculate_topk_accuracy(y_true, probabilities, k_values=[1, 3, 5, 10]):\n    \"\"\"\n    Calculate Top-K accuracy for multiple K values.\n    \n    Args:\n        y_true: Ground truth labels\n        probabilities: Prediction probabilities (N x num_classes)\n        k_values: List of K values to evaluate\n    \n    Returns:\n        Dictionary with Top-K accuracies\n    \"\"\"\n    results = {}\n    \n    for k in k_values:\n        # Get top-k predictions for each sample\n        top_k_preds = np.argsort(probabilities, axis=1)[:, -k:]\n        \n        # Check if true label is in top-k\n        correct = 0\n        for i, true_label in enumerate(y_true):\n            if true_label in top_k_preds[i]:\n                correct += 1\n        \n        accuracy = correct / len(y_true)\n        results[f'top_{k}'] = accuracy\n    \n    return results\n\n# Calculate accuracies\ntopk_results = calculate_topk_accuracy(y_true, test_probs, k_values=[1, 3, 5, 10, 20])\n\n# Display results\nprint(\"=\"*60)\nprint(\"TOP-K ACCURACY ANALYSIS\")\nprint(\"=\"*60)\nprint(f\"Dataset: {len(y_true)} test samples across {NUM_CLASSES} classes\\n\")\n\nfor metric, value in topk_results.items():\n    k = metric.split('_')[1]\n    print(f\"Top-{k:>2} Accuracy: {value*100:6.2f}%\")\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"INTERPRETATION:\")\nprint(\"=\"*60)\ntop1 = topk_results['top_1']\ntop5 = topk_results['top_5']\ngap = (top5 - top1) * 100\n\nprint(f\"Gap (Top-5 - Top-1): {gap:.2f}%\")\nif gap > 10:\n    print(\"→ Model is often 'close but not perfect' - considers similar signs\")\nelif gap > 5:\n    print(\"→ Model has moderate confusion between similar sign classes\")\nelse:\n    print(\"→ Model is highly confident - low confusion between classes\")\n\nprint(f\"\\nRandom Baseline (Top-1): {100/NUM_CLASSES:.2f}%\")\nprint(f\"Random Baseline (Top-5): {500/NUM_CLASSES:.2f}%\")"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Visualize Top-K Accuracy\nfig, ax = plt.subplots(1, 1, figsize=(12, 6))\n\nk_vals = [1, 3, 5, 10, 20]\naccuracies = [topk_results[f'top_{k}'] * 100 for k in k_vals]\n\n# Bar plot\nbars = ax.bar(range(len(k_vals)), accuracies, color='steelblue', alpha=0.8, edgecolor='black')\n\n# Add value labels on bars\nfor i, (bar, acc) in enumerate(zip(bars, accuracies)):\n    height = bar.get_height()\n    ax.text(bar.get_x() + bar.get_width()/2., height + 0.5,\n            f'{acc:.1f}%', ha='center', va='bottom', fontsize=11, fontweight='bold')\n\nax.set_xlabel('K (Top-K Predictions)', fontsize=13, fontweight='bold')\nax.set_ylabel('Accuracy (%)', fontsize=13, fontweight='bold')\nax.set_title(f'Top-K Accuracy Comparison ({NUM_CLASSES} Classes)', \n             fontsize=15, fontweight='bold', pad=20)\nax.set_xticks(range(len(k_vals)))\nax.set_xticklabels([f'Top-{k}' for k in k_vals])\nax.set_ylim([0, 105])\nax.grid(axis='y', alpha=0.3, linestyle='--')\n\n# Add random baseline line\nrandom_top1 = 100 / NUM_CLASSES\nax.axhline(y=random_top1, color='red', linestyle='--', linewidth=2, \n           label=f'Random Baseline (Top-1): {random_top1:.2f}%', alpha=0.7)\nax.legend(loc='lower right', fontsize=10)\n\nplt.tight_layout()\nplt.show()\n\nprint(f\"\\n📊 Insight: The gap between Top-1 ({topk_results['top_1']*100:.1f}%) and \"\n      f\"Top-5 ({topk_results['top_5']*100:.1f}%) is {gap:.1f}%\")\nprint(f\"   This indicates the model's second-choice predictions are valuable for {NUM_CLASSES} classes.\")"},{"cell_type":"markdown","metadata":{},"source":"### 2️⃣ Confusion Matrix Analysis - Top 10 Most Confused Pairs"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"from sklearn.metrics import confusion_matrix\n\ndef find_top_confused_pairs(y_true, y_pred, top_n=10):\n    \"\"\"\n    Find the top N most confused class pairs.\n    \n    Args:\n        y_true: Ground truth labels\n        y_pred: Predicted labels\n        top_n: Number of confused pairs to return\n    \n    Returns:\n        DataFrame with top confused pairs and confusion counts\n    \"\"\"\n    # Compute confusion matrix\n    cm = confusion_matrix(y_true, y_pred)\n    \n    # Find off-diagonal confusions (true misclassifications)\n    confusions = []\n    for i in range(len(cm)):\n        for j in range(len(cm)):\n            if i != j and cm[i, j] > 0:  # Off-diagonal elements\n                confusions.append({\n                    'true_class': i,\n                    'pred_class': j,\n                    'true_sign': ORD2SIGN.get(i, f'class_{i}'),\n                    'pred_sign': ORD2SIGN.get(j, f'class_{j}'),\n                    'count': cm[i, j],\n                    'true_total': cm[i, :].sum(),\n                })\n    \n    # Convert to DataFrame and calculate error rate\n    df_confusions = pd.DataFrame(confusions)\n    df_confusions['error_rate'] = df_confusions['count'] / df_confusions['true_total']\n    \n    # Sort by count and get top N\n    df_confusions = df_confusions.sort_values('count', ascending=False).head(top_n)\n    \n    return df_confusions, cm\n\n# Find top 10 confused pairs\ntop_confused, full_cm = find_top_confused_pairs(y_true, y_pred, top_n=10)\n\nprint(\"=\"*80)\nprint(\"TOP 10 MOST CONFUSED SIGN PAIRS\")\nprint(\"=\"*80)\nprint(\"\\nThese pairs represent the most common misclassifications:\\n\")\n\ndisplay(top_confused[[\n    'true_sign', 'pred_sign', 'count', 'true_total', 'error_rate'\n]].rename(columns={\n    'true_sign': 'True Sign',\n    'pred_sign': 'Predicted As',\n    'count': 'Mistakes',\n    'true_total': 'Total Samples',\n    'error_rate': 'Error Rate'\n}).style.format({\n    'Mistakes': '{:.0f}',\n    'Total Samples': '{:.0f}',\n    'Error Rate': '{:.2%}'\n}).background_gradient(subset=['Error Rate'], cmap='YlOrRd'))"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Visualize top confused pairs\nfig, ax = plt.subplots(figsize=(14, 8))\n\n# Create labels for confused pairs\nlabels = [f\"{row['true_sign']}\\n→\\n{row['pred_sign']}\" \n          for _, row in top_confused.iterrows()]\n\n# Plot horizontal bars\ny_pos = np.arange(len(labels))\ncounts = top_confused['count'].values\ncolors = plt.cm.Reds(np.linspace(0.4, 0.9, len(labels)))\n\nbars = ax.barh(y_pos, counts, color=colors, edgecolor='black', linewidth=1.5)\n\n# Add value labels\nfor i, (bar, count, error_rate) in enumerate(zip(bars, counts, top_confused['error_rate'])):\n    width = bar.get_width()\n    ax.text(width + max(counts)*0.01, bar.get_y() + bar.get_height()/2,\n            f'{int(count)} ({error_rate:.1%})',\n            ha='left', va='center', fontsize=10, fontweight='bold')\n\nax.set_yticks(y_pos)\nax.set_yticklabels(labels, fontsize=9)\nax.set_xlabel('Number of Misclassifications', fontsize=12, fontweight='bold')\nax.set_title('Top 10 Most Confused Sign Pairs\\n(True Sign → Predicted Sign)', \n             fontsize=14, fontweight='bold', pad=20)\nax.grid(axis='x', alpha=0.3, linestyle='--')\nax.invert_yaxis()  # Highest confusion at top\n\nplt.tight_layout()\nplt.show()\n\nprint(\"\\n💡 QUALITATIVE INSIGHTS:\")\nprint(\"=\"*60)\nprint(\"Look for patterns in confused pairs:\")\nprint(\"  • Similar handshapes (e.g., WHERE vs WHEN)\")\nprint(\"  • Similar movements or locations\")\nprint(\"  • Short vs. compound signs\")\nprint(\"  • One-handed vs. two-handed variations\")"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Create a focused confusion matrix for top confused classes\ndef plot_focused_confusion_matrix(y_true, y_pred, top_confused_df, title=\"Focused Confusion Matrix\"):\n    \"\"\"\n    Plot a confusion matrix focusing on the most confused classes.\n    \"\"\"\n    # Get unique classes involved in top confusions\n    confused_classes = set()\n    for _, row in top_confused_df.iterrows():\n        confused_classes.add(row['true_class'])\n        confused_classes.add(row['pred_class'])\n    \n    confused_classes = sorted(list(confused_classes))\n    \n    # Filter predictions to only these classes\n    mask = np.isin(y_true, confused_classes)\n    y_true_filtered = y_true[mask]\n    y_pred_filtered = y_pred[mask]\n    \n    # Compute confusion matrix\n    cm = confusion_matrix(y_true_filtered, y_pred_filtered, labels=confused_classes)\n    \n    # Normalize by row (true labels)\n    cm_normalized = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n    \n    # Plot\n    fig, ax = plt.subplots(figsize=(12, 10))\n    \n    # Use seaborn for better heatmap\n    sns.heatmap(cm_normalized, annot=True, fmt='.2f', cmap='YlOrRd', \n                xticklabels=[ORD2SIGN.get(c, f'C{c}') for c in confused_classes],\n                yticklabels=[ORD2SIGN.get(c, f'C{c}') for c in confused_classes],\n                cbar_kws={'label': 'Proportion of True Class'},\n                linewidths=0.5, linecolor='gray', ax=ax)\n    \n    ax.set_xlabel('Predicted Sign', fontsize=12, fontweight='bold')\n    ax.set_ylabel('True Sign', fontsize=12, fontweight='bold')\n    ax.set_title(title, fontsize=14, fontweight='bold', pad=20)\n    \n    plt.xticks(rotation=45, ha='right')\n    plt.yticks(rotation=0)\n    plt.tight_layout()\n    plt.show()\n    \n    return cm, confused_classes\n\n# Plot focused confusion matrix\nfocused_cm, focused_classes = plot_focused_confusion_matrix(\n    y_true, y_pred, top_confused,\n    title=f\"Confusion Matrix: Most Confused {len(set(top_confused['true_class'].tolist() + top_confused['pred_class'].tolist()))} Classes\"\n)\n\nprint(f\"\\n📊 Showing confusion patterns among {len(focused_classes)} most problematic classes\")\nprint(\"   Diagonal = correct predictions, Off-diagonal = confusions\")"},{"cell_type":"markdown","metadata":{},"source":"### 3️⃣ Per-Class F1 Score Analysis - Best & Worst Performers"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"from sklearn.metrics import classification_report, f1_score, precision_score, recall_score\n\ndef analyze_per_class_performance(y_true, y_pred):\n    \"\"\"\n    Comprehensive per-class performance analysis.\n    \n    Returns:\n        DataFrame with precision, recall, F1, support for each class\n    \"\"\"\n    # Get classification report as dict\n    report = classification_report(\n        y_true, y_pred, \n        labels=list(range(NUM_CLASSES)),\n        target_names=[ORD2SIGN.get(i, f'class_{i}') for i in range(NUM_CLASSES)],\n        output_dict=True,\n        zero_division=0\n    )\n    \n    # Convert to DataFrame\n    df = pd.DataFrame(report).T\n    \n    # Remove summary rows\n    df = df[~df.index.isin(['accuracy', 'macro avg', 'weighted avg'])]\n    \n    # Add sign ordinal\n    df['sign_ord'] = [SIGN2ORD.get(idx, -1) for idx in df.index]\n    \n    # Sort by F1 score\n    df = df.sort_values('f1-score', ascending=False)\n    \n    return df\n\n# Analyze performance\nper_class_performance = analyze_per_class_performance(y_true, y_pred)\n\nprint(\"=\"*80)\nprint(\"PER-CLASS PERFORMANCE ANALYSIS\")\nprint(\"=\"*80)\nprint(f\"\\nTotal Classes: {NUM_CLASSES}\")\nprint(f\"Overall Accuracy: {(y_true == y_pred).mean()*100:.2f}%\")\nprint(f\"Macro F1-Score: {f1_score(y_true, y_pred, average='macro')*100:.2f}%\")\nprint(f\"Weighted F1-Score: {f1_score(y_true, y_pred, average='weighted')*100:.2f}%\\n\")"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Display BEST 5 classes\nprint(\"\\n\" + \"=\"*80)\nprint(\"🏆 TOP 5 BEST PERFORMING SIGNS\")\nprint(\"=\"*80)\n\nbest_5 = per_class_performance.head(5)\ndisplay(best_5[['precision', 'recall', 'f1-score', 'support']].rename(columns={\n    'precision': 'Precision',\n    'recall': 'Recall',\n    'f1-score': 'F1-Score',\n    'support': 'Samples'\n}).style.format({\n    'Precision': '{:.3f}',\n    'Recall': '{:.3f}',\n    'F1-Score': '{:.3f}',\n    'Samples': '{:.0f}'\n}).background_gradient(subset=['F1-Score'], cmap='Greens'))\n\nprint(\"\\n💡 Why these signs perform well:\")\nprint(\"   Check if they have: distinctive movements, clear handshapes,\")\nprint(\"   unique spatial locations, or longer durations\")"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Display WORST 5 classes\nprint(\"\\n\" + \"=\"*80)\nprint(\"⚠️  TOP 5 WORST PERFORMING SIGNS\")\nprint(\"=\"*80)\n\nworst_5 = per_class_performance.tail(5)\ndisplay(worst_5[['precision', 'recall', 'f1-score', 'support']].rename(columns={\n    'precision': 'Precision',\n    'recall': 'Recall',\n    'f1-score': 'F1-Score',\n    'support': 'Samples'\n}).style.format({\n    'Precision': '{:.3f}',\n    'Recall': '{:.3f}',\n    'F1-Score': '{:.3f}',\n    'Samples': '{:.0f}'\n}).background_gradient(subset=['F1-Score'], cmap='Reds'))\n\nprint(\"\\n⚠️  Why these signs struggle:\")\nprint(\"   Likely causes: short duration, similar to other signs,\")\nprint(\"   subtle movements, or insufficient training samples\")"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Visualize F1-Score distribution\nfig, axes = plt.subplots(2, 2, figsize=(16, 12))\n\n# 1. F1-Score Histogram\nax1 = axes[0, 0]\nax1.hist(per_class_performance['f1-score'], bins=30, color='steelblue', \n         alpha=0.7, edgecolor='black')\nax1.axvline(per_class_performance['f1-score'].mean(), color='red', \n            linestyle='--', linewidth=2, label=f\"Mean: {per_class_performance['f1-score'].mean():.3f}\")\nax1.axvline(per_class_performance['f1-score'].median(), color='green', \n            linestyle='--', linewidth=2, label=f\"Median: {per_class_performance['f1-score'].median():.3f}\")\nax1.set_xlabel('F1-Score', fontsize=11, fontweight='bold')\nax1.set_ylabel('Number of Classes', fontsize=11, fontweight='bold')\nax1.set_title('F1-Score Distribution Across All Classes', fontsize=12, fontweight='bold')\nax1.legend()\nax1.grid(alpha=0.3)\n\n# 2. Best vs Worst Comparison\nax2 = axes[0, 1]\ncombined = pd.concat([\n    best_5.assign(category='Best 5'),\n    worst_5.assign(category='Worst 5')\n])\nsigns = combined.index.tolist()\nf1_scores = combined['f1-score'].tolist()\ncolors_list = ['green']*5 + ['red']*5\n\ny_pos = np.arange(len(signs))\nax2.barh(y_pos, f1_scores, color=colors_list, alpha=0.7, edgecolor='black')\nax2.set_yticks(y_pos)\nax2.set_yticklabels(signs, fontsize=9)\nax2.set_xlabel('F1-Score', fontsize=11, fontweight='bold')\nax2.set_title('Best 5 vs Worst 5 Signs (by F1-Score)', fontsize=12, fontweight='bold')\nax2.axvline(per_class_performance['f1-score'].mean(), color='black', \n            linestyle='--', alpha=0.5, label='Mean F1')\nax2.grid(axis='x', alpha=0.3)\nax2.legend()\n\n# 3. Precision vs Recall scatter\nax3 = axes[1, 0]\nscatter = ax3.scatter(per_class_performance['recall'], \n                     per_class_performance['precision'],\n                     c=per_class_performance['f1-score'],\n                     s=per_class_performance['support']*2,\n                     alpha=0.6, cmap='viridis', edgecolor='black', linewidth=0.5)\nax3.set_xlabel('Recall', fontsize=11, fontweight='bold')\nax3.set_ylabel('Precision', fontsize=11, fontweight='bold')\nax3.set_title('Precision vs Recall (size=support, color=F1)', fontsize=12, fontweight='bold')\nax3.plot([0, 1], [0, 1], 'r--', alpha=0.3, label='Perfect Balance')\nax3.grid(alpha=0.3)\nax3.legend()\nplt.colorbar(scatter, ax=ax3, label='F1-Score')\n\n# 4. Support vs F1-Score\nax4 = axes[1, 1]\nax4.scatter(per_class_performance['support'], \n           per_class_performance['f1-score'],\n           alpha=0.6, s=50, color='coral', edgecolor='black', linewidth=0.5)\nax4.set_xlabel('Number of Test Samples (Support)', fontsize=11, fontweight='bold')\nax4.set_ylabel('F1-Score', fontsize=11, fontweight='bold')\nax4.set_title('Sample Size vs Performance', fontsize=12, fontweight='bold')\nax4.grid(alpha=0.3)\n\n# Add correlation coefficient\ncorrelation = per_class_performance[['support', 'f1-score']].corr().iloc[0, 1]\nax4.text(0.05, 0.95, f'Correlation: {correlation:.3f}', \n        transform=ax4.transAxes, fontsize=10, verticalalignment='top',\n        bbox=dict(boxstyle='round', facecolor='wheat', alpha=0.5))\n\nplt.tight_layout()\nplt.show()\n\nprint(\"\\n📊 KEY METRICS SUMMARY:\")\nprint(\"=\"*60)\nprint(f\"Mean F1-Score:   {per_class_performance['f1-score'].mean():.3f}\")\nprint(f\"Median F1-Score: {per_class_performance['f1-score'].median():.3f}\")\nprint(f\"Std Dev:         {per_class_performance['f1-score'].std():.3f}\")\nprint(f\"Min F1-Score:    {per_class_performance['f1-score'].min():.3f}\")\nprint(f\"Max F1-Score:    {per_class_performance['f1-score'].max():.3f}\")\nprint(f\"\\nClasses with F1 > 0.90: {(per_class_performance['f1-score'] > 0.90).sum()} / {NUM_CLASSES}\")\nprint(f\"Classes with F1 < 0.50: {(per_class_performance['f1-score'] < 0.50).sum()} / {NUM_CLASSES}\")"},{"cell_type":"markdown","metadata":{},"source":"### 4️⃣ Error Analysis - Understanding Model Failures"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Analyze error patterns\ndef analyze_error_patterns(per_class_perf):\n    \"\"\"\n    Categorize classes by precision/recall patterns to understand error types.\n    \"\"\"\n    categories = []\n    \n    for sign, row in per_class_perf.iterrows():\n        precision = row['precision']\n        recall = row['recall']\n        f1 = row['f1-score']\n        \n        # Categorize\n        if precision > 0.8 and recall > 0.8:\n            category = 'Excellent'\n        elif precision > 0.7 and recall < 0.6:\n            category = 'High Precision, Low Recall (Missing true positives)'\n        elif recall > 0.7 and precision < 0.6:\n            category = 'High Recall, Low Precision (False positives)'\n        elif f1 < 0.5:\n            category = 'Poor Overall'\n        else:\n            category = 'Moderate'\n        \n        categories.append(category)\n    \n    per_class_perf['error_pattern'] = categories\n    return per_class_perf\n\nper_class_performance = analyze_error_patterns(per_class_performance)\n\n# Count categories\npattern_counts = per_class_performance['error_pattern'].value_counts()\n\nprint(\"=\"*80)\nprint(\"ERROR PATTERN ANALYSIS\")\nprint(\"=\"*80)\nprint(\"\\nDistribution of error patterns across classes:\\n\")\nprint(pattern_counts)\n\n# Visualize\nfig, ax = plt.subplots(figsize=(12, 6))\ncolors_map = {\n    'Excellent': 'green',\n    'Moderate': 'yellow',\n    'High Precision, Low Recall (Missing true positives)': 'orange',\n    'High Recall, Low Precision (False positives)': 'coral',\n    'Poor Overall': 'red'\n}\ncolors = [colors_map.get(cat, 'gray') for cat in pattern_counts.index]\n\nbars = ax.bar(range(len(pattern_counts)), pattern_counts.values, \n              color=colors, alpha=0.7, edgecolor='black', linewidth=1.5)\n\nax.set_xticks(range(len(pattern_counts)))\nax.set_xticklabels(pattern_counts.index, rotation=15, ha='right', fontsize=9)\nax.set_ylabel('Number of Classes', fontsize=11, fontweight='bold')\nax.set_title('Error Pattern Distribution', fontsize=13, fontweight='bold', pad=15)\nax.grid(axis='y', alpha=0.3)\n\n# Add value labels\nfor bar in bars:\n    height = bar.get_height()\n    ax.text(bar.get_x() + bar.get_width()/2., height,\n            f'{int(height)}',\n            ha='center', va='bottom', fontsize=10, fontweight='bold')\n\nplt.tight_layout()\nplt.show()\n\nprint(\"\\n💡 ACTIONABLE INSIGHTS:\")\nprint(\"=\"*60)\nif 'High Precision, Low Recall (Missing true positives)' in pattern_counts.index:\n    count = pattern_counts['High Precision, Low Recall (Missing true positives)']\n    print(f\"• {count} signs have high precision but low recall\")\n    print(\"  → Model is conservative - needs more diverse training data\")\nif 'High Recall, Low Precision (False positives)' in pattern_counts.index:\n    count = pattern_counts['High Recall, Low Precision (False positives)']\n    print(f\"• {count} signs have high recall but low precision\")\n    print(\"  → Model is too generous - needs better discrimination\")\nif 'Poor Overall' in pattern_counts.index:\n    count = pattern_counts['Poor Overall']\n    print(f\"• {count} signs perform poorly overall\")\n    print(\"  → Priority for data augmentation or model architecture changes\")"},{"cell_type":"markdown","metadata":{},"source":"### 5️⃣ Complete Classification Report Export"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Generate complete report\nprint(\"=\"*80)\nprint(\"COMPLETE CLASSIFICATION REPORT - ALL 250 CLASSES\")\nprint(\"=\"*80)\n\n# Add ranking\nper_class_performance['f1_rank'] = per_class_performance['f1-score'].rank(\n    ascending=False\n).astype(int)\n\n# Format for display\nreport_display = per_class_performance[[\n    'f1_rank', 'precision', 'recall', 'f1-score', 'support', 'error_pattern'\n]].copy()\n\nreport_display.columns = [\n    'Rank', 'Precision', 'Recall', 'F1-Score', 'Samples', 'Error Pattern'\n]\n\n# Save to CSV\nreport_display.to_csv('complete_classification_report.csv')\nprint(\"\\n✓ Complete report saved to: complete_classification_report.csv\")\n\n# Display summary statistics\nprint(\"\\n\" + \"=\"*80)\nprint(\"SUMMARY STATISTICS\")\nprint(\"=\"*80)\nprint(report_display[['Precision', 'Recall', 'F1-Score']].describe())\n\n# Show sample of middle performers\nprint(\"\\n\" + \"=\"*80)\nprint(\"SAMPLE: MIDDLE PERFORMERS (Ranks 120-130)\")\nprint(\"=\"*80)\ndisplay(report_display[\n    (report_display['Rank'] >= 120) & (report_display['Rank'] <= 130)\n].style.format({\n    'Rank': '{:.0f}',\n    'Precision': '{:.3f}',\n    'Recall': '{:.3f}',\n    'F1-Score': '{:.3f}',\n    'Samples': '{:.0f}'\n}).background_gradient(subset=['F1-Score'], cmap='YlOrRd'))"},{"cell_type":"markdown","metadata":{},"source":"### 📋 FINAL SUMMARY: Complete Classification Report\n\nThis notebook provides a **comprehensive evaluation** for the 250-class sign language recognition task:\n\n#### ✅ What We Analyzed:\n\n1. **Top-1 vs Top-5 Accuracy**\n   - Quantifies how often the correct sign is in the model's top predictions\n   - Reveals if the model is \"close but not perfect\" (important for 250 classes)\n\n2. **Confusion Matrix - Top 10 Confused Pairs**\n   - Identifies which sign pairs are most commonly confused\n   - Provides qualitative insights (e.g., WHERE vs WHEN)\n   - Helps understand **why** the model fails\n\n3. **Per-Class F1 Scores - Best 5 & Worst 5**\n   - Highlights which signs the model handles well/poorly\n   - Identifies patterns:\n     * Short signs vs compound signs\n     * Similar handshapes\n     * Specific movements or locations\n\n4. **Error Pattern Analysis**\n   - Categorizes errors by precision/recall patterns\n   - Identifies systematic issues requiring attention\n\n5. **Complete Classification Report**\n   - Full metrics for all 250 classes\n   - Exportable CSV for further analysis\n\n#### 🎯 Next Steps for Improvement:\n\nBased on this analysis, you can:\n- **Data Augmentation**: Focus on worst-performing classes\n- **Model Architecture**: Address specific confusion patterns\n- **Training Strategy**: Balance precision/recall tradeoffs\n- **Feature Engineering**: Enhance discrimination for confused pairs\n\n---\n\n**This is a production-ready evaluation framework suitable for ML research papers and model deployment decisions.**"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.8.0"}},"nbformat":4,"nbformat_minor":4}