{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This great notebook is forked from @hengck23. \\\nLearnable paramter was just added to positional encoding as was suggested by @jaylau2333 in the comments. \\\nLearnable weight makes the model more flexible. As a results, the score has increased by 0.01 \\\nI just wanted to share how small parameter can change the score ","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nimport matplotlib.pyplot as plt\nimport matplotlib as mpl\nimport seaborn as sn\n\nfrom tqdm.notebook import tqdm\nfrom sklearn.model_selection import train_test_split, GroupShuffleSplit \n\nimport glob\nimport sys\nimport os\nimport math\nimport gc\nimport sys\nimport sklearn\nimport scipy","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:51:42.976072Z","iopub.execute_input":"2023-03-23T08:51:42.977274Z","iopub.status.idle":"2023-03-23T08:51:52.402083Z","shell.execute_reply.started":"2023-03-23T08:51:42.977224Z","shell.execute_reply":"2023-03-23T08:51:52.400974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot Config","metadata":{}},{"cell_type":"code","source":"# MatplotLib Global Settings\nmpl.rcParams.update(mpl.rcParamsDefault)\nmpl.rcParams['xtick.labelsize'] = 16\nmpl.rcParams['ytick.labelsize'] = 16\nmpl.rcParams['axes.labelsize'] = 18\nmpl.rcParams['axes.titlesize'] = 24","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:51:52.404524Z","iopub.execute_input":"2023-03-23T08:51:52.405343Z","iopub.status.idle":"2023-03-23T08:51:52.412447Z","shell.execute_reply.started":"2023-03-23T08:51:52.405303Z","shell.execute_reply":"2023-03-23T08:51:52.410955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"# If True, processing data from scratch\n# If False, loads preprocessed data\nPREPROCESS_DATA = False\nTRAIN_MODEL = True\n\nN_ROWS = 543\nN_DIMS = 3\nDIM_NAMES = ['x', 'y', 'z']\nSEED = 42\nNUM_CLASSES = 250\nIS_INTERACTIVE = os.environ['KAGGLE_KERNEL_RUN_TYPE'] == 'Interactive'\nVERBOSE = 1 if IS_INTERACTIVE else 2\n\nINPUT_SIZE = 32\n\nBATCH_ALL_SIGNS_N = 4\nBATCH_SIZE = 256\nN_EPOCHS = 160\nLR_MAX = 1e-3\nN_WARMUP_EPOCHS = 0\nWD_RATIO = 0.05\nMASK_VAL = 4237","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:51:52.414677Z","iopub.execute_input":"2023-03-23T08:51:52.415904Z","iopub.status.idle":"2023-03-23T08:51:52.444944Z","shell.execute_reply.started":"2023-03-23T08:51:52.415863Z","shell.execute_reply":"2023-03-23T08:51:52.444038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utils","metadata":{}},{"cell_type":"code","source":"# Prints Shape and Dtype For List Of Variables\ndef print_shape_dtype(l, names):\n    for e, n in zip(l, names):\n        print(f'{n} shape: {e.shape}, dtype: {e.dtype}')","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:51:52.446373Z","iopub.execute_input":"2023-03-23T08:51:52.446873Z","iopub.status.idle":"2023-03-23T08:51:52.455913Z","shell.execute_reply.started":"2023-03-23T08:51:52.446832Z","shell.execute_reply":"2023-03-23T08:51:52.454783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"# Read Training Data\nif IS_INTERACTIVE or not PREPROCESS_DATA:\n    train = pd.read_csv('/kaggle/input/asl-signs/train.csv').sample(int(5e3), random_state=SEED)\nelse:\n    train = pd.read_csv('/kaggle/input/asl-signs/train.csv')\n\nN_SAMPLES = len(train)\nprint(f'N_SAMPLES: {N_SAMPLES}')","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:51:52.459957Z","iopub.execute_input":"2023-03-23T08:51:52.460272Z","iopub.status.idle":"2023-03-23T08:51:52.840166Z","shell.execute_reply.started":"2023-03-23T08:51:52.46021Z","shell.execute_reply":"2023-03-23T08:51:52.838573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Add File Path","metadata":{}},{"cell_type":"code","source":"# Get complete file path to file\ndef get_file_path(path):\n    return f'/kaggle/input/asl-signs/{path}'\n\ntrain['file_path'] = train['path'].apply(get_file_path)","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:51:52.841849Z","iopub.execute_input":"2023-03-23T08:51:52.842203Z","iopub.status.idle":"2023-03-23T08:51:52.862512Z","shell.execute_reply.started":"2023-03-23T08:51:52.842166Z","shell.execute_reply":"2023-03-23T08:51:52.861575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ordinally Encode Sign","metadata":{}},{"cell_type":"code","source":"# Add ordinally Encoded Sign (assign number to each sign name)\ntrain['sign_ord'] = train['sign'].astype('category').cat.codes\n\n# Dictionaries to translate sign <-> ordinal encoded sign\nSIGN2ORD = train[['sign', 'sign_ord']].set_index('sign').squeeze().to_dict()\nORD2SIGN = train[['sign_ord', 'sign']].set_index('sign_ord').squeeze().to_dict()","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:51:52.864222Z","iopub.execute_input":"2023-03-23T08:51:52.86475Z","iopub.status.idle":"2023-03-23T08:51:52.886133Z","shell.execute_reply.started":"2023-03-23T08:51:52.864698Z","shell.execute_reply":"2023-03-23T08:51:52.885162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train.head(30))\ndisplay(train.info())","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:51:52.887647Z","iopub.execute_input":"2023-03-23T08:51:52.888016Z","iopub.status.idle":"2023-03-23T08:51:52.921525Z","shell.execute_reply.started":"2023-03-23T08:51:52.887981Z","shell.execute_reply":"2023-03-23T08:51:52.920433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Video Statistics","metadata":{}},{"cell_type":"code","source":"N = int(1e3) if (IS_INTERACTIVE or not PREPROCESS_DATA) else int(10e3)\nN_UNIQUE_FRAMES = np.zeros(N, dtype=np.uint16)\nN_MISSING_FRAMES = np.zeros(N, dtype=np.uint16)\nMAX_FRAME = np.zeros(N, dtype=np.uint16)\n\nPERCENTILES = [0.01, 0.05, 0.25, 0.50, 0.75, 0.95, 0.99, 0.999]\n\nfor idx, file_path in enumerate(tqdm(train['file_path'].sample(N, random_state=SEED))):\n    df = pd.read_parquet(file_path)\n    N_UNIQUE_FRAMES[idx] = df['frame'].nunique()\n    N_MISSING_FRAMES[idx] = (df['frame'].max() - df['frame'].min()) - df['frame'].nunique() + 1\n    MAX_FRAME[idx] = df['frame'].max()","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:51:52.923333Z","iopub.execute_input":"2023-03-23T08:51:52.923748Z","iopub.status.idle":"2023-03-23T08:52:17.864568Z","shell.execute_reply.started":"2023-03-23T08:51:52.923695Z","shell.execute_reply":"2023-03-23T08:52:17.863512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Number of unique frames in each video\ndisplay(pd.Series(N_UNIQUE_FRAMES).describe(percentiles=PERCENTILES).to_frame('N_UNIQUE_FRAMES'))\n\nplt.figure(figsize=(15,8))\nplt.title('Number of Unique Frames', size=24)\npd.Series(N_UNIQUE_FRAMES).plot(kind='hist', bins=128)\nplt.grid()\nxlim = math.ceil(plt.xlim()[1])\nplt.xlim(0, xlim)\nplt.xticks(np.arange(0, xlim+25, 25))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:52:17.866106Z","iopub.execute_input":"2023-03-23T08:52:17.867989Z","iopub.status.idle":"2023-03-23T08:52:18.487193Z","shell.execute_reply.started":"2023-03-23T08:52:17.867948Z","shell.execute_reply":"2023-03-23T08:52:18.486141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Number of missing frames, consecutive frames with missing intermediate frame, i.e. 1,2,4,5 -> 3 is missing\ndisplay(pd.Series(N_MISSING_FRAMES).describe(percentiles=PERCENTILES).to_frame('N_MISSING_FRAMES'))\n\nplt.figure(figsize=(15,8))\nplt.title('Number of Missing Frames', size=24)\npd.Series(N_MISSING_FRAMES).plot(kind='hist', bins=128)\nplt.grid()\nplt.xlim(0, math.ceil(plt.xlim()[1]))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:52:18.488529Z","iopub.execute_input":"2023-03-23T08:52:18.489168Z","iopub.status.idle":"2023-03-23T08:52:19.084119Z","shell.execute_reply.started":"2023-03-23T08:52:18.489132Z","shell.execute_reply":"2023-03-23T08:52:19.08316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Maximum frame number\ndisplay(pd.Series(MAX_FRAME).describe(percentiles=PERCENTILES).to_frame('MAX_FRAME'))\n\nplt.figure(figsize=(15,8))\nplt.title('Maximum Frames Index', size=24)\npd.Series(MAX_FRAME).plot(kind='hist', bins=128)\nplt.grid()\nplt.xlim(0, math.ceil(plt.xlim()[1]))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:52:19.088213Z","iopub.execute_input":"2023-03-23T08:52:19.090781Z","iopub.status.idle":"2023-03-23T08:52:19.606886Z","shell.execute_reply.started":"2023-03-23T08:52:19.090725Z","shell.execute_reply":"2023-03-23T08:52:19.605741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Process Data Tensorflow","metadata":{}},{"cell_type":"code","source":"# Source: https://www.kaggle.com/competitions/asl-signs/overview/evaluation\nROWS_PER_FRAME = 543  # number of landmarks per frame\n\ndef load_relevant_data_subset(pq_path):\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:52:19.608457Z","iopub.execute_input":"2023-03-23T08:52:19.608825Z","iopub.status.idle":"2023-03-23T08:52:19.616253Z","shell.execute_reply.started":"2023-03-23T08:52:19.608785Z","shell.execute_reply":"2023-03-23T08:52:19.614289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n    Tensorflow layer to process data in TFLite\n    Data needs to be processed in the model itself, so we can not use Python\n\"\"\" \nclass PreprocessLayer(tf.keras.layers.Layer):\n    def __init__(self):\n        super(PreprocessLayer, self).__init__()\n        \n    def pad_edge(self, t, repeats, side):\n        if side == 'LEFT':\n            return tf.concat((tf.repeat(t[:1], repeats=repeats, axis=0), t), axis=0)\n        elif side == 'RIGHT':\n            return tf.concat((t, tf.repeat(t[-1:], repeats=repeats, axis=0)), axis=0)\n    \n    @tf.function(\n        input_signature=(tf.TensorSpec(shape=[None,N_ROWS,N_DIMS], dtype=tf.float32),),\n    )\n    def call(self, data0):\n        # Number of Frames in Video\n        N_FRAMES0 = tf.shape(data0)[0]\n        \n        # Filter Out Frames With Empty Hand Data\n        frames_hands_nansum = tf.experimental.numpy.nanmean(tf.gather(data0, HAND_IDXS0, axis=1), axis=[1,2])\n        non_empty_frames_idxs = tf.where(frames_hands_nansum > 0)\n        non_empty_frames_idxs = tf.squeeze(non_empty_frames_idxs, axis=1)\n        data = tf.gather(data0, non_empty_frames_idxs, axis=0)\n        \n        # Cast Indices in float32 to be compatible with Tensorflow Lite\n        non_empty_frames_idxs = tf.cast(non_empty_frames_idxs, tf.float32) \n\n        \n        # Number of Frames in Filtered Video\n        N_FRAMES = tf.shape(data)[0]\n        \n        # Gather Relevant Landmark Columns\n        data = tf.gather(data, LANDMARK_IDXS0, axis=1)\n        \n        # Video fits in INPUT_SIZE\n        if N_FRAMES < INPUT_SIZE:\n            # Pad With -1 to indicate padding\n            non_empty_frames_idxs = tf.pad(non_empty_frames_idxs, [[0, INPUT_SIZE-N_FRAMES]], constant_values=-1)\n            # Pad Data With Zeros\n            data = tf.pad(data, [[0, INPUT_SIZE-N_FRAMES], [0,0], [0,0]], constant_values=0)\n            # Fill NaN Values With 0\n            data = tf.where(tf.math.is_nan(data), 0.0, data)\n            return data, non_empty_frames_idxs\n        # Video needs to be downsampled to INPUT_SIZE\n        else:\n            # Repeat\n            if N_FRAMES < INPUT_SIZE**2:\n                repeats = tf.math.floordiv(INPUT_SIZE * INPUT_SIZE, N_FRAMES0)\n                data = tf.repeat(data, repeats=repeats, axis=0)\n                non_empty_frames_idxs = tf.repeat(non_empty_frames_idxs, repeats=repeats, axis=0)\n\n            # Pad To Multiple Of Input Size\n            pool_size = tf.math.floordiv(len(data), INPUT_SIZE)\n            if tf.math.mod(len(data), INPUT_SIZE) > 0:\n                pool_size += 1\n\n            if pool_size == 1:\n                pad_size = (pool_size * INPUT_SIZE) - len(data)\n            else:\n                pad_size = (pool_size * INPUT_SIZE) % len(data)\n\n            # Pad Start/End with Start/End value\n            pad_left = tf.math.floordiv(pad_size, 2) + tf.math.floordiv(INPUT_SIZE, 2)\n            pad_right = tf.math.floordiv(pad_size, 2) + tf.math.floordiv(INPUT_SIZE, 2)\n            if tf.math.mod(pad_size, 2) > 0:\n                pad_right += 1\n\n            # Pad By Concatenating Left/Right Edge Values\n            data = self.pad_edge(data, pad_left, 'LEFT')\n            data = self.pad_edge(data, pad_right, 'RIGHT')\n\n            # Pad Non Empty Frame Indices\n            non_empty_frames_idxs = self.pad_edge(non_empty_frames_idxs, pad_left, 'LEFT')\n            non_empty_frames_idxs = self.pad_edge(non_empty_frames_idxs, pad_right, 'RIGHT')\n\n            # Reshape to Mean Pool\n            data = tf.reshape(data, [INPUT_SIZE, -1, N_COLS, N_DIMS])\n            non_empty_frames_idxs = tf.reshape(non_empty_frames_idxs, [INPUT_SIZE, -1])\n\n            # Mean Pool\n            data = tf.experimental.numpy.nanmean(data, axis=1)\n            non_empty_frames_idxs = tf.experimental.numpy.nanmean(non_empty_frames_idxs, axis=1)\n\n            # Fill NaN Values With 0\n            data = tf.where(tf.math.is_nan(data), 0.0, data)\n            \n            return data, non_empty_frames_idxs\n    \npreprocess_layer = PreprocessLayer()","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:52:19.623468Z","iopub.execute_input":"2023-03-23T08:52:19.623768Z","iopub.status.idle":"2023-03-23T08:52:19.653075Z","shell.execute_reply.started":"2023-03-23T08:52:19.623716Z","shell.execute_reply":"2023-03-23T08:52:19.651886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Interpolate NaN Values","metadata":{}},{"cell_type":"code","source":"\"\"\"\n    face: 0:468\n    left_hand: 468:489\n    pose: 489:522\n    right_hand: 522:544\n        \n\"\"\"\ndef get_data(file_path):\n    # Load Raw Data\n    data = load_relevant_data_subset(file_path)\n    # Process Data Using Tensorflow\n    data = preprocess_layer(data)\n    \n    return data","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:52:19.655726Z","iopub.execute_input":"2023-03-23T08:52:19.656126Z","iopub.status.idle":"2023-03-23T08:52:19.661328Z","shell.execute_reply.started":"2023-03-23T08:52:19.656088Z","shell.execute_reply":"2023-03-23T08:52:19.66031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"USE_TYPES = ['left_hand', 'pose', 'right_hand']\nSTART_IDX = 468\nLIPS_IDXS0 = np.array([\n        61, 185, 40, 39, 37, 0, 267, 269, 270, 409,\n        291, 146, 91, 181, 84, 17, 314, 405, 321, 375,\n        78, 191, 80, 81, 82, 13, 312, 311, 310, 415,\n        95, 88, 178, 87, 14, 317, 402, 318, 324, 308,\n    ])\n# Landmark indices in original data\nLEFT_HAND_IDXS0 = np.arange(468,489)\nRIGHT_HAND_IDXS0 = np.arange(522,543)\nPOSE_IDXS0 = np.arange(502, 512)\nLANDMARK_IDXS0 = np.concatenate((LIPS_IDXS0, LEFT_HAND_IDXS0, RIGHT_HAND_IDXS0, POSE_IDXS0))\nHAND_IDXS0 = np.concatenate((LEFT_HAND_IDXS0, RIGHT_HAND_IDXS0), axis=0)\nN_COLS = LANDMARK_IDXS0.size\n# Landmark indices in processed data\nLIPS_IDXS = np.argwhere(np.isin(LANDMARK_IDXS0, LIPS_IDXS0)).squeeze()\nLEFT_HAND_IDXS = np.argwhere(np.isin(LANDMARK_IDXS0, LEFT_HAND_IDXS0)).squeeze()\nRIGHT_HAND_IDXS = np.argwhere(np.isin(LANDMARK_IDXS0, RIGHT_HAND_IDXS0)).squeeze()\nHAND_IDXS = np.argwhere(np.isin(LANDMARK_IDXS0, HAND_IDXS0)).squeeze()\nPOSE_IDXS = np.argwhere(np.isin(LANDMARK_IDXS0, POSE_IDXS0)).squeeze()\n\nprint(f'# HAND_IDXS: {len(HAND_IDXS)}, N_COLS: {N_COLS}')","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:52:19.663135Z","iopub.execute_input":"2023-03-23T08:52:19.663971Z","iopub.status.idle":"2023-03-23T08:52:19.678419Z","shell.execute_reply.started":"2023-03-23T08:52:19.663932Z","shell.execute_reply":"2023-03-23T08:52:19.677371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LIPS_START = 0\nLEFT_HAND_START = LIPS_IDXS.size\nRIGHT_HAND_START = LEFT_HAND_START + LEFT_HAND_IDXS.size\nPOSE_START = RIGHT_HAND_START + RIGHT_HAND_IDXS.size\n\nprint(f'LIPS_START: {LIPS_START}, LEFT_HAND_START: {LEFT_HAND_START}, RIGHT_HAND_START: {RIGHT_HAND_START}, POSE_START: {POSE_START}')","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:52:19.679911Z","iopub.execute_input":"2023-03-23T08:52:19.680349Z","iopub.status.idle":"2023-03-23T08:52:19.687995Z","shell.execute_reply.started":"2023-03-23T08:52:19.680314Z","shell.execute_reply":"2023-03-23T08:52:19.686746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Dataset","metadata":{}},{"cell_type":"code","source":"# Get the full dataset\ndef get_x_y():\n    # Create arrays to save data\n    X = np.zeros([N_SAMPLES, INPUT_SIZE, N_COLS, N_DIMS], dtype=np.float32)\n    y = np.zeros([N_SAMPLES], dtype=np.int32)\n    NON_EMPTY_FRAME_IDXS = np.full([N_SAMPLES, INPUT_SIZE], -1, dtype=np.float32)\n\n    for row_idx, (file_path, sign_ord) in enumerate(tqdm(train[['file_path', 'sign_ord']].values)):\n        if row_idx % 5000 == 0:\n            print(f'Generated {row_idx}/{N_SAMPLES}')\n\n        data, non_empty_frame_idxs = get_data(file_path)\n        X[row_idx] = data\n        y[row_idx] = sign_ord\n        NON_EMPTY_FRAME_IDXS[row_idx] = non_empty_frame_idxs\n        if np.isnan(data).sum() > 0:\n            print(row_idx)\n            return data\n\n    # Save X/y\n    np.save('X.npy', X)\n    np.save('y.npy', y)\n    np.save('NON_EMPTY_FRAME_IDXS.npy', NON_EMPTY_FRAME_IDXS)\n    \n    return X, y, NON_EMPTY_FRAME_IDXS","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:52:19.689757Z","iopub.execute_input":"2023-03-23T08:52:19.690188Z","iopub.status.idle":"2023-03-23T08:52:19.699716Z","shell.execute_reply.started":"2023-03-23T08:52:19.690153Z","shell.execute_reply":"2023-03-23T08:52:19.698464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if PREPROCESS_DATA:\n    X, y, NON_EMPTY_FRAME_IDXS = get_x_y()\nelse:\n    X = np.load('/kaggle/input/gislr-dataset-public/X.npy')\n    y = np.load('/kaggle/input/gislr-dataset-public/y.npy')\n    NON_EMPTY_FRAME_IDXS = np.load('/kaggle/input/gislr-dataset-public/NON_EMPTY_FRAME_IDXS.npy')\n\nprint_shape_dtype([X, y, NON_EMPTY_FRAME_IDXS], ['X', 'y', 'NON_EMPTY_FRAME_IDXS'])\nprint(f'# NaN Values X: {np.isnan(X).sum()}')","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:52:19.701363Z","iopub.execute_input":"2023-03-23T08:52:19.701909Z","iopub.status.idle":"2023-03-23T08:52:42.037965Z","shell.execute_reply.started":"2023-03-23T08:52:19.70186Z","shell.execute_reply":"2023-03-23T08:52:42.036708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(pd.Series(y).value_counts().to_frame('Class Count').iloc[[0,1,2,3,4, -5,-4,-3,-2,-1]])","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:52:42.039419Z","iopub.execute_input":"2023-03-23T08:52:42.040003Z","iopub.status.idle":"2023-03-23T08:52:42.053234Z","shell.execute_reply.started":"2023-03-23T08:52:42.039962Z","shell.execute_reply":"2023-03-23T08:52:42.052061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Number Of Frames","metadata":{}},{"cell_type":"code","source":"# Vast majority of samples fits has less than 32 non empty frames\nN_EMPTY_FRAMES = (NON_EMPTY_FRAME_IDXS == -1).sum(axis=1) \nN_EMPTY_FRAMES_WATERFALL = []\nfor n in range(INPUT_SIZE):\n    N_EMPTY_FRAMES_WATERFALL.append(sum(N_EMPTY_FRAMES >= n))\n\nplt.figure(figsize=(15,8))\nplt.title('Waterfall Plot For Number Of Non Empty Frames')\npd.Series(N_EMPTY_FRAMES_WATERFALL).plot(kind='bar')\nplt.grid(axis='y')\nplt.xticks(np.arange(INPUT_SIZE), np.arange(1, INPUT_SIZE+1))\nplt.yticks(np.arange(0, 100e3+10e3, 10e3))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:52:42.054865Z","iopub.execute_input":"2023-03-23T08:52:42.055327Z","iopub.status.idle":"2023-03-23T08:52:48.121215Z","shell.execute_reply.started":"2023-03-23T08:52:42.055292Z","shell.execute_reply":"2023-03-23T08:52:48.120081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Statistics - Lips","metadata":{}},{"cell_type":"code","source":"# LIPS\nLIPS_MEAN_X = np.zeros([LIPS_IDXS.size], dtype=np.float32)\nLIPS_MEAN_Y = np.zeros([LIPS_IDXS.size], dtype=np.float32)\nLIPS_STD_X = np.zeros([LIPS_IDXS.size], dtype=np.float32)\nLIPS_STD_Y = np.zeros([LIPS_IDXS.size], dtype=np.float32)\n\nfig, axes = plt.subplots(3, 1, figsize=(15, N_DIMS*6))\n   \nfor col, ll in enumerate(tqdm( np.transpose(X[:,:,LIPS_IDXS], [2,3,0,1]).reshape([LIPS_IDXS.size, N_DIMS, -1]) )):\n    for dim, l in enumerate(ll):\n        v = l[np.nonzero(l)]\n        if dim == 0: # X\n            LIPS_MEAN_X[col] = v.mean()\n            LIPS_STD_X[col] = v.std()\n        if dim == 1: # Y\n            LIPS_MEAN_Y[col] = v.mean()\n            LIPS_STD_Y[col] = v.std()\n        \n        axes[dim].boxplot(v, notch=False, showfliers=False, positions=[col], whis=[5,95])\n        \nfor ax, dim_name in zip(axes, DIM_NAMES):\n    ax.set_title(f'Lips {dim_name.upper()} Dimension', size=24)\n    ax.tick_params(axis='x', labelsize=8)\n    ax.grid(axis='y')\n\nplt.subplots_adjust(hspace=0.50)\nplt.show()\n\nLIPS_MEAN = np.array([LIPS_MEAN_X, LIPS_MEAN_Y]).T\nLIPS_STD = np.array([LIPS_STD_X, LIPS_STD_Y]).T","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:52:48.122476Z","iopub.execute_input":"2023-03-23T08:52:48.122882Z","iopub.status.idle":"2023-03-23T08:53:01.303532Z","shell.execute_reply.started":"2023-03-23T08:52:48.122842Z","shell.execute_reply":"2023-03-23T08:53:01.302393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Statistics - Hands","metadata":{}},{"cell_type":"code","source":"# LEFT HAND\nLEFT_HANDS_MEAN_X = np.zeros([LEFT_HAND_IDXS.size], dtype=np.float32)\nLEFT_HANDS_MEAN_Y = np.zeros([LEFT_HAND_IDXS.size], dtype=np.float32)\nLEFT_HANDS_STD_X = np.zeros([LEFT_HAND_IDXS.size], dtype=np.float32)\nLEFT_HANDS_STD_Y = np.zeros([LEFT_HAND_IDXS.size], dtype=np.float32)\n# RIGHT HAND\nRIGHT_HANDS_MEAN_X = np.zeros([RIGHT_HAND_IDXS.size], dtype=np.float32)\nRIGHT_HANDS_MEAN_Y = np.zeros([RIGHT_HAND_IDXS.size], dtype=np.float32)\nRIGHT_HANDS_STD_X = np.zeros([RIGHT_HAND_IDXS.size], dtype=np.float32)\nRIGHT_HANDS_STD_Y = np.zeros([RIGHT_HAND_IDXS.size], dtype=np.float32)\n\nfig, axes = plt.subplots(3, 1, figsize=(15, N_DIMS*6))\n   \nfor col, ll in enumerate(tqdm( np.transpose(X[:,:,HAND_IDXS], [2,3,0,1]).reshape([HAND_IDXS.size, N_DIMS, -1]) )):\n    for dim, l in enumerate(ll):\n        v = l[np.nonzero(l)]\n        if dim == 0: # X\n            if col < RIGHT_HAND_IDXS.size: # LEFT HAND\n                LEFT_HANDS_MEAN_X[col] = v.mean()\n                LEFT_HANDS_STD_X[col] = v.std()\n            else:\n                RIGHT_HANDS_MEAN_X[col - LEFT_HAND_IDXS.size] = v.mean()\n                RIGHT_HANDS_STD_X[col - LEFT_HAND_IDXS.size] = v.std()\n        if dim == 1: # Y\n            if col < RIGHT_HAND_IDXS.size: # LEFT HAND\n                LEFT_HANDS_MEAN_Y[col] = v.mean()\n                LEFT_HANDS_STD_Y[col] = v.std()\n            else: # RIGHT HAND\n                RIGHT_HANDS_MEAN_Y[col - LEFT_HAND_IDXS.size] = v.mean()\n                RIGHT_HANDS_STD_Y[col - LEFT_HAND_IDXS.size] = v.std()\n        \n        axes[dim].boxplot(v, notch=False, showfliers=False, positions=[col], whis=[5,95])\n        \nfor ax, dim_name in zip(axes, DIM_NAMES):\n    ax.set_title(f'Hands {dim_name.upper()} Dimension', size=24)\n    ax.tick_params(axis='x', labelsize=8)\n    ax.grid(axis='y')\n\nplt.subplots_adjust(hspace=0.50)\nplt.show()\n\nLEFT_HANDS_MEAN = np.array([LEFT_HANDS_MEAN_X, LEFT_HANDS_MEAN_Y]).T\nLEFT_HANDS_STD = np.array([LEFT_HANDS_STD_X, LEFT_HANDS_STD_Y]).T\nRIGHT_HANDS_MEAN = np.array([RIGHT_HANDS_MEAN_X, RIGHT_HANDS_MEAN_Y]).T\nRIGHT_HANDS_STD = np.array([RIGHT_HANDS_STD_X, RIGHT_HANDS_STD_Y]).T","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:53:01.305171Z","iopub.execute_input":"2023-03-23T08:53:01.305723Z","iopub.status.idle":"2023-03-23T08:53:14.383418Z","shell.execute_reply.started":"2023-03-23T08:53:01.305687Z","shell.execute_reply":"2023-03-23T08:53:14.382438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Statistics - Pose","metadata":{}},{"cell_type":"code","source":"# POSE\nPOSE_MEAN_X = np.zeros([POSE_IDXS.size], dtype=np.float32)\nPOSE_MEAN_Y = np.zeros([POSE_IDXS.size], dtype=np.float32)\nPOSE_STD_X = np.zeros([POSE_IDXS.size], dtype=np.float32)\nPOSE_STD_Y = np.zeros([POSE_IDXS.size], dtype=np.float32)\n\nfig, axes = plt.subplots(3, 1, figsize=(15, N_DIMS*6))\n   \nfor col, ll in enumerate(tqdm( np.transpose(X[:,:,POSE_IDXS], [2,3,0,1]).reshape([POSE_IDXS.size, N_DIMS, -1]) )):\n    for dim, l in enumerate(ll):\n        v = l[np.nonzero(l)]\n        if dim == 0: # X\n            POSE_MEAN_X[col] = v.mean()\n            POSE_STD_X[col] = v.std()\n        if dim == 1: # Y\n            POSE_MEAN_Y[col] = v.mean()\n            POSE_STD_Y[col] = v.std()\n        \n        axes[dim].boxplot(v, notch=False, showfliers=False, positions=[col], whis=[5,95])\n        \nfor ax, dim_name in zip(axes, DIM_NAMES):\n    ax.set_title(f'Pose {dim_name.upper()} Dimension', size=24)\n    ax.tick_params(axis='x', labelsize=8)\n    ax.grid(axis='y')\n\nplt.subplots_adjust(hspace=0.50)\nplt.show()\n\nPOSE_MEAN = np.array([POSE_MEAN_X, POSE_MEAN_Y]).T\nPOSE_STD = np.array([POSE_STD_X, POSE_STD_Y]).T","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:53:14.384945Z","iopub.execute_input":"2023-03-23T08:53:14.385523Z","iopub.status.idle":"2023-03-23T08:53:17.967597Z","shell.execute_reply.started":"2023-03-23T08:53:14.385478Z","shell.execute_reply":"2023-03-23T08:53:17.966652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Samples","metadata":{}},{"cell_type":"code","source":"# Custom sampler to get a batch containing N times all signs\ndef get_train_batch_all_signs(X, y, NON_EMPTY_FRAME_IDXS, n=BATCH_ALL_SIGNS_N):\n    # Arrays to store batch in\n    X_batch = np.zeros([NUM_CLASSES*n, INPUT_SIZE, N_COLS, N_DIMS], dtype=np.float32)\n    y_batch = np.arange(0, NUM_CLASSES, step=1/n, dtype=np.float32).astype(np.int64)\n    non_empty_frame_idxs_batch = np.zeros([NUM_CLASSES*n, INPUT_SIZE], dtype=np.float32)\n    \n    # Dictionary mapping ordinally encoded sign to corresponding sample indices\n    CLASS2IDXS = {}\n    for i in range(NUM_CLASSES):\n        CLASS2IDXS[i] = np.argwhere(y == i).squeeze().astype(np.int32)\n            \n    while True:\n        # Fill batch arrays\n        for i in range(NUM_CLASSES):\n            idxs = np.random.choice(CLASS2IDXS[i], n)\n            X_batch[i*n:(i+1)*n] = X[idxs]\n            non_empty_frame_idxs_batch[i*n:(i+1)*n] = NON_EMPTY_FRAME_IDXS[idxs]\n        \n        yield { 'frames': X_batch, 'non_empty_frame_idxs': non_empty_frame_idxs_batch }, y_batch","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:53:17.969386Z","iopub.execute_input":"2023-03-23T08:53:17.970156Z","iopub.status.idle":"2023-03-23T08:53:17.979085Z","shell.execute_reply.started":"2023-03-23T08:53:17.970115Z","shell.execute_reply":"2023-03-23T08:53:17.978061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dummy_dataset = get_train_batch_all_signs(X, y, NON_EMPTY_FRAME_IDXS)\nX_batch, y_batch = next(dummy_dataset)\n\nfor k, v in X_batch.items():\n    print(f'{k} shape: {v.shape}, dtype: {v.dtype}')\n\n# Batch shape/dtype\nprint(f'y_batch shape: {y_batch.shape}, dtype: {y_batch.dtype}')\n# Verify each batch contains each sign exactly N times\ndisplay(pd.Series(y_batch).value_counts().to_frame('Counts'))","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:53:17.981054Z","iopub.execute_input":"2023-03-23T08:53:17.981445Z","iopub.status.idle":"2023-03-23T08:53:18.048199Z","shell.execute_reply.started":"2023-03-23T08:53:17.98138Z","shell.execute_reply":"2023-03-23T08:53:18.047128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Config","metadata":{}},{"cell_type":"code","source":"# Epsilon value for layer normalisation\nLAYER_NORM_EPS = 1e-6\n\n# Dense layer units for landmarks\nLIPS_UNITS = 384\nHANDS_UNITS = 384\nPOSE_UNITS = 384\n# final embedding and transformer embedding size\nUNITS = 384\n\n# Transformer\nNUM_BLOCKS = 4\nMLP_RATIO = 2\n\n# Dropout\nEMBEDDING_DROPOUT = 0.00\nMLP_DROPOUT_RATIO = 0.30\nCLASSIFIER_DROPOUT_RATIO = 0.10\n\n# Initiailizers\nINIT_HE_UNIFORM = tf.keras.initializers.he_uniform\nINIT_GLOROT_UNIFORM = tf.keras.initializers.glorot_uniform\nINIT_ZEROS = tf.keras.initializers.constant(0.0)\n# Activations\nGELU = tf.keras.activations.gelu","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:53:18.049602Z","iopub.execute_input":"2023-03-23T08:53:18.050589Z","iopub.status.idle":"2023-03-23T08:53:18.057966Z","shell.execute_reply.started":"2023-03-23T08:53:18.050549Z","shell.execute_reply":"2023-03-23T08:53:18.056656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transformer\n\nNeed to implement transformer from scratch as TFLite does not support the native TF implementation of MultiHeadAttention.","metadata":{}},{"cell_type":"code","source":"# based on: https://stackoverflow.com/questions/67342988/verifying-the-implementation-of-multihead-attention-in-transformer\n# replaced softmax with softmax layer to support masked softmax\ndef scaled_dot_product(q,k,v, softmax, attention_mask):\n    #calculates Q . K(transpose)\n    qkt = tf.matmul(q,k,transpose_b=True)\n    #caculates scaling factor\n    dk = tf.math.sqrt(tf.cast(q.shape[-1],dtype=tf.float32))\n    scaled_qkt = qkt/dk\n    softmax = softmax(scaled_qkt, mask=attention_mask)\n    \n    z = tf.matmul(softmax,v)\n    #shape: (m,Tx,depth), same shape as q,k,v\n    return z\n\nclass MultiHeadAttention(tf.keras.layers.Layer):\n    def __init__(self,d_model,num_of_heads):\n        super(MultiHeadAttention,self).__init__()\n        self.d_model = d_model\n        self.num_of_heads = num_of_heads\n        self.depth = d_model//num_of_heads\n        self.wq = [tf.keras.layers.Dense(self.depth) for i in range(num_of_heads)]\n        self.wk = [tf.keras.layers.Dense(self.depth) for i in range(num_of_heads)]\n        self.wv = [tf.keras.layers.Dense(self.depth) for i in range(num_of_heads)]\n        self.wo = tf.keras.layers.Dense(d_model)\n        self.softmax = tf.keras.layers.Softmax()\n        \n    def call(self,x, attention_mask):\n        \n        multi_attn = []\n        for i in range(self.num_of_heads):\n            Q = self.wq[i](x)\n            K = self.wk[i](x)\n            V = self.wv[i](x)\n            multi_attn.append(scaled_dot_product(Q,K,V, self.softmax, attention_mask))\n            \n        multi_head = tf.concat(multi_attn,axis=-1)\n        multi_head_attention = self.wo(multi_head)\n        return multi_head_attention","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:53:18.059431Z","iopub.execute_input":"2023-03-23T08:53:18.060395Z","iopub.status.idle":"2023-03-23T08:53:18.073137Z","shell.execute_reply.started":"2023-03-23T08:53:18.06036Z","shell.execute_reply":"2023-03-23T08:53:18.072169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Full Transformer\nclass Transformer(tf.keras.Model):\n    def __init__(self, num_blocks):\n        super(Transformer, self).__init__(name='transformer')\n        self.num_blocks = num_blocks\n    \n    def build(self, input_shape):\n        self.ln_1s = []\n        self.mhas = []\n        self.ln_2s = []\n        self.mlps = []\n        # Make Transformer Blocks\n        for i in range(self.num_blocks):\n            # First Layer Normalisation\n            self.ln_1s.append(tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS))\n            # Multi Head Attention\n            self.mhas.append(MultiHeadAttention(UNITS, 8))\n            # Second Layer Normalisation\n            self.ln_2s.append(tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS))\n            # Multi Layer Perception\n            self.mlps.append(tf.keras.Sequential([\n                tf.keras.layers.Dense(UNITS * MLP_RATIO, activation=GELU, kernel_initializer=INIT_GLOROT_UNIFORM),\n                tf.keras.layers.Dropout(MLP_DROPOUT_RATIO),\n                tf.keras.layers.Dense(UNITS, kernel_initializer=INIT_HE_UNIFORM),\n            ]))\n        \n    def call(self, x, attention_mask):\n        # Iterate input over transformer blocks\n        for ln_1, mha, ln_2, mlp in zip(self.ln_1s, self.mhas, self.ln_2s, self.mlps):\n            x1 = ln_1(x)\n            attention_output = mha(x1, attention_mask)\n            x2 = x1 + attention_output\n            x3 = ln_2(x2)\n            x3 = mlp(x3)\n            x = x3 + x2\n    \n        return x","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:53:18.074346Z","iopub.execute_input":"2023-03-23T08:53:18.074929Z","iopub.status.idle":"2023-03-23T08:53:18.089199Z","shell.execute_reply.started":"2023-03-23T08:53:18.074881Z","shell.execute_reply":"2023-03-23T08:53:18.088224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Landmark Embedding","metadata":{}},{"cell_type":"code","source":"class LandmarkEmbedding(tf.keras.Model):\n    def __init__(self, units, name):\n        super(LandmarkEmbedding, self).__init__(name=f'{name}_embedding')\n        self.units = units\n        \n    def build(self, input_shape):\n        # Embedding for missing landmark in frame, initizlied with zeros\n        self.empty_embedding = self.add_weight(\n            name=f'{self.name}_empty_embedding',\n            shape=[self.units],\n            initializer=INIT_ZEROS,\n        )\n        # Embedding\n        self.dense = tf.keras.Sequential([\n            tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_1', use_bias=True, kernel_initializer=INIT_GLOROT_UNIFORM, activation=GELU),\n            tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_2', use_bias=True, kernel_initializer=INIT_HE_UNIFORM),\n        ], name=f'{self.name}_dense')\n\n    def call(self, x):\n        return tf.where(\n                # Checks whether landmark is missing in frame\n                tf.reduce_sum(x, axis=2, keepdims=True) == 0,\n                # If so, the empty embedding is used\n                self.empty_embedding,\n                # Otherwise the landmark data is embedded\n                self.dense(x),\n            )","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:53:18.091053Z","iopub.execute_input":"2023-03-23T08:53:18.091454Z","iopub.status.idle":"2023-03-23T08:53:18.104085Z","shell.execute_reply.started":"2023-03-23T08:53:18.09142Z","shell.execute_reply":"2023-03-23T08:53:18.102966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Embedding","metadata":{}},{"cell_type":"code","source":"class Embedding(tf.keras.Model):\n    def __init__(self):\n        super(Embedding, self).__init__()\n        \n    def get_diffs(self, l):\n        S = l.shape[2]\n        other = tf.expand_dims(l, 3)\n        other = tf.repeat(other, S, axis=3)\n        other = tf.transpose(other, [0,1,3,2])\n        diffs = tf.expand_dims(l, 3) - other\n        diffs = tf.reshape(diffs, [-1, INPUT_SIZE, S*S])\n        return diffs\n\n    def build(self, input_shape):\n        # Positional Embedding, initialized with zeros\n        self.positional_embedding = tf.keras.layers.Embedding(INPUT_SIZE+1, UNITS, embeddings_initializer=INIT_ZEROS)\n        self.positional_weight = tf.Variable(1.)\n        # Embedding layer for Landmarks\n        self.lips_embedding = LandmarkEmbedding(LIPS_UNITS, 'lips')\n        self.left_hand_embedding = LandmarkEmbedding(HANDS_UNITS, 'left_hand')\n        self.right_hand_embedding = LandmarkEmbedding(HANDS_UNITS, 'right_hand')\n        self.pose_embedding = LandmarkEmbedding(POSE_UNITS, 'pose')\n        # Landmark Weights\n        self.landmark_weights = tf.Variable(tf.zeros([4], dtype=tf.float32), name='landmark_weights')\n        # Fully Connected Layers for combined landmarks\n        self.fc = tf.keras.Sequential([\n            tf.keras.layers.Dense(UNITS, name='fully_connected_1', use_bias=False, kernel_initializer=INIT_GLOROT_UNIFORM, activation=GELU),\n            tf.keras.layers.Dense(UNITS, name='fully_connected_2', use_bias=False, kernel_initializer=INIT_HE_UNIFORM),\n        ], name='fc')\n\n\n    def call(self, lips0, left_hand0, right_hand0, pose0, non_empty_frame_idxs, training=False):\n        # Lips\n        lips_embedding = self.lips_embedding(lips0)\n        # Left Hand\n        left_hand_embedding = self.left_hand_embedding(left_hand0)\n        # Right Hand\n        right_hand_embedding = self.right_hand_embedding(right_hand0)\n        # Pose\n        pose_embedding = self.pose_embedding(pose0)\n        # Merge Embeddings of all landmarks with mean pooling\n        x = tf.stack((lips_embedding, left_hand_embedding, right_hand_embedding, pose_embedding), axis=3)\n        # Merge Landmarks with trainable attention weights\n        x = x * tf.nn.softmax(self.landmark_weights)\n        x = tf.reduce_sum(x, axis=3)\n        # Fully Connected Layers\n        x = self.fc(x)\n        # Add Positional Embedding\n        normalised_non_empty_frame_idxs = tf.where(\n            tf.math.equal(non_empty_frame_idxs, -1.0),\n            INPUT_SIZE,\n            tf.cast(\n                non_empty_frame_idxs / tf.reduce_max(non_empty_frame_idxs, axis=1, keepdims=True) * INPUT_SIZE,\n                tf.int32,\n            ),\n        )\n        x = x + self.positional_weight*self.positional_embedding(normalised_non_empty_frame_idxs)\n        \n        return x","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:53:18.105806Z","iopub.execute_input":"2023-03-23T08:53:18.106161Z","iopub.status.idle":"2023-03-23T08:53:18.121431Z","shell.execute_reply.started":"2023-03-23T08:53:18.106127Z","shell.execute_reply":"2023-03-23T08:53:18.120255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model():\n    # Inputs\n    frames = tf.keras.layers.Input([INPUT_SIZE, N_COLS, N_DIMS], dtype=tf.float32, name='frames')\n    non_empty_frame_idxs = tf.keras.layers.Input([INPUT_SIZE], dtype=tf.float32, name='non_empty_frame_idxs')\n    # Padding Mask\n    mask = tf.cast(tf.math.not_equal(non_empty_frame_idxs, -1), tf.float32)\n    mask = tf.expand_dims(mask, axis=2)\n    \n    \"\"\"\n        left_hand: 468:489\n        pose: 489:522\n        right_hand: 522:543\n    \"\"\"\n    x = frames\n    x = tf.slice(x, [0,0,0,0], [-1,INPUT_SIZE, N_COLS, 2])\n    # LIPS\n    lips = tf.slice(x, [0,0,LIPS_START,0], [-1,INPUT_SIZE, 40, 2])\n    lips = tf.where(\n            tf.math.equal(lips, 0.0),\n            0.0,\n            (lips - LIPS_MEAN) / LIPS_STD,\n        )\n    lips = tf.reshape(lips, [-1, INPUT_SIZE, 40*2])\n    # LEFT HAND\n    left_hand = tf.slice(x, [0,0,40,0], [-1,INPUT_SIZE, 21, 2])\n    left_hand = tf.where(\n            tf.math.equal(left_hand, 0.0),\n            0.0,\n            (left_hand - LEFT_HANDS_MEAN) / LEFT_HANDS_STD,\n        )\n    left_hand = tf.reshape(left_hand, [-1, INPUT_SIZE, 21*2])\n    # RIGHT HAND\n    right_hand = tf.slice(x, [0,0,61,0], [-1,INPUT_SIZE, 21, 2])\n    right_hand = tf.where(\n            tf.math.equal(right_hand, 0.0),\n            0.0,\n            (right_hand - RIGHT_HANDS_MEAN) / RIGHT_HANDS_STD,\n        )\n    right_hand = tf.reshape(right_hand, [-1, INPUT_SIZE, 21*2])\n    # POSE\n    pose = tf.slice(x, [0,0,82,0], [-1,INPUT_SIZE, 10, 2])\n    pose = tf.where(\n            tf.math.equal(pose, 0.0),\n            0.0,\n            (pose - POSE_MEAN) / POSE_STD,\n        )\n    pose = tf.reshape(pose, [-1, INPUT_SIZE, 10*2])\n    \n    x = lips, left_hand, right_hand, pose\n        \n    x = Embedding()(lips, left_hand, right_hand, pose, non_empty_frame_idxs)\n    \n    # Encoder Transformer Blocks\n    x = Transformer(NUM_BLOCKS)(x, mask)\n    \n    # Pooling\n    x = tf.reduce_sum(x * mask, axis=1) / tf.reduce_sum(mask, axis=1)\n    # Classification Layer\n    x = tf.keras.layers.Dense(NUM_CLASSES, activation=tf.keras.activations.softmax, kernel_initializer=INIT_GLOROT_UNIFORM)(x)\n    \n    outputs = x\n    \n    # Create Tensorflow Model\n    model = tf.keras.models.Model(inputs=[frames, non_empty_frame_idxs], outputs=outputs)\n    \n    # Simple Categorical Crossentropy Loss\n    loss = tf.keras.losses.SparseCategoricalCrossentropy()\n    \n    # Adam Optimizer with weight decay\n    optimizer = tfa.optimizers.AdamW(learning_rate=1e-3, weight_decay=1e-5, clipnorm=1.0)\n    \n    # TopK Metrics\n    metrics = [\n        tf.keras.metrics.SparseCategoricalAccuracy(name='acc'),\n        tf.keras.metrics.SparseTopKCategoricalAccuracy(k=5, name='top_5_acc'),\n        tf.keras.metrics.SparseTopKCategoricalAccuracy(k=10, name='top_10_acc'),\n    ]\n    \n    model.compile(loss=loss, optimizer=optimizer, metrics=metrics)\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:53:18.123266Z","iopub.execute_input":"2023-03-23T08:53:18.123613Z","iopub.status.idle":"2023-03-23T08:53:18.141572Z","shell.execute_reply.started":"2023-03-23T08:53:18.123579Z","shell.execute_reply":"2023-03-23T08:53:18.140559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.backend.clear_session()\n\nmodel = get_model()","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:53:18.142871Z","iopub.execute_input":"2023-03-23T08:53:18.143297Z","iopub.status.idle":"2023-03-23T08:53:24.30276Z","shell.execute_reply.started":"2023-03-23T08:53:18.143263Z","shell.execute_reply":"2023-03-23T08:53:24.301528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot model summary\nmodel.summary(expand_nested=True)","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:53:24.305025Z","iopub.execute_input":"2023-03-23T08:53:24.305782Z","iopub.status.idle":"2023-03-23T08:53:24.478166Z","shell.execute_reply.started":"2023-03-23T08:53:24.305723Z","shell.execute_reply":"2023-03-23T08:53:24.477358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.utils.plot_model(model, show_shapes=True, show_dtype=True, show_layer_names=True, expand_nested=True, show_layer_activations=True)","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:53:24.47955Z","iopub.execute_input":"2023-03-23T08:53:24.47992Z","iopub.status.idle":"2023-03-23T08:53:25.312916Z","shell.execute_reply.started":"2023-03-23T08:53:24.479881Z","shell.execute_reply":"2023-03-23T08:53:25.311823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Weight Initialization","metadata":{}},{"cell_type":"code","source":"if TRAIN_MODEL:\n    N = 32\n    y_pred = model.predict(dummy_dataset, verbose=VERBOSE, steps=N).flatten()\n\n    plt.figure(figsize=(12,5))\n    plt.title(f'Softmax Output Initialized Model | µ={y_pred.mean():.3f}, σ={y_pred.std():.3f}', pad=25)\n    pd.Series(y_pred).plot(kind='hist', bins=128, label='Class Probability')\n    plt.xlim(0, max(y_pred) * 1.1)\n    plt.vlines([1 / NUM_CLASSES], 0, plt.ylim()[1], color='red', label=f'Random Guessing Baseline 1/NUM_CLASSES={1 / NUM_CLASSES:.3f}')\n    plt.grid()\n    plt.legend()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:53:25.315104Z","iopub.execute_input":"2023-03-23T08:53:25.315754Z","iopub.status.idle":"2023-03-23T08:53:39.607875Z","shell.execute_reply.started":"2023-03-23T08:53:25.315701Z","shell.execute_reply":"2023-03-23T08:53:39.606787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Learning Rate Scheduler","metadata":{}},{"cell_type":"code","source":"def lrfn(current_step, num_warmup_steps, lr_max, num_cycles=0.50, num_training_steps=N_EPOCHS):\n    \n    if current_step < num_warmup_steps:\n        if WARMUP_METHOD == 'log':\n            return lr_max * 0.10 ** (num_warmup_steps - current_step)\n        else:\n            return lr_max * 2 ** -(num_warmup_steps - current_step)\n    else:\n        progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))\n\n        return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))) * lr_max","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:53:39.609403Z","iopub.execute_input":"2023-03-23T08:53:39.61055Z","iopub.status.idle":"2023-03-23T08:53:39.61845Z","shell.execute_reply.started":"2023-03-23T08:53:39.610508Z","shell.execute_reply":"2023-03-23T08:53:39.617198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_lr_schedule(lr_schedule, epochs):\n    fig = plt.figure(figsize=(20, 10))\n    plt.plot([None] + lr_schedule + [None])\n    # X Labels\n    x = np.arange(1, epochs + 1)\n    x_axis_labels = [i if epochs <= 40 or i % 5 == 0 or i == 1 else None for i in range(1, epochs + 1)]\n    plt.xlim([1, epochs])\n    plt.xticks(x, x_axis_labels) # set tick step to 1 and let x axis start at 1\n    \n    # Increase y-limit for better readability\n    plt.ylim([0, max(lr_schedule) * 1.1])\n    \n    # Title\n    schedule_info = f'start: {lr_schedule[0]:.1E}, max: {max(lr_schedule):.1E}, final: {lr_schedule[-1]:.1E}'\n    plt.title(f'Step Learning Rate Schedule, {schedule_info}', size=18, pad=12)\n    \n    # Plot Learning Rates\n    for x, val in enumerate(lr_schedule):\n        if epochs <= 40 or x % 5 == 0 or x is epochs - 1:\n            if x < len(lr_schedule) - 1:\n                if lr_schedule[x - 1] < val:\n                    ha = 'right'\n                else:\n                    ha = 'left'\n            elif x == 0:\n                ha = 'right'\n            else:\n                ha = 'left'\n            plt.plot(x + 1, val, 'o', color='black');\n            offset_y = (max(lr_schedule) - min(lr_schedule)) * 0.02\n            plt.annotate(f'{val:.1E}', xy=(x + 1, val + offset_y), size=12, ha=ha)\n    \n    plt.xlabel('Epoch', size=16, labelpad=5)\n    plt.ylabel('Learning Rate', size=16, labelpad=5)\n    plt.grid()\n    plt.show()\n\n# Learning rate for encoder\nLR_SCHEDULE = [lrfn(step, num_warmup_steps=N_WARMUP_EPOCHS, lr_max=LR_MAX, num_cycles=0.50) for step in range(N_EPOCHS)]\n# Plot Learning Rate Schedule\nplot_lr_schedule(LR_SCHEDULE, epochs=N_EPOCHS)\n# Learning Rate Callback\nlr_callback = tf.keras.callbacks.LearningRateScheduler(lambda step: LR_SCHEDULE[step], verbose=1)","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:53:39.620272Z","iopub.execute_input":"2023-03-23T08:53:39.620705Z","iopub.status.idle":"2023-03-23T08:53:39.993699Z","shell.execute_reply.started":"2023-03-23T08:53:39.620666Z","shell.execute_reply":"2023-03-23T08:53:39.992717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Weight Decay Callback","metadata":{}},{"cell_type":"code","source":"# Custom callback to update weight decay with learning rate\nclass WeightDecayCallback(tf.keras.callbacks.Callback):\n    def __init__(self, wd_ratio=WD_RATIO):\n        self.step_counter = 0\n        self.wd_ratio = wd_ratio\n    \n    def on_epoch_begin(self, epoch, logs=None):\n        model.optimizer.weight_decay = model.optimizer.learning_rate * self.wd_ratio\n        print(f'learning rate: {model.optimizer.learning_rate.numpy():.2e}, weight decay: {model.optimizer.weight_decay.numpy():.2e}')","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:53:39.995501Z","iopub.execute_input":"2023-03-23T08:53:39.996235Z","iopub.status.idle":"2023-03-23T08:53:40.003297Z","shell.execute_reply.started":"2023-03-23T08:53:39.996195Z","shell.execute_reply":"2023-03-23T08:53:40.001991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Performance Benchmark","metadata":{}},{"cell_type":"code","source":"%%timeit -n 100\nif TRAIN_MODEL:\n    # Verify model prediction is <<<100ms\n    model.predict_on_batch({ 'frames': X[:1], 'non_empty_frame_idxs': NON_EMPTY_FRAME_IDXS[:1] })\n    pass","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:53:40.004871Z","iopub.execute_input":"2023-03-23T08:53:40.005378Z","iopub.status.idle":"2023-03-23T08:53:47.824469Z","shell.execute_reply.started":"2023-03-23T08:53:40.00534Z","shell.execute_reply":"2023-03-23T08:53:47.823415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"# Split data based on participant id\nsplitter = GroupShuffleSplit(test_size=0.10, n_splits=2, random_state=SEED)\n\nPARTICIPANT_IDS = pd.read_csv('/kaggle/input/asl-signs/train.csv')['participant_id']\n\ntrain_idxs, val_idxs = next(splitter.split(X, y, groups=PARTICIPANT_IDS))\n\nX_train = X[train_idxs]\nX_val = X[val_idxs]\nNON_EMPTY_FRAME_IDXS_TRAIN = NON_EMPTY_FRAME_IDXS[train_idxs]\nNON_EMPTY_FRAME_IDXS_VAL = NON_EMPTY_FRAME_IDXS[val_idxs]\ny_train = y[train_idxs]\ny_val = y[val_idxs]\n\nprint(f'Patient ID Intersection Train/Val: {set(PARTICIPANT_IDS[train_idxs]).intersection(PARTICIPANT_IDS[val_idxs])}')\nprint(f'X_train shape: {X_train.shape}, X_val shape: {X_val.shape}')\nprint(f'y_train shape: {y_train.shape}, y_val shape: {y_val.shape}')","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:53:47.833005Z","iopub.execute_input":"2023-03-23T08:53:47.833338Z","iopub.status.idle":"2023-03-23T08:53:51.001215Z","shell.execute_reply.started":"2023-03-23T08:53:47.833305Z","shell.execute_reply":"2023-03-23T08:53:51.000066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Verify Validation Dataset Covers All Signs\nprint(f'# Unique Signs in Validation Set: {pd.Series(y_val).nunique()}')\n# Value Counts\ndisplay(pd.Series(y_val).value_counts().to_frame('Count').iloc[[1,2,3,-3,-2,-1]])","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:53:51.002845Z","iopub.execute_input":"2023-03-23T08:53:51.003505Z","iopub.status.idle":"2023-03-23T08:53:51.016779Z","shell.execute_reply.started":"2023-03-23T08:53:51.003466Z","shell.execute_reply":"2023-03-23T08:53:51.015468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluate Initialzied Model","metadata":{}},{"cell_type":"code","source":"# Sanity Check\nif TRAIN_MODEL:\n    _ = model.evaluate({ 'frames': X_val, 'non_empty_frame_idxs': NON_EMPTY_FRAME_IDXS_VAL }, y_val, verbose=2)","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:53:51.018415Z","iopub.execute_input":"2023-03-23T08:53:51.01882Z","iopub.status.idle":"2023-03-23T08:54:00.232337Z","shell.execute_reply.started":"2023-03-23T08:53:51.018782Z","shell.execute_reply":"2023-03-23T08:54:00.231282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"if TRAIN_MODEL:\n    # Clear all models in GPU\n    tf.keras.backend.clear_session()\n\n    # Get new fresh model\n    model = get_model()\n    \n    # Sanity Check\n    model.summary()\n\n    # Actual Training\n    history = model.fit(\n            x=get_train_batch_all_signs(X_train, y_train, NON_EMPTY_FRAME_IDXS_TRAIN),\n            steps_per_epoch=len(X_train) // (NUM_CLASSES * BATCH_ALL_SIGNS_N),\n            epochs=N_EPOCHS,\n            # Only used for validation data since training data is a generator\n            batch_size=BATCH_SIZE,\n            validation_data=({ 'frames': X_val, 'non_empty_frame_idxs': NON_EMPTY_FRAME_IDXS_VAL }, y_val),\n            callbacks=[\n                lr_callback,\n                WeightDecayCallback(),\n            ],\n            verbose = VERBOSE,\n        )","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:54:00.235518Z","iopub.execute_input":"2023-03-23T08:54:00.236448Z","iopub.status.idle":"2023-03-23T08:58:37.269182Z","shell.execute_reply.started":"2023-03-23T08:54:00.236407Z","shell.execute_reply":"2023-03-23T08:58:37.268029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save Model Weights\nmodel.save_weights('model.h5')","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:58:37.271723Z","iopub.execute_input":"2023-03-23T08:58:37.27217Z","iopub.status.idle":"2023-03-23T08:58:37.433634Z","shell.execute_reply.started":"2023-03-23T08:58:37.27213Z","shell.execute_reply":"2023-03-23T08:58:37.432582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Validation Predictions\ny_val_pred = model.predict({ 'frames': X_val, 'non_empty_frame_idxs': NON_EMPTY_FRAME_IDXS_VAL }, verbose=2).argmax(axis=1)\n# Label\nlabels = [ORD2SIGN.get(i).replace(' ', '_') for i in range(NUM_CLASSES)]","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:58:37.43508Z","iopub.execute_input":"2023-03-23T08:58:37.435444Z","iopub.status.idle":"2023-03-23T08:58:46.312706Z","shell.execute_reply.started":"2023-03-23T08:58:37.435407Z","shell.execute_reply":"2023-03-23T08:58:46.311623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Landmark Attention Weights","metadata":{}},{"cell_type":"code","source":"# Landmark Weights\nweights = scipy.special.softmax(model.get_layer('embedding').weights[15])\nlandmarks = ['lips_embedding', 'left_hand_embedding', 'right_hand_embedding', 'pose_embedding']\n\n# Learned attention weights, initialized at uniform 25%\nfor w, lm in zip(weights, landmarks):\n    print(f'{lm} weight: {(w*100):.1f}%')","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:58:46.314063Z","iopub.execute_input":"2023-03-23T08:58:46.314441Z","iopub.status.idle":"2023-03-23T08:58:46.327551Z","shell.execute_reply.started":"2023-03-23T08:58:46.314398Z","shell.execute_reply":"2023-03-23T08:58:46.326429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Classification Report","metadata":{}},{"cell_type":"code","source":"# Classification report for all signs\nclassification_report = sklearn.metrics.classification_report(\n        y_val,\n        y_val_pred,\n        target_names=labels,\n        output_dict=True,\n    )\n# Round Data for better readability\nclassification_report = pd.DataFrame(classification_report).T\nclassification_report = classification_report.round(2)\nclassification_report = classification_report.astype({\n        'support': np.uint16,\n    })\n# Add signs\nclassification_report['sign'] = [e if e in SIGN2ORD else -1 for e in classification_report.index]\nclassification_report['sign_ord'] = classification_report['sign'].apply(SIGN2ORD.get).fillna(-1).astype(np.int16)\n# Sort on F1-score\nclassification_report = pd.concat((\n    classification_report.head(NUM_CLASSES).sort_values('f1-score', ascending=False),\n    classification_report.tail(3),\n))\n\npd.options.display.max_rows = 999\ndisplay(classification_report)","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:58:46.32957Z","iopub.execute_input":"2023-03-23T08:58:46.330544Z","iopub.status.idle":"2023-03-23T08:58:46.464123Z","shell.execute_reply.started":"2023-03-23T08:58:46.330503Z","shell.execute_reply":"2023-03-23T08:58:46.462938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training History","metadata":{}},{"cell_type":"code","source":"def plot_history_metric(metric, f_best=np.argmax, ylim=None, yscale=None, yticks=None):\n    plt.figure(figsize=(20, 10))\n    \n    values = history.history[metric]\n    N_EPOCHS = len(values)\n    val = 'val' in ''.join(history.history.keys())\n    # Epoch Ticks\n    if N_EPOCHS <= 20:\n        x = np.arange(1, N_EPOCHS + 1)\n    else:\n        x = [1, 5] + [10 + 5 * idx for idx in range((N_EPOCHS - 10) // 5 + 1)]\n\n    x_ticks = np.arange(1, N_EPOCHS+1)\n\n    # Validation\n    if val:\n        val_values = history.history[f'val_{metric}']\n        val_argmin = f_best(val_values)\n        plt.plot(x_ticks, val_values, label=f'val')\n\n    # summarize history for accuracy\n    plt.plot(x_ticks, values, label=f'train')\n    argmin = f_best(values)\n    plt.scatter(argmin + 1, values[argmin], color='red', s=75, marker='o', label=f'train_best')\n    if val:\n        plt.scatter(val_argmin + 1, val_values[val_argmin], color='purple', s=75, marker='o', label=f'val_best')\n\n    plt.title(f'Model {metric}', fontsize=24, pad=10)\n    plt.ylabel(metric, fontsize=20, labelpad=10)\n\n    if ylim:\n        plt.ylim(ylim)\n\n    if yscale is not None:\n        plt.yscale(yscale)\n        \n    if yticks is not None:\n        plt.yticks(yticks, fontsize=16)\n\n    plt.xlabel('epoch', fontsize=20, labelpad=10)        \n    plt.tick_params(axis='x', labelsize=8)\n    plt.xticks(x, fontsize=16) # set tick step to 1 and let x axis start at 1\n    plt.yticks(fontsize=16)\n    \n    plt.legend(prop={'size': 10})\n    plt.grid()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:58:46.465683Z","iopub.execute_input":"2023-03-23T08:58:46.466243Z","iopub.status.idle":"2023-03-23T08:58:46.481701Z","shell.execute_reply.started":"2023-03-23T08:58:46.466205Z","shell.execute_reply":"2023-03-23T08:58:46.480706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if TRAIN_MODEL:\n    plot_history_metric('loss', f_best=np.argmin)","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:58:46.483148Z","iopub.execute_input":"2023-03-23T08:58:46.483607Z","iopub.status.idle":"2023-03-23T08:58:46.852878Z","shell.execute_reply.started":"2023-03-23T08:58:46.483567Z","shell.execute_reply":"2023-03-23T08:58:46.851692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if TRAIN_MODEL:\n    plot_history_metric('acc', ylim=[0,1], yticks=np.arange(0.0, 1.1, 0.1))","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:58:46.85427Z","iopub.execute_input":"2023-03-23T08:58:46.855035Z","iopub.status.idle":"2023-03-23T08:58:47.206755Z","shell.execute_reply.started":"2023-03-23T08:58:46.854988Z","shell.execute_reply":"2023-03-23T08:58:47.205718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if TRAIN_MODEL:\n    plot_history_metric('top_5_acc', ylim=[0,1], yticks=np.arange(0.0, 1.1, 0.1))","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:58:47.208259Z","iopub.execute_input":"2023-03-23T08:58:47.209312Z","iopub.status.idle":"2023-03-23T08:58:47.570769Z","shell.execute_reply.started":"2023-03-23T08:58:47.209273Z","shell.execute_reply":"2023-03-23T08:58:47.569725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if TRAIN_MODEL:\n    plot_history_metric('top_10_acc', ylim=[0,1], yticks=np.arange(0.0, 1.1, 0.1))","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:58:47.572218Z","iopub.execute_input":"2023-03-23T08:58:47.572853Z","iopub.status.idle":"2023-03-23T08:58:47.928836Z","shell.execute_reply.started":"2023-03-23T08:58:47.572813Z","shell.execute_reply":"2023-03-23T08:58:47.927754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission\n\nSubmission code loosley based on [this notebook](https://www.kaggle.com/code/dschettler8845/gislr-learn-eda-baseline#baseline) by [Darien Schettler\n](https://www.kaggle.com/dschettler8845)","metadata":{}},{"cell_type":"code","source":"# TFLite model for submission\nclass TFLiteModel(tf.Module):\n    def __init__(self, model):\n        super(TFLiteModel, self).__init__()\n\n        # Load the feature generation and main models\n        self.preprocess_layer = preprocess_layer\n        self.model = model\n    \n    @tf.function(input_signature=[tf.TensorSpec(shape=[None, N_ROWS, N_DIMS], dtype=tf.float32, name='inputs')])\n    def __call__(self, inputs):\n        # Preprocess Data\n        x, non_empty_frame_idxs = self.preprocess_layer(inputs)\n        # Add Batch Dimension\n        x = tf.expand_dims(x, axis=0)\n        non_empty_frame_idxs = tf.expand_dims(non_empty_frame_idxs, axis=0)\n        # Make Prediction\n        outputs = self.model({ 'frames': x, 'non_empty_frame_idxs': non_empty_frame_idxs })\n        # Squeeze Output 1x250 -> 250\n        outputs = tf.squeeze(outputs, axis=0)\n\n        # Return a dictionary with the output tensor\n        return {'outputs': outputs}\n\n# Define TF Lite Model\ntflite_keras_model = TFLiteModel(model)\n\n# Sanity Check\ndemo_raw_data = load_relevant_data_subset(train['file_path'].values[5])\nprint(f'demo_raw_data shape: {demo_raw_data.shape}, dtype: {demo_raw_data.dtype}')\ndemo_output = tflite_keras_model(demo_raw_data)[\"outputs\"]\nprint(f'demo_output shape: {demo_output.shape}, dtype: {demo_output.dtype}')\ndemo_prediction = demo_output.numpy().argmax()\nprint(f'demo_prediction: {demo_prediction}, correct: {train.iloc[0][\"sign_ord\"]}')","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:58:47.930407Z","iopub.execute_input":"2023-03-23T08:58:47.931269Z","iopub.status.idle":"2023-03-23T08:58:49.834758Z","shell.execute_reply.started":"2023-03-23T08:58:47.931228Z","shell.execute_reply":"2023-03-23T08:58:49.83351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create Model Converter\nkeras_model_converter = tf.lite.TFLiteConverter.from_keras_model(tflite_keras_model)\n# Convert Model\ntflite_model = keras_model_converter.convert()\n# Write Model\nwith open('/kaggle/working/model.tflite', 'wb') as f:\n    f.write(tflite_model)\n    \n# Zip Model\n!zip submission.zip /kaggle/working/model.tflite","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:58:49.836431Z","iopub.execute_input":"2023-03-23T08:58:49.836828Z","iopub.status.idle":"2023-03-23T08:59:32.121804Z","shell.execute_reply.started":"2023-03-23T08:58:49.836788Z","shell.execute_reply":"2023-03-23T08:59:32.120476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Verify TFLite model can be loaded and used for prediction\n!pip install tflite-runtime\nimport tflite_runtime.interpreter as tflite\n\ninterpreter = tflite.Interpreter(\"/kaggle/working/model.tflite\")\nfound_signatures = list(interpreter.get_signature_list().keys())\nprediction_fn = interpreter.get_signature_runner(\"serving_default\")\n\noutput = prediction_fn(inputs=demo_raw_data)\nsign = output['outputs'].argmax()\n\nprint(\"PRED : \", ORD2SIGN.get(sign), f'[{sign}]')\nprint(\"TRUE : \", train.sign.values[0], f'[{train.sign_ord.values[0]}]')","metadata":{"execution":{"iopub.status.busy":"2023-03-23T08:59:32.124083Z","iopub.execute_input":"2023-03-23T08:59:32.124845Z","iopub.status.idle":"2023-03-23T08:59:44.022528Z","shell.execute_reply.started":"2023-03-23T08:59:32.124801Z","shell.execute_reply":"2023-03-23T08:59:44.021189Z"},"trusted":true},"execution_count":null,"outputs":[]}]}