{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":46105,"databundleVersionId":5087314,"sourceType":"competition"}],"dockerImageVersionId":30458,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n#import numpy as np # linear algebra\n#import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n#import os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n    #for filename in filenames:\n        #print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-06T10:34:34.518597Z","iopub.execute_input":"2023-05-06T10:34:34.519315Z","iopub.status.idle":"2023-05-06T10:34:34.544954Z","shell.execute_reply.started":"2023-05-06T10:34:34.519276Z","shell.execute_reply":"2023-05-06T10:34:34.543924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os, json, random, math, scipy, wandb\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm\nfrom sklearn.model_selection import StratifiedGroupKFold \nfrom types import SimpleNamespace\nfrom pathlib import Path","metadata":{"execution":{"iopub.status.busy":"2024-02-04T04:58:38.069850Z","iopub.execute_input":"2024-02-04T04:58:38.070858Z","iopub.status.idle":"2024-02-04T04:58:52.407561Z","shell.execute_reply.started":"2024-02-04T04:58:38.070771Z","shell.execute_reply":"2024-02-04T04:58:52.405286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    from kaggle_secrets import UserSecretsClient\n    user_secrets = UserSecretsClient()\n    secret_value_0 = user_secrets.get_secret(\"c3322906e8e18a9813781ef3a30884d19b7d5e48\")\n    wandb.login(key=secret_value_0)\nexcept:\n    wandb.login()","metadata":{"execution":{"iopub.status.busy":"2024-02-04T04:58:52.411286Z","iopub.execute_input":"2024-02-04T04:58:52.412778Z","iopub.status.idle":"2024-02-04T04:59:03.142741Z","shell.execute_reply.started":"2024-02-04T04:58:52.412658Z","shell.execute_reply":"2024-02-04T04:59:03.141238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROWS_PER_FRAME = 543\n\ndef load_relevant_data_subset(pq_path):\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2024-02-04T04:59:05.273495Z","iopub.execute_input":"2024-02-04T04:59:05.275231Z","iopub.status.idle":"2024-02-04T04:59:05.285206Z","shell.execute_reply.started":"2024-02-04T04:59:05.275131Z","shell.execute_reply":"2024-02-04T04:59:05.283410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Config and Setup","metadata":{}},{"cell_type":"code","source":"cfg = SimpleNamespace()","metadata":{"execution":{"iopub.status.busy":"2024-02-04T04:59:11.905539Z","iopub.execute_input":"2024-02-04T04:59:11.906085Z","iopub.status.idle":"2024-02-04T04:59:11.912857Z","shell.execute_reply.started":"2024-02-04T04:59:11.906038Z","shell.execute_reply":"2024-02-04T04:59:11.911209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"iskaggle = os.environ.get('KAGGLE_KERNEL_RUN_TYPE', '')\niskaggle","metadata":{"execution":{"iopub.status.busy":"2024-02-04T04:59:13.977818Z","iopub.execute_input":"2024-02-04T04:59:13.978348Z","iopub.status.idle":"2024-02-04T04:59:13.989871Z","shell.execute_reply.started":"2024-02-04T04:59:13.978302Z","shell.execute_reply":"2024-02-04T04:59:13.987987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_DIR         = Path('../data/') if not iskaggle else Path('/kaggle/input/asl-signs/')\nTRAIN_CSV_PATH   = DATA_DIR/'train.csv'\nLANDMARK_DIR     = DATA_DIR/'train_landmark_files'\nLABEL_MAP_PATH   = DATA_DIR/'sign_to_prediction_index_map.json'\nDATA_DIR","metadata":{"execution":{"iopub.status.busy":"2024-02-04T04:59:17.236736Z","iopub.execute_input":"2024-02-04T04:59:17.237584Z","iopub.status.idle":"2024-02-04T04:59:17.253365Z","shell.execute_reply.started":"2024-02-04T04:59:17.237500Z","shell.execute_reply":"2024-02-04T04:59:17.251477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cfg.PREPROCESS_DATA = False\ncfg.TRAIN_MODEL = True\ncfg.N_ROWS = 543\ncfg.N_DIMS = 3\ncfg.DIM_NAMES = ['x', 'y', 'z']\ncfg.SEED = 42\ncfg.NUM_CLASSES = 250\ncfg.IS_INTERACTIVE = True\ncfg.VERBOSE = 2\ncfg.INPUT_SIZE = 32\ncfg.BATCH_ALL_SIGNS_N = 4\ncfg.BATCH_SIZE = 256\ncfg.N_EPOCHS = 50\ncfg.LR_MAX = 1e-3\ncfg.N_WARMUP_EPOCHS = 0\ncfg.WD_RATIO = 0.05\ncfg.MASK_VAL = 4237","metadata":{"execution":{"iopub.status.busy":"2024-02-04T04:59:19.122994Z","iopub.execute_input":"2024-02-04T04:59:19.123849Z","iopub.status.idle":"2024-02-04T04:59:19.134515Z","shell.execute_reply.started":"2024-02-04T04:59:19.123767Z","shell.execute_reply":"2024-02-04T04:59:19.133300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(TRAIN_CSV_PATH)\nN_SAMPLES = len(train)\nN_SAMPLES","metadata":{"execution":{"iopub.status.busy":"2024-02-04T04:59:21.233815Z","iopub.execute_input":"2024-02-04T04:59:21.234342Z","iopub.status.idle":"2024-02-04T04:59:21.538391Z","shell.execute_reply.started":"2024-02-04T04:59:21.234299Z","shell.execute_reply":"2024-02-04T04:59:21.536271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Below, instead of updating the path on the train file, we create a symlink instead.","metadata":{}},{"cell_type":"code","source":"# Get complete file path to file\ndef get_file_path(path):\n    return f'/kaggle/input/asl-signs/{path}'\n\n!ln -s {LANDMARK_DIR} ./train_landmark_files\ntrain['file_path'] = train['path'].values\ntrain.head(2)","metadata":{"execution":{"iopub.status.busy":"2024-02-04T04:59:23.985641Z","iopub.execute_input":"2024-02-04T04:59:23.986194Z","iopub.status.idle":"2024-02-04T04:59:25.199206Z","shell.execute_reply.started":"2024-02-04T04:59:23.986146Z","shell.execute_reply":"2024-02-04T04:59:25.197688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(LABEL_MAP_PATH, 'r') as f:\n    data = json.load(f)\n\n# Create a dictionary to map the indexes to their corresponding values\nindex_to_value = {str(index): value for index, value in data.items()}\n\n# add a new column with the corresponding keys using map\ntrain['sign_ord'] = train['sign'].map(index_to_value)\ntrain.head(2)","metadata":{"execution":{"iopub.status.busy":"2024-02-04T04:59:26.684564Z","iopub.execute_input":"2024-02-04T04:59:26.685234Z","iopub.status.idle":"2024-02-04T04:59:26.730734Z","shell.execute_reply.started":"2024-02-04T04:59:26.685177Z","shell.execute_reply":"2024-02-04T04:59:26.729250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"value_to_index = {j:i for i,j in index_to_value.items()}","metadata":{"execution":{"iopub.status.busy":"2024-02-04T06:42:43.190708Z","iopub.execute_input":"2024-02-04T06:42:43.191689Z","iopub.status.idle":"2024-02-04T06:42:43.201406Z","shell.execute_reply.started":"2024-02-04T06:42:43.191590Z","shell.execute_reply":"2024-02-04T06:42:43.198729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Kfold","metadata":{}},{"cell_type":"markdown","source":"Below, we create K-folds using StratifiedGroupKFold, the idea is to have different participants in\ntrain and validation.","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedGroupKFold","metadata":{"execution":{"iopub.status.busy":"2024-02-04T04:59:28.831609Z","iopub.execute_input":"2024-02-04T04:59:28.832425Z","iopub.status.idle":"2024-02-04T04:59:28.837797Z","shell.execute_reply.started":"2024-02-04T04:59:28.832379Z","shell.execute_reply":"2024-02-04T04:59:28.836307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_PARTICIPANTS = train.participant_id.nunique()\nsgkf = StratifiedGroupKFold(n_splits=7, shuffle=True, random_state=43)\ntrain['fold'] = -1\nfor i, (train_idx, val_idx) in enumerate(sgkf.split(train.index, train.sign, train.participant_id)):\n    train.loc[val_idx, 'fold'] = i\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-04T04:59:31.916124Z","iopub.execute_input":"2024-02-04T04:59:31.916667Z","iopub.status.idle":"2024-02-04T04:59:32.676801Z","shell.execute_reply.started":"2024-02-04T04:59:31.916621Z","shell.execute_reply":"2024-02-04T04:59:32.674607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create indexes using fold `0` for now\ntrain_idxs = train.query(\"fold!=0\").index.values\nval_idxs = train.query(\"fold==0\").index.values","metadata":{"execution":{"iopub.status.busy":"2024-02-04T04:59:34.375854Z","iopub.execute_input":"2024-02-04T04:59:34.376384Z","iopub.status.idle":"2024-02-04T04:59:34.432144Z","shell.execute_reply.started":"2024-02-04T04:59:34.376337Z","shell.execute_reply":"2024-02-04T04:59:34.430475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.read_parquet(train.loc[0, \"file_path\"])\ndata[data['frame'] == 20]","metadata":{"execution":{"iopub.status.busy":"2024-02-04T04:59:36.356081Z","iopub.execute_input":"2024-02-04T04:59:36.357662Z","iopub.status.idle":"2024-02-04T04:59:36.566559Z","shell.execute_reply.started":"2024-02-04T04:59:36.357590Z","shell.execute_reply":"2024-02-04T04:59:36.564984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Process Data Tensorflow\n","metadata":{}},{"cell_type":"code","source":"# landmark indices in original data\nLIPS_IDXS0 = np.array([\n        61, 185, 40, 39, 37, 0, 267, 269, 270, 409,\n        291, 146, 91, 181, 84, 17, 314, 405, 321, 375,\n        78, 191, 80, 81, 82, 13, 312, 311, 310, 415,\n        95, 88, 178, 87, 14, 317, 402, 318, 324, 308,\n    ])\nLEFT_HAND_IDXS0  = np.arange(468,489)\nRIGHT_HAND_IDXS0 = np.arange(522,543)\nPOSE_IDXS0       = np.arange(502, 512)\nLANDMARK_IDXS0   = np.concatenate((LIPS_IDXS0, LEFT_HAND_IDXS0, RIGHT_HAND_IDXS0, POSE_IDXS0))\nHAND_IDXS0       = np.concatenate((LEFT_HAND_IDXS0, RIGHT_HAND_IDXS0), axis=0)\nN_COLS           = LANDMARK_IDXS0.size\nN_COLS, LANDMARK_IDXS0","metadata":{"execution":{"iopub.status.busy":"2024-02-04T04:59:39.462878Z","iopub.execute_input":"2024-02-04T04:59:39.463478Z","iopub.status.idle":"2024-02-04T04:59:39.482724Z","shell.execute_reply.started":"2024-02-04T04:59:39.463431Z","shell.execute_reply":"2024-02-04T04:59:39.481292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"HAND_IDXS0","metadata":{"execution":{"iopub.status.busy":"2024-02-03T13:31:16.059084Z","iopub.execute_input":"2024-02-03T13:31:16.059538Z","iopub.status.idle":"2024-02-03T13:31:16.067008Z","shell.execute_reply.started":"2024-02-03T13:31:16.059482Z","shell.execute_reply":"2024-02-03T13:31:16.065782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Landmark indices in processed data\nLIPS_IDXS       = np.argwhere(np.isin(LANDMARK_IDXS0, LIPS_IDXS0)).squeeze()\nLEFT_HAND_IDXS  = np.argwhere(np.isin(LANDMARK_IDXS0, LEFT_HAND_IDXS0)).squeeze()\nRIGHT_HAND_IDXS = np.argwhere(np.isin(LANDMARK_IDXS0, RIGHT_HAND_IDXS0)).squeeze()\nHAND_IDXS       = np.argwhere(np.isin(LANDMARK_IDXS0, HAND_IDXS0)).squeeze()\nPOSE_IDXS       = np.argwhere(np.isin(LANDMARK_IDXS0, POSE_IDXS0)).squeeze()\nHAND_IDXS","metadata":{"execution":{"iopub.status.busy":"2024-02-04T04:59:43.652829Z","iopub.execute_input":"2024-02-04T04:59:43.653396Z","iopub.status.idle":"2024-02-04T04:59:43.670411Z","shell.execute_reply.started":"2024-02-04T04:59:43.653348Z","shell.execute_reply":"2024-02-04T04:59:43.668526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# function to crunch the entire video to 32 non empty frames and each frame only contains 92 indices (considering only lips in entire face)\nclass PreprocessLayer(tf.keras.layers.Layer):\n    def __init__(self):\n        super(PreprocessLayer, self).__init__()\n        \n    def pad_edge(self, t, repeats, side):\n        if side == 'LEFT':\n            return tf.concat((tf.repeat(t[:1], repeats=repeats, axis=0), t), axis=0)\n        elif side == 'RIGHT':\n            return tf.concat((t, tf.repeat(t[-1:], repeats=repeats, axis=0)), axis=0)\n    \n    @tf.function(\n        input_signature=(tf.TensorSpec(shape=[None,cfg.N_ROWS,cfg.N_DIMS], dtype=tf.float32),),\n    )\n    def call(self, data0):\n        # Number of Frames in Video\n        N_FRAMES0 = tf.shape(data0)[0]\n        \n        # Keep only non-empty frames in data\n        frames_hands_nansum = tf.experimental.numpy.nanmean(tf.gather(data0, HAND_IDXS0, axis=1), axis=[1,2])\n        non_empty_frames_idxs = tf.where(frames_hands_nansum > 0)\n        non_empty_frames_idxs = tf.squeeze(non_empty_frames_idxs, axis=1)\n        data = tf.gather(data0, non_empty_frames_idxs, axis=0)\n        \n        non_empty_frames_idxs = tf.cast(non_empty_frames_idxs, tf.float32) \n        \n        # Number of non-empty frames\n        N_FRAMES = tf.shape(data)[0]\n        data = tf.gather(data, LANDMARK_IDXS0, axis=1)\n        \n        if N_FRAMES < cfg.INPUT_SIZE:\n            # Video fits in cfg.INPUT_SIZE\n            non_empty_frames_idxs = tf.pad(non_empty_frames_idxs, [[0, cfg.INPUT_SIZE-N_FRAMES]], constant_values=-1)\n            data = tf.pad(data, [[0, cfg.INPUT_SIZE-N_FRAMES], [0,0], [0,0]], constant_values=0)\n            data = tf.where(tf.math.is_nan(data), 0.0, data)\n            return data, non_empty_frames_idxs\n        else:\n            # Video needs to be downsampled to cfg.INPUT_SIZE\n            if N_FRAMES < cfg.INPUT_SIZE**2:\n                repeats = tf.math.floordiv(cfg.INPUT_SIZE * cfg.INPUT_SIZE, N_FRAMES0)\n                data = tf.repeat(data, repeats=repeats, axis=0)\n                non_empty_frames_idxs = tf.repeat(non_empty_frames_idxs, repeats=repeats, axis=0)\n\n            # Pad To Multiple Of Input Size\n            pool_size = tf.math.floordiv(len(data), cfg.INPUT_SIZE)\n            if tf.math.mod(len(data), cfg.INPUT_SIZE) > 0:\n                pool_size += 1\n            if pool_size == 1:\n                pad_size = (pool_size * cfg.INPUT_SIZE) - len(data)\n            else:\n                pad_size = (pool_size * cfg.INPUT_SIZE) % len(data)\n\n            # Pad Start/End with Start/End value\n            pad_left = tf.math.floordiv(pad_size, 2) + tf.math.floordiv(cfg.INPUT_SIZE, 2)\n            pad_right = tf.math.floordiv(pad_size, 2) + tf.math.floordiv(cfg.INPUT_SIZE, 2)\n            if tf.math.mod(pad_size, 2) > 0:\n                pad_right += 1\n\n            # Pad By Concatenating Left/Right Edge Values\n            data = self.pad_edge(data, pad_left, 'LEFT')\n            data = self.pad_edge(data, pad_right, 'RIGHT')\n\n            # Pad Non Empty Frame Indices\n            non_empty_frames_idxs = self.pad_edge(non_empty_frames_idxs, pad_left, 'LEFT')\n            non_empty_frames_idxs = self.pad_edge(non_empty_frames_idxs, pad_right, 'RIGHT')\n\n            # Reshape to Mean Pool\n            data = tf.reshape(data, [cfg.INPUT_SIZE, -1, N_COLS, cfg.N_DIMS])\n            non_empty_frames_idxs = tf.reshape(non_empty_frames_idxs, [cfg.INPUT_SIZE, -1])\n\n            # Mean Pool\n            data = tf.experimental.numpy.nanmean(data, axis=1)\n            non_empty_frames_idxs = tf.experimental.numpy.nanmean(non_empty_frames_idxs, axis=1)\n\n            # Fill NaN Values With 0\n            data = tf.where(tf.math.is_nan(data), 0.0, data)\n            \n            return data, non_empty_frames_idxs\n    \npreprocess_layer = PreprocessLayer()","metadata":{"execution":{"iopub.status.busy":"2024-02-04T04:59:44.248018Z","iopub.execute_input":"2024-02-04T04:59:44.249329Z","iopub.status.idle":"2024-02-04T04:59:44.299303Z","shell.execute_reply.started":"2024-02-04T04:59:44.249267Z","shell.execute_reply":"2024-02-04T04:59:44.297627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = load_relevant_data_subset(train.path[3])\nsample.shape # with empty frames and entire facial data","metadata":{"execution":{"iopub.status.busy":"2024-02-04T04:59:56.291518Z","iopub.execute_input":"2024-02-04T04:59:56.292199Z","iopub.status.idle":"2024-02-04T04:59:56.328688Z","shell.execute_reply.started":"2024-02-04T04:59:56.292097Z","shell.execute_reply":"2024-02-04T04:59:56.326791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data, non_empty_frames_idxs = preprocess_layer(sample)\ndata.shape # without empty frames and only lip , lh, rh, pose","metadata":{"execution":{"iopub.status.busy":"2024-02-04T04:59:56.712095Z","iopub.execute_input":"2024-02-04T04:59:56.713838Z","iopub.status.idle":"2024-02-04T04:59:58.600172Z","shell.execute_reply.started":"2024-02-04T04:59:56.713768Z","shell.execute_reply":"2024-02-04T04:59:58.598756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"non_empty_frames_idxs.shape","metadata":{"execution":{"iopub.status.busy":"2024-02-04T06:37:23.440444Z","iopub.execute_input":"2024-02-04T06:37:23.441582Z","iopub.status.idle":"2024-02-04T06:37:23.453440Z","shell.execute_reply.started":"2024-02-04T06:37:23.441509Z","shell.execute_reply":"2024-02-04T06:37:23.451473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LIPS_START = 0\nLEFT_HAND_START = LIPS_IDXS.size\nRIGHT_HAND_START = LEFT_HAND_START + LEFT_HAND_IDXS.size\nPOSE_START = RIGHT_HAND_START + RIGHT_HAND_IDXS.size","metadata":{"execution":{"iopub.status.busy":"2024-02-04T05:00:01.802532Z","iopub.execute_input":"2024-02-04T05:00:01.803081Z","iopub.status.idle":"2024-02-04T05:00:01.812732Z","shell.execute_reply.started":"2024-02-04T05:00:01.803037Z","shell.execute_reply":"2024-02-04T05:00:01.810781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_data(file_path):\n    data = load_relevant_data_subset(file_path)\n    data = preprocess_layer(data)\n    return data","metadata":{"execution":{"iopub.status.busy":"2024-02-04T05:00:05.418303Z","iopub.execute_input":"2024-02-04T05:00:05.419841Z","iopub.status.idle":"2024-02-04T05:00:05.428274Z","shell.execute_reply.started":"2024-02-04T05:00:05.419723Z","shell.execute_reply":"2024-02-04T05:00:05.426508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_x_y():\n    # Create arrays to save data\n    X = np.zeros([N_SAMPLES, cfg.INPUT_SIZE, N_COLS, cfg.N_DIMS], dtype=np.float32)\n    y = np.zeros([N_SAMPLES], dtype=np.int32)\n    NON_EMPTY_FRAME_IDXS = np.full([N_SAMPLES, cfg.INPUT_SIZE], -1, dtype=np.float32)\n\n    for row_idx, (file_path, sign_ord) in enumerate(tqdm(train[['file_path', 'sign_ord']].values)):\n\n        data, non_empty_frame_idxs = get_data(file_path)\n        X[row_idx] = data\n        y[row_idx] = sign_ord\n        NON_EMPTY_FRAME_IDXS[row_idx] = non_empty_frame_idxs\n        if np.isnan(data).sum() > 0: return data\n\n    # Save X/y\n    np.save('X.npy', X)\n    np.save('y.npy', y)\n    np.save('NON_EMPTY_FRAME_IDXS.npy', NON_EMPTY_FRAME_IDXS)\n    \n    return X, y, NON_EMPTY_FRAME_IDXS","metadata":{"execution":{"iopub.status.busy":"2024-02-04T05:00:06.826847Z","iopub.execute_input":"2024-02-04T05:00:06.827347Z","iopub.status.idle":"2024-02-04T05:00:06.840940Z","shell.execute_reply.started":"2024-02-04T05:00:06.827307Z","shell.execute_reply":"2024-02-04T05:00:06.838959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not cfg.PREPROCESS_DATA:\n    X, y, NON_EMPTY_FRAME_IDXS = get_x_y()\nelse:\n    X = np.load('/kaggle/working/X.npy')\n    y = np.load('/kaggle/working/y.npy')\n    NON_EMPTY_FRAME_IDXS = np.load('/kaggle/working/NON_EMPTY_FRAME_IDXS.npy')","metadata":{"execution":{"iopub.status.busy":"2024-02-04T09:02:19.254602Z","iopub.execute_input":"2024-02-04T09:02:19.255938Z","iopub.status.idle":"2024-02-04T09:02:19.377005Z","shell.execute_reply.started":"2024-02-04T09:02:19.255882Z","shell.execute_reply":"2024-02-04T09:02:19.374978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.shape, y.shape, NON_EMPTY_FRAME_IDXS.shape","metadata":{"execution":{"iopub.status.busy":"2024-02-04T05:49:54.129907Z","iopub.execute_input":"2024-02-04T05:49:54.130555Z","iopub.status.idle":"2024-02-04T05:49:54.149816Z","shell.execute_reply.started":"2024-02-04T05:49:54.130463Z","shell.execute_reply":"2024-02-04T05:49:54.148012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Statistics - Lips\n","metadata":{}},{"cell_type":"code","source":"# LIPS\nLIPS_MEAN_X  = np.zeros([LIPS_IDXS.size], dtype=np.float32)\nLIPS_MEAN_Y  = np.zeros([LIPS_IDXS.size], dtype=np.float32)\nLIPS_STD_X   = np.zeros([LIPS_IDXS.size], dtype=np.float32)\nLIPS_STD_Y   = np.zeros([LIPS_IDXS.size], dtype=np.float32)\n\nfor col, ll in enumerate(tqdm( np.transpose(X[:,:,LIPS_IDXS], [2,3,0,1]).reshape([LIPS_IDXS.size, cfg.N_DIMS, -1]) )):\n    for dim, l in enumerate(ll):\n        v = l[np.nonzero(l)]\n        if dim == 0: # X\n            LIPS_MEAN_X[col] = v.mean()\n            LIPS_STD_X[col] = v.std()\n        if dim == 1: # Y\n            LIPS_MEAN_Y[col] = v.mean()\n            LIPS_STD_Y[col] = v.std()\n        \nLIPS_MEAN = np.array([LIPS_MEAN_X, LIPS_MEAN_Y]).T\nLIPS_STD = np.array([LIPS_STD_X, LIPS_STD_Y]).T","metadata":{"execution":{"iopub.status.busy":"2024-02-04T05:50:17.808988Z","iopub.execute_input":"2024-02-04T05:50:17.809601Z","iopub.status.idle":"2024-02-04T05:50:32.073889Z","shell.execute_reply.started":"2024-02-04T05:50:17.809554Z","shell.execute_reply":"2024-02-04T05:50:32.072312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.save('/kaggle/working/lips_mean.npy' , LIPS_MEAN)\nnp.save('/kaggle/working/lips_std.npy' , LIPS_STD)","metadata":{"execution":{"iopub.status.busy":"2024-02-04T05:52:41.119392Z","iopub.execute_input":"2024-02-04T05:52:41.119975Z","iopub.status.idle":"2024-02-04T05:52:41.131004Z","shell.execute_reply.started":"2024-02-04T05:52:41.119924Z","shell.execute_reply":"2024-02-04T05:52:41.128439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Statistics - Hands","metadata":{}},{"cell_type":"code","source":"# LEFT HAND\nLEFT_HANDS_MEAN_X = np.zeros([LEFT_HAND_IDXS.size], dtype=np.float32)\nLEFT_HANDS_MEAN_Y = np.zeros([LEFT_HAND_IDXS.size], dtype=np.float32)\nLEFT_HANDS_STD_X = np.zeros([LEFT_HAND_IDXS.size], dtype=np.float32)\nLEFT_HANDS_STD_Y = np.zeros([LEFT_HAND_IDXS.size], dtype=np.float32)\n# RIGHT HAND\nRIGHT_HANDS_MEAN_X = np.zeros([RIGHT_HAND_IDXS.size], dtype=np.float32)\nRIGHT_HANDS_MEAN_Y = np.zeros([RIGHT_HAND_IDXS.size], dtype=np.float32)\nRIGHT_HANDS_STD_X = np.zeros([RIGHT_HAND_IDXS.size], dtype=np.float32)\nRIGHT_HANDS_STD_Y = np.zeros([RIGHT_HAND_IDXS.size], dtype=np.float32)\n\nfor col, ll in enumerate(tqdm( np.transpose(X[:,:,HAND_IDXS], [2,3,0,1]).reshape([HAND_IDXS.size, cfg.N_DIMS, -1]) )):\n    for dim, l in enumerate(ll):\n        v = l[np.nonzero(l)]\n        if dim == 0: # X\n            if col < RIGHT_HAND_IDXS.size: # LEFT HAND\n                LEFT_HANDS_MEAN_X[col] = v.mean()\n                LEFT_HANDS_STD_X[col] = v.std()\n            else:\n                RIGHT_HANDS_MEAN_X[col - LEFT_HAND_IDXS.size] = v.mean()\n                RIGHT_HANDS_STD_X[col - LEFT_HAND_IDXS.size] = v.std()\n        if dim == 1: # Y\n            if col < RIGHT_HAND_IDXS.size: # LEFT HAND\n                LEFT_HANDS_MEAN_Y[col] = v.mean()\n                LEFT_HANDS_STD_Y[col] = v.std()\n            else: # RIGHT HAND\n                RIGHT_HANDS_MEAN_Y[col - LEFT_HAND_IDXS.size] = v.mean()\n                RIGHT_HANDS_STD_Y[col - LEFT_HAND_IDXS.size] = v.std()\n        \nLEFT_HANDS_MEAN = np.array([LEFT_HANDS_MEAN_X, LEFT_HANDS_MEAN_Y]).T\nLEFT_HANDS_STD = np.array([LEFT_HANDS_STD_X, LEFT_HANDS_STD_Y]).T\nRIGHT_HANDS_MEAN = np.array([RIGHT_HANDS_MEAN_X, RIGHT_HANDS_MEAN_Y]).T\nRIGHT_HANDS_STD = np.array([RIGHT_HANDS_STD_X, RIGHT_HANDS_STD_Y]).T","metadata":{"execution":{"iopub.status.busy":"2024-02-04T05:53:25.436150Z","iopub.execute_input":"2024-02-04T05:53:25.436846Z","iopub.status.idle":"2024-02-04T05:53:40.141159Z","shell.execute_reply.started":"2024-02-04T05:53:25.436789Z","shell.execute_reply":"2024-02-04T05:53:40.138956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.save('/kaggle/working/lh_mean.npy' , LEFT_HANDS_MEAN)\nnp.save('/kaggle/working/lh_std.npy' , LEFT_HANDS_STD)\nnp.save('/kaggle/working/rh_mean.npy' , RIGHT_HANDS_MEAN)\nnp.save('/kaggle/working/rh_std.npy' , RIGHT_HANDS_STD)","metadata":{"execution":{"iopub.status.busy":"2024-02-04T05:54:29.427093Z","iopub.execute_input":"2024-02-04T05:54:29.430718Z","iopub.status.idle":"2024-02-04T05:54:29.445428Z","shell.execute_reply.started":"2024-02-04T05:54:29.430549Z","shell.execute_reply":"2024-02-04T05:54:29.443894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Statistics - Pose","metadata":{}},{"cell_type":"code","source":"# POSE\nPOSE_MEAN_X = np.zeros([POSE_IDXS.size], dtype=np.float32)\nPOSE_MEAN_Y = np.zeros([POSE_IDXS.size], dtype=np.float32)\nPOSE_STD_X = np.zeros([POSE_IDXS.size], dtype=np.float32)\nPOSE_STD_Y = np.zeros([POSE_IDXS.size], dtype=np.float32)\n\nfor col, ll in enumerate(tqdm( np.transpose(X[:,:,POSE_IDXS], [2,3,0,1]).reshape([POSE_IDXS.size, cfg.N_DIMS, -1]) )):\n    for dim, l in enumerate(ll):\n        v = l[np.nonzero(l)]\n        if dim == 0: # X\n            POSE_MEAN_X[col] = v.mean()\n            POSE_STD_X[col] = v.std()\n        if dim == 1: # Y\n            POSE_MEAN_Y[col] = v.mean()\n            POSE_STD_Y[col] = v.std()\n        \nPOSE_MEAN = np.array([POSE_MEAN_X, POSE_MEAN_Y]).T\nPOSE_STD = np.array([POSE_STD_X, POSE_STD_Y]).T","metadata":{"execution":{"iopub.status.busy":"2024-02-04T05:55:03.957350Z","iopub.execute_input":"2024-02-04T05:55:03.958202Z","iopub.status.idle":"2024-02-04T05:55:07.789023Z","shell.execute_reply.started":"2024-02-04T05:55:03.958015Z","shell.execute_reply":"2024-02-04T05:55:07.786781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.save('/kaggle/working/pose_mean.npy' , POSE_MEAN)\nnp.save('/kaggle/working/pose_std.npy' , POSE_STD)","metadata":{"execution":{"iopub.status.busy":"2024-02-04T05:55:28.658668Z","iopub.execute_input":"2024-02-04T05:55:28.659275Z","iopub.status.idle":"2024-02-04T05:55:28.669015Z","shell.execute_reply.started":"2024-02-04T05:55:28.659227Z","shell.execute_reply":"2024-02-04T05:55:28.666552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Custom sampler to get a batch containing N times all signs\ndef get_train_batch_all_signs(X, y, NON_EMPTY_FRAME_IDXS, n=cfg.BATCH_ALL_SIGNS_N):\n    # Arrays to store batch in\n    X_batch = np.zeros([cfg.NUM_CLASSES*n, cfg.INPUT_SIZE, N_COLS, cfg.N_DIMS], dtype=np.float32)\n    y_batch = np.arange(0, cfg.NUM_CLASSES, step=1/n, dtype=np.float32).astype(np.int64)\n    non_empty_frame_idxs_batch = np.zeros([cfg.NUM_CLASSES*n, cfg.INPUT_SIZE], dtype=np.float32)\n    \n    # Dictionary mapping ordinally encoded sign to corresponding sample indices\n    CLASS2IDXS = {}\n    for i in range(cfg.NUM_CLASSES):\n        CLASS2IDXS[i] = np.argwhere(y == i).squeeze().astype(np.int32)\n            \n    while True:\n        # Fill batch arrays\n        for i in range(cfg.NUM_CLASSES):\n            idxs = np.random.choice(CLASS2IDXS[i], n)\n            X_batch[i*n:(i+1)*n] = X[idxs]\n            non_empty_frame_idxs_batch[i*n:(i+1)*n] = NON_EMPTY_FRAME_IDXS[idxs]\n        \n        yield { 'frames': X_batch, 'non_empty_frame_idxs': non_empty_frame_idxs_batch }, y_batch","metadata":{"execution":{"iopub.status.busy":"2024-02-04T06:25:14.586183Z","iopub.execute_input":"2024-02-04T06:25:14.586941Z","iopub.status.idle":"2024-02-04T06:25:14.620495Z","shell.execute_reply.started":"2024-02-04T06:25:14.586859Z","shell.execute_reply":"2024-02-04T06:25:14.616356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dummy_dataset = get_train_batch_all_signs(X, y, NON_EMPTY_FRAME_IDXS)\nX_batch, y_batch = next(dummy_dataset)\nX_batch.keys(), X_batch['frames'].shape","metadata":{"execution":{"iopub.status.busy":"2024-02-04T06:25:16.644282Z","iopub.execute_input":"2024-02-04T06:25:16.644910Z","iopub.status.idle":"2024-02-04T06:25:16.762853Z","shell.execute_reply.started":"2024-02-04T06:25:16.644688Z","shell.execute_reply":"2024-02-04T06:25:16.760699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Config\n","metadata":{}},{"cell_type":"code","source":"# Epsilon value for layer normalisation\nLAYER_NORM_EPS = 1e-6\n\n# Dense layer units for landmarks\nLIPS_UNITS = 384\nHANDS_UNITS = 384\nPOSE_UNITS = 384\n# final embedding and transformer embedding size\nUNITS = 384\n\n# Transformer\nNUM_BLOCKS = 2\nMLP_RATIO = 2\n\n# Dropout\nEMBEDDING_DROPOUT = 0.00\nMLP_DROPOUT_RATIO = 0.30\nCLASSIFIER_DROPOUT_RATIO = 0.10\n\n# Initiailizers\nINIT_HE_UNIFORM = tf.keras.initializers.he_uniform\nINIT_GLOROT_UNIFORM = tf.keras.initializers.glorot_uniform\nINIT_ZEROS = tf.keras.initializers.constant(0.0)\n# Activations\nGELU = tf.keras.activations.gelu","metadata":{"execution":{"iopub.status.busy":"2024-02-04T06:25:20.775946Z","iopub.execute_input":"2024-02-04T06:25:20.777399Z","iopub.status.idle":"2024-02-04T06:25:20.789090Z","shell.execute_reply.started":"2024-02-04T06:25:20.777329Z","shell.execute_reply":"2024-02-04T06:25:20.786330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Transformer\n\nNeed to implement transformer from scratch as TFLite does not support the native TF implementation of MultiHeadAttention.","metadata":{}},{"cell_type":"code","source":"def scaled_dot_product(q,k,v, softmax, attention_mask):\n    #calculates Q . K(transpose)\n    qkt = tf.matmul(q,k,transpose_b=True)\n    #caculates scaling factor\n    dk = tf.math.sqrt(tf.cast(q.shape[-1],dtype=tf.float32))\n    scaled_qkt = qkt/dk\n    softmax = softmax(scaled_qkt, mask=attention_mask)\n    \n    z = tf.matmul(softmax,v)\n    #shape: (m,Tx,depth), same shape as q,k,v\n    return z\n\nclass MultiHeadAttention(tf.keras.layers.Layer):\n    def __init__(self,d_model,num_of_heads):\n        super(MultiHeadAttention,self).__init__()\n        self.d_model = d_model\n        self.num_of_heads = num_of_heads\n        self.depth = d_model//num_of_heads\n        self.wq = [tf.keras.layers.Dense(self.depth) for i in range(num_of_heads)]\n        self.wk = [tf.keras.layers.Dense(self.depth) for i in range(num_of_heads)]\n        self.wv = [tf.keras.layers.Dense(self.depth) for i in range(num_of_heads)]\n        self.wo = tf.keras.layers.Dense(d_model)\n        self.softmax = tf.keras.layers.Softmax()\n        \n    def call(self,x, attention_mask):\n        \n        multi_attn = []\n        for i in range(self.num_of_heads):\n            Q = self.wq[i](x)\n            K = self.wk[i](x)\n            V = self.wv[i](x)\n            multi_attn.append(scaled_dot_product(Q,K,V, self.softmax, attention_mask))\n            \n        multi_head = tf.concat(multi_attn,axis=-1)\n        multi_head_attention = self.wo(multi_head)\n        return multi_head_attention","metadata":{"execution":{"iopub.status.busy":"2024-02-04T06:25:21.773883Z","iopub.execute_input":"2024-02-04T06:25:21.774466Z","iopub.status.idle":"2024-02-04T06:25:21.797030Z","shell.execute_reply.started":"2024-02-04T06:25:21.774418Z","shell.execute_reply":"2024-02-04T06:25:21.795412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Transformer(tf.keras.Model):\n    def __init__(self, num_blocks):\n        super(Transformer, self).__init__(name='transformer')\n        self.num_blocks = num_blocks\n    \n    def build(self, input_shape):\n        self.ln_1s = []\n        self.mhas = []\n        self.ln_2s = []\n        self.mlps = []\n        # Make Transformer Blocks\n        for i in range(self.num_blocks):\n            # First Layer Normalisation\n            self.ln_1s.append(tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS))\n            # Multi Head Attention\n            self.mhas.append(MultiHeadAttention(UNITS, 8))\n            # Second Layer Normalisation\n            self.ln_2s.append(tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS))\n            # Multi Layer Perception\n            self.mlps.append(tf.keras.Sequential([\n                tf.keras.layers.Dense(UNITS * MLP_RATIO, activation=GELU, kernel_initializer=INIT_GLOROT_UNIFORM),\n                tf.keras.layers.Dropout(MLP_DROPOUT_RATIO),\n                tf.keras.layers.Dense(UNITS, kernel_initializer=INIT_HE_UNIFORM),\n            ]))\n        \n    def call(self, x, attention_mask):\n        # Iterate input over transformer blocks\n        for ln_1, mha, ln_2, mlp in zip(self.ln_1s, self.mhas, self.ln_2s, self.mlps):\n            x1 = ln_1(x)\n            attention_output = mha(x1, attention_mask)\n            x2 = x1 + attention_output\n            x3 = ln_2(x2)\n            x3 = mlp(x3)\n            x = x3 + x2\n    \n        return x","metadata":{"execution":{"iopub.status.busy":"2024-02-04T06:25:22.221801Z","iopub.execute_input":"2024-02-04T06:25:22.222657Z","iopub.status.idle":"2024-02-04T06:25:22.242417Z","shell.execute_reply.started":"2024-02-04T06:25:22.222590Z","shell.execute_reply":"2024-02-04T06:25:22.239984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Landmark Embedding","metadata":{}},{"cell_type":"code","source":"class LandmarkEmbedding(tf.keras.Model):\n    def __init__(self, units, name):\n        super(LandmarkEmbedding, self).__init__(name=f'{name}_embedding')\n        self.units = units\n        \n    def build(self, input_shape):\n        # Embedding for missing landmark in frame, initizlied with zeros\n        self.empty_embedding = self.add_weight(\n            name=f'{self.name}_empty_embedding',\n            shape=[self.units],\n            initializer=INIT_ZEROS,\n        )\n        # Embedding\n        self.dense = tf.keras.Sequential([\n            tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_1', use_bias=False, kernel_initializer=INIT_GLOROT_UNIFORM, activation=GELU),\n            tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_2', use_bias=False, kernel_initializer=INIT_HE_UNIFORM),\n        ], name=f'{self.name}_dense')\n\n    def call(self, x):\n        return tf.where(\n                # Checks whether landmark is missing in frame\n                tf.reduce_sum(x, axis=2, keepdims=True) == 0,\n                # If so, the empty embedding is used\n                self.empty_embedding,\n                # Otherwise the landmark data is embedded\n                self.dense(x),\n            )","metadata":{"execution":{"iopub.status.busy":"2024-02-04T06:25:23.149970Z","iopub.execute_input":"2024-02-04T06:25:23.150533Z","iopub.status.idle":"2024-02-04T06:25:23.166782Z","shell.execute_reply.started":"2024-02-04T06:25:23.150488Z","shell.execute_reply":"2024-02-04T06:25:23.164297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Embedding","metadata":{}},{"cell_type":"code","source":"class CustomEmbedding(tf.keras.Model):\n    def __init__(self):\n        super(CustomEmbedding, self).__init__()\n        \n    def get_diffs(self, l):\n        S = l.shape[2]\n        other = tf.expand_dims(l, 3)\n        other = tf.repeat(other, S, axis=3)\n        other = tf.transpose(other, [0,1,3,2])\n        diffs = tf.expand_dims(l, 3) - other\n        diffs = tf.reshape(diffs, [-1, cfg.INPUT_SIZE, S*S])\n        return diffs\n\n    def build(self, input_shape):\n        # Positional Embedding, initialized with zeros\n        self.positional_embedding = tf.keras.layers.Embedding(cfg.INPUT_SIZE+1, UNITS, embeddings_initializer=INIT_ZEROS)\n        # Embedding layer for Landmarks\n        self.lips_embedding = LandmarkEmbedding(LIPS_UNITS, 'lips')\n        self.left_hand_embedding = LandmarkEmbedding(HANDS_UNITS, 'left_hand')\n        self.right_hand_embedding = LandmarkEmbedding(HANDS_UNITS, 'right_hand')\n        self.pose_embedding = LandmarkEmbedding(POSE_UNITS, 'pose')\n        # Landmark Weights\n        self.landmark_weights = tf.Variable(tf.zeros([4], dtype=tf.float32), name='landmark_weights')\n        # Fully Connected Layers for combined landmarks\n        self.fc = tf.keras.Sequential([\n            tf.keras.layers.Dense(UNITS, name='fully_connected_1', use_bias=False, kernel_initializer=INIT_GLOROT_UNIFORM, activation=GELU),\n            tf.keras.layers.Dense(UNITS, name='fully_connected_2', use_bias=False, kernel_initializer=INIT_HE_UNIFORM),\n        ], name='fc')\n\n\n    def call(self, lips0, left_hand0, right_hand0, pose0, non_empty_frame_idxs, training=False):\n        # Lips\n        lips_embedding = self.lips_embedding(lips0)\n        # Left Hand\n        left_hand_embedding = self.left_hand_embedding(left_hand0)\n        # Right Hand\n        right_hand_embedding = self.right_hand_embedding(right_hand0)\n        # Pose\n        pose_embedding = self.pose_embedding(pose0)\n        # Merge Embeddings of all landmarks with mean pooling\n        x = tf.stack((lips_embedding, left_hand_embedding, right_hand_embedding, pose_embedding), axis=3)\n        # Merge Landmarks with trainable attention weights\n        x = x * tf.nn.softmax(self.landmark_weights)\n        x = tf.reduce_sum(x, axis=3)\n        # Fully Connected Layers\n        x = self.fc(x)\n        # Add Positional Embedding\n        normalised_non_empty_frame_idxs = tf.where(\n            tf.math.equal(non_empty_frame_idxs, -1.0),\n            cfg.INPUT_SIZE,\n            tf.cast(\n                non_empty_frame_idxs / tf.reduce_max(non_empty_frame_idxs, axis=1, keepdims=True) * cfg.INPUT_SIZE,\n                tf.int32,\n            ),\n        )\n        x = x + self.positional_embedding(normalised_non_empty_frame_idxs)\n        \n        return x","metadata":{"execution":{"iopub.status.busy":"2024-02-04T06:25:25.396815Z","iopub.execute_input":"2024-02-04T06:25:25.397381Z","iopub.status.idle":"2024-02-04T06:25:25.421515Z","shell.execute_reply.started":"2024-02-04T06:25:25.397324Z","shell.execute_reply":"2024-02-04T06:25:25.420044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_lr_metric(optimizer):\n    def lr(y_true, y_pred):\n        return optimizer.lr\n    return lr","metadata":{"execution":{"iopub.status.busy":"2024-02-04T06:25:26.284288Z","iopub.execute_input":"2024-02-04T06:25:26.286244Z","iopub.status.idle":"2024-02-04T06:25:26.295583Z","shell.execute_reply.started":"2024-02-04T06:25:26.286149Z","shell.execute_reply":"2024-02-04T06:25:26.293090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model():\n    # Inputs\n    frames = tf.keras.layers.Input([cfg.INPUT_SIZE, N_COLS, cfg.N_DIMS], dtype=tf.float32, name='frames')\n    non_empty_frame_idxs = tf.keras.layers.Input([cfg.INPUT_SIZE], dtype=tf.float32, name='non_empty_frame_idxs')\n    # Padding Mask\n    mask = tf.cast(tf.math.not_equal(non_empty_frame_idxs, -1), tf.float32)\n    mask = tf.expand_dims(mask, axis=2)\n    \n    x = frames\n    x = tf.slice(x, [0,0,0,0], [-1,cfg.INPUT_SIZE, N_COLS, 2])\n    # LIPS\n    lips = tf.slice(x, [0,0,LIPS_START,0], [-1,cfg.INPUT_SIZE, 40, 2])\n    lips = tf.where(\n            tf.math.equal(lips, 0.0),\n            0.0,\n            (lips - LIPS_MEAN) / LIPS_STD,\n        )\n    lips = tf.reshape(lips, [-1, cfg.INPUT_SIZE, 40*2])\n    # LEFT HAND\n    left_hand = tf.slice(x, [0,0,40,0], [-1,cfg.INPUT_SIZE, 21, 2])\n    left_hand = tf.where(\n            tf.math.equal(left_hand, 0.0),\n            0.0,\n            (left_hand - LEFT_HANDS_MEAN) / LEFT_HANDS_STD,\n        )\n    left_hand = tf.reshape(left_hand, [-1, cfg.INPUT_SIZE, 21*2])\n    # RIGHT HAND\n    right_hand = tf.slice(x, [0,0,61,0], [-1,cfg.INPUT_SIZE, 21, 2])\n    right_hand = tf.where(\n            tf.math.equal(right_hand, 0.0),\n            0.0,\n            (right_hand - RIGHT_HANDS_MEAN) / RIGHT_HANDS_STD,\n        )\n    right_hand = tf.reshape(right_hand, [-1, cfg.INPUT_SIZE, 21*2])\n    # POSE\n    pose = tf.slice(x, [0,0,82,0], [-1,cfg.INPUT_SIZE, 10, 2])\n    pose = tf.where(\n            tf.math.equal(pose, 0.0),\n            0.0,\n            (pose - POSE_MEAN) / POSE_STD,\n        )\n    pose = tf.reshape(pose, [-1, cfg.INPUT_SIZE, 10*2])\n    x = lips, left_hand, right_hand, pose\n    x = CustomEmbedding()(lips, left_hand, right_hand, pose, non_empty_frame_idxs)\n    # Encoder Transformer Blocks\n    x = Transformer(NUM_BLOCKS)(x, mask)\n    # Pooling\n    x = tf.reduce_sum(x * mask, axis=1) / tf.reduce_sum(mask, axis=1)\n    # Classification Layer\n    x = tf.keras.layers.Dense(cfg.NUM_CLASSES, activation=tf.keras.activations.softmax, kernel_initializer=INIT_GLOROT_UNIFORM)(x)\n    outputs = x\n    \n    # Create Tensorflow Model\n    model = tf.keras.models.Model(inputs=[frames, non_empty_frame_idxs], outputs=outputs)\n    \n    # Simple Categorical Crossentropy Loss\n    loss = tf.keras.losses.SparseCategoricalCrossentropy()\n    \n    # Adam Optimizer with weight decay\n    optimizer = tfa.optimizers.AdamW(learning_rate=1e-3, weight_decay=1e-5, clipnorm=1.0)\n    \n    lr_metric = get_lr_metric(optimizer)\n    metrics = [\"acc\",lr_metric]\n    model.compile(loss=loss, optimizer=optimizer, metrics=metrics)\n    return model","metadata":{"execution":{"iopub.status.busy":"2024-02-04T06:25:26.947469Z","iopub.execute_input":"2024-02-04T06:25:26.948138Z","iopub.status.idle":"2024-02-04T06:25:26.978133Z","shell.execute_reply.started":"2024-02-04T06:25:26.948074Z","shell.execute_reply":"2024-02-04T06:25:26.976974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.backend.clear_session()\n\nmodel_one = get_model()","metadata":{"execution":{"iopub.status.busy":"2024-02-04T06:25:28.928745Z","iopub.execute_input":"2024-02-04T06:25:28.929915Z","iopub.status.idle":"2024-02-04T06:25:33.453148Z","shell.execute_reply.started":"2024-02-04T06:25:28.929854Z","shell.execute_reply":"2024-02-04T06:25:33.451145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_one.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def lrfn(current_step, num_warmup_steps, lr_max, num_cycles=0.50, num_training_steps=cfg.N_EPOCHS):\n    \n    if current_step < num_warmup_steps:\n        if WARMUP_METHOD == 'log':\n            return lr_max * 0.10 ** (num_warmup_steps - current_step)\n        else:\n            return lr_max * 2 ** -(num_warmup_steps - current_step)\n    else:\n        progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))\n\n        return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))) * lr_max","metadata":{"execution":{"iopub.status.busy":"2024-02-04T06:25:33.455983Z","iopub.execute_input":"2024-02-04T06:25:33.456551Z","iopub.status.idle":"2024-02-04T06:25:33.470602Z","shell.execute_reply.started":"2024-02-04T06:25:33.456497Z","shell.execute_reply":"2024-02-04T06:25:33.468713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_lr_schedule(lr_schedule, epochs):\n    fig = plt.figure(figsize=(20, 10))\n    plt.plot([None] + lr_schedule + [None])\n    # X Labels\n    x = np.arange(1, epochs + 1)\n    x_axis_labels = [i if epochs <= 40 or i % 5 == 0 or i == 1 else None for i in range(1, epochs + 1)]\n    plt.xlim([1, epochs])\n    plt.xticks(x, x_axis_labels) # set tick step to 1 and let x axis start at 1\n    \n    # Increase y-limit for better readability\n    plt.ylim([0, max(lr_schedule) * 1.1])\n    \n    # Title\n    schedule_info = f'start: {lr_schedule[0]:.1E}, max: {max(lr_schedule):.1E}, final: {lr_schedule[-1]:.1E}'\n    plt.title(f'Step Learning Rate Schedule, {schedule_info}', size=18, pad=12)\n    \n    # Plot Learning Rates\n    for x, val in enumerate(lr_schedule):\n        if epochs <= 40 or x % 5 == 0 or x is epochs - 1:\n            if x < len(lr_schedule) - 1:\n                if lr_schedule[x - 1] < val:\n                    ha = 'right'\n                else:\n                    ha = 'left'\n            elif x == 0:\n                ha = 'right'\n            else:\n                ha = 'left'\n            plt.plot(x + 1, val, 'o', color='black');\n            offset_y = (max(lr_schedule) - min(lr_schedule)) * 0.02\n            plt.annotate(f'{val:.1E}', xy=(x + 1, val + offset_y), size=12, ha=ha)\n    \n    plt.xlabel('Epoch', size=16, labelpad=5)\n    plt.ylabel('Learning Rate', size=16, labelpad=5)\n    plt.grid()\n    plt.show()\n\n# Learning rate for encoder\nLR_SCHEDULE = [lrfn(step, num_warmup_steps=cfg.N_WARMUP_EPOCHS, lr_max=cfg.LR_MAX, num_cycles=0.50) for step in range(cfg.N_EPOCHS)]\n# Plot Learning Rate Schedule\nplot_lr_schedule(LR_SCHEDULE, epochs=cfg.N_EPOCHS)\n# Learning Rate Callback\nlr_callback = tf.keras.callbacks.LearningRateScheduler(lambda step: LR_SCHEDULE[step], verbose=1)","metadata":{"execution":{"iopub.status.busy":"2024-02-04T06:25:34.555020Z","iopub.execute_input":"2024-02-04T06:25:34.555682Z","iopub.status.idle":"2024-02-04T06:25:35.347077Z","shell.execute_reply.started":"2024-02-04T06:25:34.555632Z","shell.execute_reply":"2024-02-04T06:25:35.345371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Weight Decay Callback","metadata":{}},{"cell_type":"code","source":"class WeightDecayCallback(tf.keras.callbacks.Callback):\n    def __init__(self, wd_ratio=cfg.WD_RATIO):\n        self.step_counter = 0\n        self.wd_ratio = wd_ratio\n    \n    def on_epoch_begin(self, epoch, logs=None):\n        model_one.optimizer.weight_decay = model_one.optimizer.learning_rate * self.wd_ratio\n        print(f'learning rate: {model_one.optimizer.learning_rate.numpy():.2e}, weight decay: {model_one.optimizer.weight_decay.numpy():.2e}')","metadata":{"execution":{"iopub.status.busy":"2024-02-04T06:25:38.373342Z","iopub.execute_input":"2024-02-04T06:25:38.374905Z","iopub.status.idle":"2024-02-04T06:25:38.384341Z","shell.execute_reply.started":"2024-02-04T06:25:38.374841Z","shell.execute_reply":"2024-02-04T06:25:38.381811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Performance Benchmark","metadata":{}},{"cell_type":"code","source":"%%timeit -n 100\nif cfg.TRAIN_MODEL:\n    # Verify model_one prediction is <<<100ms\n    model_one.predict_on_batch({ 'frames': X[:1], 'non_empty_frame_idxs': NON_EMPTY_FRAME_IDXS[:1] })","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"X_train = X[train_idxs]\nX_val = X[val_idxs]\nNON_EMPTY_FRAME_IDXS_TRAIN = NON_EMPTY_FRAME_IDXS[train_idxs]\nNON_EMPTY_FRAME_IDXS_VAL = NON_EMPTY_FRAME_IDXS[val_idxs]\ny_train = y[train_idxs]\ny_val = y[val_idxs]","metadata":{"execution":{"iopub.status.busy":"2024-02-04T06:25:43.209454Z","iopub.execute_input":"2024-02-04T06:25:43.210134Z","iopub.status.idle":"2024-02-04T06:25:47.833350Z","shell.execute_reply.started":"2024-02-04T06:25:43.210063Z","shell.execute_reply":"2024-02-04T06:25:47.831427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NON_EMPTY_FRAME_IDXS_VAL[0]","metadata":{"execution":{"iopub.status.busy":"2024-02-04T06:27:28.285518Z","iopub.execute_input":"2024-02-04T06:27:28.286034Z","iopub.status.idle":"2024-02-04T06:27:28.298839Z","shell.execute_reply.started":"2024-02-04T06:27:28.285989Z","shell.execute_reply":"2024-02-04T06:27:28.296232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if cfgargsort.TRAIN_MODEL:\n    run = wandb.init(project=\"kaggle-asl-signs\", config=cfg, tags=['transformer', 'final-model'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if cfg.TRAIN_MODEL:\n    tf.keras.backend.clear_session()\n    callbacks=[\n            lr_callback,\n            WeightDecayCallback(),\n            wandb.keras.WandbCallback()\n    ]\n    model_one.fit(\n        x=get_train_batch_all_signs(X, y, NON_EMPTY_FRAME_IDXS),\n        steps_per_epoch=len(X) // (cfg.NUM_CLASSES * cfg.BATCH_ALL_SIGNS_N),\n        epochs=cfg.N_EPOCHS,\n        batch_size=cfg.BATCH_SIZE,\n        callbacks=callbacks,\n        verbose = 2,) ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train[0].shape","metadata":{"execution":{"iopub.status.busy":"2024-02-03T08:45:52.011780Z","iopub.execute_input":"2024-02-03T08:45:52.012241Z","iopub.status.idle":"2024-02-03T08:45:52.025930Z","shell.execute_reply.started":"2024-02-03T08:45:52.012197Z","shell.execute_reply":"2024-02-03T08:45:52.023561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if cfg.TRAIN_MODEL:\n    model_one.save('./final_model_one')\n    model_one.save_weights('./final_model_one_weights')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if cfg.TRAIN_MODEL:\n    artifact = wandb.Artifact('final_model_one', type='model')\n    artifact.add_file('./final_model_one_weights.data-00000-of-00001')\n    artifact.add_file('./final_model_one_weights.index')\n    run.log_artifact(artifact)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# download model\ntry:\n    artifact = run.use_artifact('amanarora/asl-sings/final_model_one:v1', type='model')\nexcept NameError:\n    run = wandb.init(project=\"kaggle-asl-signs\")\n    artifact = run.use_artifact('amanarora/asl-sings/final_model_one:v1', type='model')\nartifact_dir = artifact.download()\n!ls {artifact_dir}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load model weights and do sanity c\nmodel_one.load_weights(f\"{artifact_dir}/final_model_one_weights\")\ny_val_pred = model_one.predict({ 'frames': X_val, 'non_empty_frame_idxs': NON_EMPTY_FRAME_IDXS_VAL }, verbose=2).argmax(axis=1)\nnp.mean(y_val == y_val_pred)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# delete variables as we go to free up RAM\ndel X; del y; del X_train; del X_val; del y_train; del y_val","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Landmark Weights\nweights = scipy.special.softmax(model_one.get_layer('custom_embedding').weights[15])\nlandmarks = ['lips_embedding', 'left_hand_embedding', 'right_hand_embedding', 'pose_embedding']\n\n# Learned attention weights, initialized at uniform 25%\nfor w, lm in zip(weights, landmarks):\n    print(f'{lm} weight: {(w*100):.1f}%')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"converter = tf.lite.TFLiteConverter.from_keras_model(model_one)\nconverter.optimizations = [tf.lite.Optimize.DEFAULT]\ntflite_model = converter.convert()\nwith open('./model.tflite', 'wb') as f:\n    f.write(tflite_model)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!du -sh ./model.tflite","metadata":{"execution":{"iopub.status.busy":"2023-05-06T11:55:24.132336Z","iopub.execute_input":"2023-05-06T11:55:24.132716Z","iopub.status.idle":"2023-05-06T11:55:25.367415Z","shell.execute_reply.started":"2023-05-06T11:55:24.132677Z","shell.execute_reply":"2023-05-06T11:55:25.366194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}