{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nimport matplotlib.pyplot as plt\nimport matplotlib as mpl\nimport seaborn as sn\n\nfrom tensorflow import keras\nfrom tqdm.notebook import tqdm\nfrom sklearn.model_selection import train_test_split, GroupShuffleSplit \n\nimport glob\nimport sys\nimport os\nimport math\nimport gc\nimport sys\nimport sklearn\nimport scipy\n\nprint(f'Tensorflow V{tf.__version__}')\nprint(f'Keras V{tf.keras.__version__}')\nprint(f'Python V{sys.version}')","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:48:18.886555Z","iopub.execute_input":"2023-04-20T20:48:18.887023Z","iopub.status.idle":"2023-04-20T20:48:26.833346Z","shell.execute_reply.started":"2023-04-20T20:48:18.886988Z","shell.execute_reply":"2023-04-20T20:48:26.832176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"导入数据分析和机器学习库。\n- numpy: 一个科学计算库，提供多维数组和矩阵数据结构以及各种数学函数来操作它们。\n- pandas: 一个数据分析和处理库，提供了操作数值表格和时间序列的数据结构和操作。\n- tensorflow: 一个开源的机器学习平台，提供了一个全面和灵活的框架来开发和部署各种类型的神经网络。\n- tensorflow_addons: 一个贡献库，遵循了一些成熟的API模式，但实现了一些在核心TensorFlow中不可用的新功能。\n- matplotlib: 一个绘图库，可以在各种格式和交互环境中生成出版质量的图形。\n- seaborn: 一个基于matplotlib的数据可视化库，提供了一个高级接口来绘制美观和有信息量的统计图形。\n- tqdm: 一个快速和可扩展的Python和CLI进度条。\n- sklearn: 一个机器学习库，提供了各种分类、回归、聚类、降维、模型选择和预处理算法。\n- glob: 一个模块，提供了一个函数来生成匹配给定模式的文件列表。\n- sys: 一个模块，提供了访问一些由解释器使用或维护的变量和函数的接口。\n- os: 一个模块，提供了一种使用操作系统依赖功能的可移植方式。\n- math: 一个模块，提供了访问C标准定义的数学函数的接口。\n- gc: 一个模块，提供了一个接口来访问可选的垃圾收集器。\n- scipy: 一个科学计算库，提供了许多用户友好和高效的数值例程，如数值积分、插值、优化、线性代数和统计。\n\n最后打印TensorFlow、Keras和Python的版本。","metadata":{}},{"cell_type":"markdown","source":"# matplotlib参数配置","metadata":{}},{"cell_type":"code","source":"# MatplotLib Global Settings\n# 重置所有参数为默认值\nmpl.rcParams.update(mpl.rcParamsDefault)\n# 设置 x 轴和 y 轴的刻度标签字体大小为 16\nmpl.rcParams['xtick.labelsize'] = 16\nmpl.rcParams['ytick.labelsize'] = 16\n# 设置坐标轴标签和标题的字体大小为 18 和 24\nmpl.rcParams['axes.labelsize'] = 18\nmpl.rcParams['axes.titlesize'] = 24\n","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:48:26.835694Z","iopub.execute_input":"2023-04-20T20:48:26.836824Z","iopub.status.idle":"2023-04-20T20:48:26.843932Z","shell.execute_reply.started":"2023-04-20T20:48:26.836769Z","shell.execute_reply":"2023-04-20T20:48:26.842940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 训练参数配置","metadata":{}},{"cell_type":"code","source":"# If True, processing data from scratch\n# If False, loads preprocessed data\n# True，则从头开始处理数据，False，则加载预处理的数据\n\nPREPROCESS_DATA = False\nTRAIN_MODEL = True\n# True: use 10% of participants as validation set\n# False: use all data for training -> gives better LB result更好的leaderboard排名\nUSE_VAL = False\n\nN_ROWS = 543#设置行数\nN_DIMS = 3#设置维度\nDIM_NAMES = ['x', 'y', 'z']#设置维度名称\nSEED = 42#设置随机数种子\nNUM_CLASSES = 250#共250种类别\nIS_INTERACTIVE = os.environ['KAGGLE_KERNEL_RUN_TYPE'] == 'Interactive'\n#判断是否为交互式环境\nVERBOSE = 1 if IS_INTERACTIVE else 2\n\nINPUT_SIZE = 64#输入大小\nBATCH_ALL_SIGNS_N = 4\nBATCH_SIZE = 512\nN_EPOCHS = 220\nLR_MAX = 0.001#设置学习率最大值\nN_WARMUP_EPOCHS = 0#设置热身轮数\nWD_RATIO = 0.05#设置权重衰减比例\nMASK_VAL = 4237#设置掩码值","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:48:26.845422Z","iopub.execute_input":"2023-04-20T20:48:26.847574Z","iopub.status.idle":"2023-04-20T20:48:26.859099Z","shell.execute_reply.started":"2023-04-20T20:48:26.847536Z","shell.execute_reply":"2023-04-20T20:48:26.858094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"N_ROWS = 543是在设置行数，\nN_DIMS = 3是在设置维度，\nDIM_NAMES = [‘x’, ‘y’, ‘z’]是在设置维度名称，\nSEED = 42是在设置随机数种子，\nNUM_CLASSES = 250是在设置类别数量，\nIS_INTERACTIVE = os.environ[‘KAGGLE_KERNEL_RUN_TYPE’] == 'Interactive’是在判断是否为交互式环境，VERBOSE = 1 if IS_INTERACTIVE else 2是在设置详细程度，\nINPUT_SIZE = 64是在设置输入大小，\nBATCH_ALL_SIGNS_N = 4是在设置批量大小，\nBATCH_SIZE = 512是在设置批量大小，\nN_EPOCHS = 200是在设置训练轮数，\nLR_MAX = 1e-3是在设置学习率最大值，\nN_WARMUP_EPOCHS = 0是在设置热身轮数，\nWD_RATIO = 0.05是在设置权重衰减比例，\nMASK_VAL = 4237是在设置掩码值。","metadata":{}},{"cell_type":"code","source":"# Prints Shape and Dtype For List Of Variables\ndef print_shape_dtype(l, names):\n for e, n in zip(l, names):\n print(f'{n} shape: {e.shape}, dtype: {e.dtype}')","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:48:26.861668Z","iopub.execute_input":"2023-04-20T20:48:26.862246Z","iopub.status.idle":"2023-04-20T20:48:26.869780Z","shell.execute_reply.started":"2023-04-20T20:48:26.862209Z","shell.execute_reply":"2023-04-20T20:48:26.868769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"遍历变量列表和名称列表，然后打印每个变量的形状和数据类型","metadata":{}},{"cell_type":"markdown","source":"# 读取训练数据","metadata":{}},{"cell_type":"code","source":"# Read Training Data\n\nif IS_INTERACTIVE or not PREPROCESS_DATA:\n train = pd.read_csv('/kaggle/input/asl-signs/train.csv').sample(int(5e3), random_state=SEED)\nelse:\n train = pd.read_csv('/kaggle/input/asl-signs/train.csv')\n\nN_SAMPLES = len(train)\nprint(f'N_SAMPLES: {N_SAMPLES}')","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:48:26.870978Z","iopub.execute_input":"2023-04-20T20:48:26.873122Z","iopub.status.idle":"2023-04-20T20:48:27.058201Z","shell.execute_reply.started":"2023-04-20T20:48:26.873078Z","shell.execute_reply":"2023-04-20T20:48:27.057064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"如果IS_INTERACTIVE或PREPROCESS_DATA为False，\n\n则使用Pandas的sample()函数从CSV文件中读取5000行随机样本。\n\n否则，它将读取CSV文件中的所有行。\n","metadata":{}},{"cell_type":"markdown","source":"# 添加文件路径","metadata":{}},{"cell_type":"markdown","source":"这段代码定义了一个名为get_file_path()的函数，它接受一个参数path，并返回一个完整的文件路径。该函数用于将文件路径转换为Kaggle数据集中的文件路径。\n代码使用Pandas的apply()函数将get_file_path()函数应用于名为train['path']的Pandas Series对象中的每个元素。然后，代码将结果存储在名为train['file_path']的新列中。","metadata":{}},{"cell_type":"code","source":"# Get complete file path to file\ndef get_file_path(path):\n return f'/kaggle/input/asl-signs/{path}'\n\ntrain['file_path'] = train['path'].apply(get_file_path)","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:48:27.059851Z","iopub.execute_input":"2023-04-20T20:48:27.060627Z","iopub.status.idle":"2023-04-20T20:48:27.073232Z","shell.execute_reply.started":"2023-04-20T20:48:27.060570Z","shell.execute_reply":"2023-04-20T20:48:27.072228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 对符号进行数字编码","metadata":{}},{"cell_type":"code","source":"# Add ordinally Encoded Sign (assign number to each sign name)\n# 添加序数编码符号（为每个符号名称分配编号）\ntrain['sign_ord'] = train['sign'].astype('category').cat.codes\n\n# Dictionaries to translate sign <-> ordinal encoded sign\n# 用于符号<->序数编码之间相互转换的字典\nSIGN2ORD = train[['sign', 'sign_ord']].set_index('sign').squeeze().to_dict()\nORD2SIGN = train[['sign_ord', 'sign']].set_index('sign_ord').squeeze().to_dict()","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:48:27.075449Z","iopub.execute_input":"2023-04-20T20:48:27.075971Z","iopub.status.idle":"2023-04-20T20:48:27.098792Z","shell.execute_reply.started":"2023-04-20T20:48:27.075924Z","shell.execute_reply":"2023-04-20T20:48:27.097919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train.head(30))\ndisplay(train.info())","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:48:27.100434Z","iopub.execute_input":"2023-04-20T20:48:27.100798Z","iopub.status.idle":"2023-04-20T20:48:27.134199Z","shell.execute_reply.started":"2023-04-20T20:48:27.100759Z","shell.execute_reply":"2023-04-20T20:48:27.133132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 统计视频数据","metadata":{}},{"cell_type":"markdown","source":"这段代码是用于计算视频数据集中每个视频的不同帧数、缺失帧数和最大帧数的统计信息。\n其中，`N` 是根据条件设置的值，`N_UNIQUE_FRAMES`、`N_MISSING_FRAMES` 和 `MAX_FRAME` 分别是存储不同帧数、缺失帧数和最大帧数的数组。\n在循环中，代码会遍历数据集中的每个视频，读取视频数据并计算不同帧数、缺失帧数和最大帧数。\n最后，代码会显示这些统计信息，并绘制相应的直方图。","metadata":{}},{"cell_type":"code","source":"N = int(1e3) if (IS_INTERACTIVE or not PREPROCESS_DATA) else int(10e3) # 根据条件设置 N 的值\nN_UNIQUE_FRAMES = np.zeros(N, dtype=np.uint16) # 初始化 N_UNIQUE_FRAMES 数组\nN_MISSING_FRAMES = np.zeros(N, dtype=np.uint16) # 初始化 N_MISSING_FRAMES 数组\nMAX_FRAME = np.zeros(N, dtype=np.uint16) # 初始化 MAX_FRAME 数组\n\nPERCENTILES = [0.01, 0.05, 0.25, 0.50, 0.75, 0.95, 0.99, 0.999] # 定义 PERCENTILES 列表\n\nfor idx, file_path in enumerate(tqdm(train['file_path'].sample(N, random_state=SEED))): # 遍历 train['file_path'] 中的文件路径\n df = pd.read_parquet(file_path) # 读取文件\n N_UNIQUE_FRAMES[idx] = df['frame'].nunique() # 计算每个文件中不同帧的数量\n N_MISSING_FRAMES[idx] = (df['frame'].max() - df['frame'].min()) - df['frame'].nunique() + 1 # 计算每个文件中缺失帧的数量\n MAX_FRAME[idx] = df['frame'].max() # 计算每个文件中最大帧数","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:48:27.136150Z","iopub.execute_input":"2023-04-20T20:48:27.136517Z","iopub.status.idle":"2023-04-20T20:48:50.259095Z","shell.execute_reply.started":"2023-04-20T20:48:27.136480Z","shell.execute_reply":"2023-04-20T20:48:50.257960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Number of unique frames in each video\ndisplay(pd.Series(N_UNIQUE_FRAMES).describe(percentiles=PERCENTILES).to_frame('N_UNIQUE_FRAMES'))\n\nplt.figure(figsize=(15,8))\nplt.title('Number of Unique Frames', size=24)\npd.Series(N_UNIQUE_FRAMES).plot(kind='hist', bins=128)\nplt.grid()\nxlim = math.ceil(plt.xlim()[1])\nplt.xlim(0, xlim)\nplt.xticks(np.arange(0, xlim+25, 25))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:48:50.263433Z","iopub.execute_input":"2023-04-20T20:48:50.263725Z","iopub.status.idle":"2023-04-20T20:48:50.889682Z","shell.execute_reply.started":"2023-04-20T20:48:50.263697Z","shell.execute_reply":"2023-04-20T20:48:50.888640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Number of missing frames, consecutive frames with missing intermediate frame, i.e. 1,2,4,5 -> 3 is missing\n#丢失帧数，丢失中间帧的连续帧\ndisplay(pd.Series(N_MISSING_FRAMES).describe(percentiles=PERCENTILES).to_frame('N_MISSING_FRAMES'))\n\nplt.figure(figsize=(15,8))\nplt.title('Number of Missing Frames', size=24)\npd.Series(N_MISSING_FRAMES).plot(kind='hist', bins=128)\nplt.grid()\nplt.xlim(0, math.ceil(plt.xlim()[1]))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:48:50.894034Z","iopub.execute_input":"2023-04-20T20:48:50.896736Z","iopub.status.idle":"2023-04-20T20:48:51.493104Z","shell.execute_reply.started":"2023-04-20T20:48:50.896695Z","shell.execute_reply":"2023-04-20T20:48:51.492053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Maximum frame number\ndisplay(pd.Series(MAX_FRAME).describe(percentiles=PERCENTILES).to_frame('MAX_FRAME'))\n\nplt.figure(figsize=(15,8))\nplt.title('Maximum Frames Index', size=24)\npd.Series(MAX_FRAME).plot(kind='hist', bins=128)\nplt.grid()\nplt.xlim(0, math.ceil(plt.xlim()[1]))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:48:51.497535Z","iopub.execute_input":"2023-04-20T20:48:51.499899Z","iopub.status.idle":"2023-04-20T20:48:52.104090Z","shell.execute_reply.started":"2023-04-20T20:48:51.499844Z","shell.execute_reply":"2023-04-20T20:48:52.103035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Landmark Indices 关键点索引","metadata":{}},{"cell_type":"markdown","source":"在机器学习中，Landmark Indices通常指的是人脸关键点的索引。人脸关键点是人脸上的一些特定点，例如眼睛、鼻子、嘴巴等，它们可以用于人脸识别、表情识别等任务。在机器学习中，我们可以使用这些关键点来训练模型，从而实现人脸识别等任务。¹","metadata":{}},{"cell_type":"code","source":"# 定义三种数据类型：左手、姿态和右手\nUSE_TYPES = ['left_hand', 'pose', 'right_hand']\n# 定义原始数据中的起始索引\nSTART_IDX = 468\n# 定义原始数据中的嘴唇关键点索引，共40个\nLIPS_IDXS0 = np.array([\n 61, 185, 40, 39, 37, 0, 267, 269, 270, 409,\n 291, 146, 91, 181, 84, 17, 314, 405, 321, 375,\n 78, 191, 80, 81, 82, 13, 312, 311, 310, 415,\n 95, 88, 178, 87, 14, 317, 402, 318, 324, 308,\n ])\n# 定义原始数据中的左手关键点索引，共21个\nLEFT_HAND_IDXS0 = np.arange(468,489)\n# 定义原始数据中的右手关键点索引，共21个\nRIGHT_HAND_IDXS0 = np.arange(522,543)\n# 定义原始数据中的左侧姿态关键点索引，共5个\nLEFT_POSE_IDXS0 = np.array([502, 504, 506, 508, 510])\n# 定义原始数据中的右侧姿态关键点索引，共5个\nRIGHT_POSE_IDXS0 = np.array([503, 505, 507, 509, 511])\n# 定义左手优先的关键点索引，包括嘴唇、左手和左侧姿态，共66个\nLANDMARK_IDXS_LEFT_DOMINANT0 = np.concatenate((LIPS_IDXS0, LEFT_HAND_IDXS0, LEFT_POSE_IDXS0))\n# 定义右手优先的关键点索引，包括嘴唇、右手和右侧姿态，共66个\nLANDMARK_IDXS_RIGHT_DOMINANT0 = np.concatenate((LIPS_IDXS0, RIGHT_HAND_IDXS0, RIGHT_POSE_IDXS0))\n# 定义所有手部关键点索引，包括左手和右手，共42个\nHAND_IDXS0 = np.concatenate((LEFT_HAND_IDXS0, RIGHT_HAND_IDXS0), axis=0)\n# 定义处理后数据的列数，等于66\nN_COLS = LANDMARK_IDXS_LEFT_DOMINANT0.size\n# 定义处理后数据中的嘴唇关键点索引，从0到39\nLIPS_IDXS = np.argwhere(np.isin(LANDMARK_IDXS_LEFT_DOMINANT0, LIPS_IDXS0)).squeeze()\n# 定义处理后数据中的左手关键点索引，从40到60\nLEFT_HAND_IDXS = np.argwhere(np.isin(LANDMARK_IDXS_LEFT_DOMINANT0, LEFT_HAND_IDXS0)).squeeze()\n# 定义处理后数据中的右手关键点索引，从40到60\nRIGHT_HAND_IDXS = np.argwhere(np.isin(LANDMARK_IDXS_LEFT_DOMINANT0, RIGHT_HAND_IDXS0)).squeeze()\n# 定义处理后数据中的所有手部关键点索引，从40到81\nHAND_IDXS = np.argwhere(np.isin(LANDMARK_IDXS_LEFT_DOMINANT0, HAND_IDXS0)).squeeze()\n# 定义处理后数据中的姿态关键点索引，从61到65\nPOSE_IDXS = np.argwhere(np.isin(LANDMARK_IDXS_LEFT_DOMINANT0, LEFT_POSE_IDXS0)).squeeze()\n# 打印出手部关键点索引的长度和处理后数据的列数\nprint(f'# HAND_IDXS: {len(HAND_IDXS)}, N_COLS: {N_COLS}')\n#这段代码是用于定义处理后数据中不同部位的关键点的起始位置，以便于后续的切片或索引操作","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:48:52.108633Z","iopub.execute_input":"2023-04-20T20:48:52.111008Z","iopub.status.idle":"2023-04-20T20:48:52.131705Z","shell.execute_reply.started":"2023-04-20T20:48:52.110967Z","shell.execute_reply":"2023-04-20T20:48:52.130372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 定义嘴唇关键点的起始位置，为0\nLIPS_START = 0\n# 定义左手关键点的起始位置，为嘴唇关键点的数量\nLEFT_HAND_START = LIPS_IDXS.size\n# 定义右手关键点的起始位置，为左手关键点的起始位置加上左手关键点的数量\nRIGHT_HAND_START = LEFT_HAND_START + LEFT_HAND_IDXS.size\n# 定义姿态关键点的起始位置，为右手关键点的起始位置加上右手关键点的数量\nPOSE_START = RIGHT_HAND_START + RIGHT_HAND_IDXS.size\n# 打印出不同部位的关键点的起始位置\nprint(f'LIPS_START: {LIPS_START}, LEFT_HAND_START: {LEFT_HAND_START}, RIGHT_HAND_START: {RIGHT_HAND_START}, POSE_START: {POSE_START}')","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:48:52.133627Z","iopub.execute_input":"2023-04-20T20:48:52.134080Z","iopub.status.idle":"2023-04-20T20:48:52.147918Z","shell.execute_reply.started":"2023-04-20T20:48:52.134043Z","shell.execute_reply":"2023-04-20T20:48:52.146710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Process Data Tensorflow","metadata":{}},{"cell_type":"code","source":"# Source: https://www.kaggle.com/competitions/asl-signs/overview/evaluation\nROWS_PER_FRAME = 543 # number of landmarks per frame每帧的标记数量\n#`load_relevant_data_subset 加载相关数据子集\n#提取数据集中的x、y、z三列数据，将数据集按照每帧的地标数量进行切分，返回一个三维数组。\n#其中第一维表示帧数，第二维表示每帧的地标数量，第三维表示x、y、z三个坐标轴。\ndef load_relevant_data_subset(pq_path):\n data_columns = ['x', 'y', 'z']\n data = pd.read_parquet(pq_path, columns=data_columns)\n n_frames = int(len(data) / ROWS_PER_FRAME)\n data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n return data.astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:48:52.153045Z","iopub.execute_input":"2023-04-20T20:48:52.154309Z","iopub.status.idle":"2023-04-20T20:48:52.163366Z","shell.execute_reply.started":"2023-04-20T20:48:52.154243Z","shell.execute_reply":"2023-04-20T20:48:52.162352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"这段代码定义了一个名为ROWS_PER_FRAME的常量，它的值为543，表示每帧的地标数量。函数load_relevant_data_subset(pq_path)读取指定路径下的parquet文件，提取数据集中的x、y、z三列数据，将数据集按照每帧的地标数量进行切分，返回一个三维数组。其中第一维表示帧数，第二维表示每帧的地标数量，第三维表示x、y、z三个坐标轴。函数返回值类型为numpy.ndarray，数据类型为np.float32。","metadata":{}},{"cell_type":"markdown","source":"# 自定义一个使用TF的数据预处理层","metadata":{}},{"cell_type":"markdown","source":"PreprocessLayer 类是一个继承自 TensorFlow 的 tf.keras.layers.Layer 类的自定义层，用于在 TensorFlow Lite 模型中处理数据。这个自定义层包含了一些函数，用于处理输入数据。","metadata":{}},{"cell_type":"markdown","source":"首先，该层在 __init__ 方法中创建了一个名为 normalisation_correction 的常量。该常量是一个矩阵，其行数等于数据中特定类型的标志点的数量，列数是 3（即 x、y 和 z 坐标）。这个矩阵用于校正相机的拍摄方向，将左手调整为右手，右手调整为左手。\n\n该层还定义了一个名为 pad_edge 的方法，用于在给定张量的左侧或右侧填充一定数量的重复元素。接下来，该层使用 @tf.function 装饰器装饰了一个 call 方法，用于处理输入数据。\n\n该方法首先计算了输入数据的帧数（N_FRAMES0），然后通过计算左右手各自在数据中的坐标之和，找到了数据中支配性手的标志点。接下来，该方法计算了每个帧的支配性手中非 NaN 值的数量，以确定哪些帧需要保留。然后，它使用这些索引从输入数据中收集标志点数据。\n\n该方法接下来将帧索引的数据类型从整数转换为浮点数，然后将其规范化为以 0 开始。接下来，它再次计算了经过筛选的数据的帧数（N_FRAMES），然后从这些数据中收集了特定类型的标志点数据。如果数据的帧数小于指定的输入大小（INPUT_SIZE），则使用 -1 进行填充，将数据的帧数扩展到指定的输入大小，并将 NaN 值替换为 0。如果数据的帧数大于指定的输入大小，则使用重复数据将其缩小到指定的输入大小，并填充任何缺失的数据。\n\n最后，该方法返回经过处理的数据和相应的帧索引。","metadata":{}},{"cell_type":"markdown","source":"这段代码是一个 TensorFlow Keras 自定义层，命名为 PreprocessLayer。它用于对输入数据进行预处理，包括对手部关键点数据进行处理、填充和归一化等操作。\n\n该层的主要功能包括：\n\n初始化操作：在 __init__ 方法中，通过调用父类 tf.keras.layers.Layer 的 __init__ 方法进行初始化，并定义了一个名为 normalisation_correction 的常量张量，并将其转置存储在 self.normalisation_correction 中。\n\npad_edge 方法：用于在输入数据的边缘填充数据，根据指定的填充方向（'LEFT' 或 'RIGHT'）和填充的重复次数。\n\ncall 方法：通过使用 @tf.function 装饰器，定义了一个计算图（Graph）的操作，用于对输入数据进行处理。具体步骤如下：\n\na. 获取输入数据的第一个维度（帧数）并存储在 N_FRAMES0 变量中。\n\nb. 判断左手或右手哪只手是主导手，并根据主导手的结果，计算每一帧中手部关键点非 NaN（非空）值的和，并存储在 left_hand_sum 和 right_hand_sum 中。\n\nc. 根据主导手的结果，计算每一帧中主导手的手部关键点非 NaN（非空）值的和，并存储在 frames_hands_non_nan_sum 中。\n\nd. 根据 frames_hands_non_nan_sum 中的结果，找到非空帧的索引，并存储在 non_empty_frames_idxs 中。\n\ne. 根据 non_empty_frames_idxs 过滤输入数据，并进行一系列归一化和填充操作，最终返回处理后的数据和填充后的帧索引。\n\n总体而言，PreprocessLayer 自定义层主要用于对输入数据进行预处理，包括对手部关键点数据的处理、填充和归一化等操作，以满足后续模型的输入要求。","metadata":{}},{"cell_type":"code","source":"\"\"\"\n Tensorflow layer to process data in TFLite\n Data needs to be processed in the model itself, so we can not use Python\n\"\"\" \nclass PreprocessLayer(tf.keras.layers.Layer):\n def __init__(self):\n super(PreprocessLayer, self).__init__()\n normalisation_correction = tf.constant([\n # Add 0.50 to left hand (original right hand) and substract 0.50 of right hand (original left hand)\n [0] * len(LIPS_IDXS) + [0.50] * len(LEFT_HAND_IDXS) + [0.50] * len(POSE_IDXS),\n # Y coordinates stay intact\n [0] * len(LANDMARK_IDXS_LEFT_DOMINANT0),\n # Z coordinates stay intact\n [0] * len(LANDMARK_IDXS_LEFT_DOMINANT0),\n ],\n dtype=tf.float32,\n )\n self.normalisation_correction = tf.transpose(normalisation_correction, [1,0])\n \n def pad_edge(self, t, repeats, side):\n if side == 'LEFT':\n return tf.concat((tf.repeat(t[:1], repeats=repeats, axis=0), t), axis=0)\n elif side == 'RIGHT':\n return tf.concat((t, tf.repeat(t[-1:], repeats=repeats, axis=0)), axis=0)\n \n @tf.function(\n input_signature=(tf.TensorSpec(shape=[None,N_ROWS,N_DIMS], dtype=tf.float32),),\n )\n def call(self, data0):\n # Number of Frames in Video\n N_FRAMES0 = tf.shape(data0)[0]\n \n # Find dominant hand by comparing summed absolute coordinates\n left_hand_sum = tf.math.reduce_sum(tf.where(tf.math.is_nan(tf.gather(data0, LEFT_HAND_IDXS0, axis=1)), 0, 1))\n right_hand_sum = tf.math.reduce_sum(tf.where(tf.math.is_nan(tf.gather(data0, RIGHT_HAND_IDXS0, axis=1)), 0, 1))\n left_dominant = left_hand_sum >= right_hand_sum\n \n # Count non NaN Hand values in each frame for the dominant hand\n if left_dominant:\n frames_hands_non_nan_sum = tf.math.reduce_sum(\n tf.where(tf.math.is_nan(tf.gather(data0, LEFT_HAND_IDXS0, axis=1)), 0, 1),\n axis=[1, 2],\n )\n else:\n frames_hands_non_nan_sum = tf.math.reduce_sum(\n tf.where(tf.math.is_nan(tf.gather(data0, RIGHT_HAND_IDXS0, axis=1)), 0, 1),\n axis=[1, 2],\n )\n \n # Find frames indices with coordinates of dominant hand\n non_empty_frames_idxs = tf.where(frames_hands_non_nan_sum > 0)\n non_empty_frames_idxs = tf.squeeze(non_empty_frames_idxs, axis=1)\n # Filter frames\n data = tf.gather(data0, non_empty_frames_idxs, axis=0)\n \n # Cast Indices in float32 to be compatible with Tensorflow Lite\n non_empty_frames_idxs = tf.cast(non_empty_frames_idxs, tf.float32)\n # Normalize to start with 0\n non_empty_frames_idxs -= tf.reduce_min(non_empty_frames_idxs)\n \n # Number of Frames in Filtered Video\n N_FRAMES = tf.shape(data)[0]\n \n # Gather Relevant Landmark Columns\n if left_dominant:\n data = tf.gather(data, LANDMARK_IDXS_LEFT_DOMINANT0, axis=1)\n else:\n data = tf.gather(data, LANDMARK_IDXS_RIGHT_DOMINANT0, axis=1)\n data = (\n self.normalisation_correction + (\n (data - self.normalisation_correction) * tf.where(self.normalisation_correction != 0, -1.0, 1.0))\n )\n \n # Video fits in INPUT_SIZE\n if N_FRAMES < INPUT_SIZE:\n # Pad With -1 to indicate padding\n non_empty_frames_idxs = tf.pad(non_empty_frames_idxs, [[0, INPUT_SIZE-N_FRAMES]], constant_values=-1)\n # Pad Data With Zeros\n data = tf.pad(data, [[0, INPUT_SIZE-N_FRAMES], [0,0], [0,0]], constant_values=0)\n # Fill NaN Values With 0\n data = tf.where(tf.math.is_nan(data), 0.0, data)\n return data, non_empty_frames_idxs\n # Video needs to be downsampled to INPUT_SIZE\n else:\n # Repeat\n if N_FRAMES < INPUT_SIZE**2:\n repeats = tf.math.floordiv(INPUT_SIZE * INPUT_SIZE, N_FRAMES0)\n data = tf.repeat(data, repeats=repeats, axis=0)\n non_empty_frames_idxs = tf.repeat(non_empty_frames_idxs, repeats=repeats, axis=0)\n\n # Pad To Multiple Of Input Size\n pool_size = tf.math.floordiv(len(data), INPUT_SIZE)\n if tf.math.mod(len(data), INPUT_SIZE) > 0:\n pool_size += 1\n\n if pool_size == 1:\n pad_size = (pool_size * INPUT_SIZE) - len(data)\n else:\n pad_size = (pool_size * INPUT_SIZE) % len(data)\n\n # Pad Start/End with Start/End value\n pad_left = tf.math.floordiv(pad_size, 2) + tf.math.floordiv(INPUT_SIZE, 2)\n pad_right = tf.math.floordiv(pad_size, 2) + tf.math.floordiv(INPUT_SIZE, 2)\n if tf.math.mod(pad_size, 2) > 0:\n pad_right += 1\n\n # Pad By Concatenating Left/Right Edge Values\n data = self.pad_edge(data, pad_left, 'LEFT')\n data = self.pad_edge(data, pad_right, 'RIGHT')\n\n # Pad Non Empty Frame Indices\n non_empty_frames_idxs = self.pad_edge(non_empty_frames_idxs, pad_left, 'LEFT')\n non_empty_frames_idxs = self.pad_edge(non_empty_frames_idxs, pad_right, 'RIGHT')\n\n # Reshape to Mean Pool\n data = tf.reshape(data, [INPUT_SIZE, -1, N_COLS, N_DIMS])\n non_empty_frames_idxs = tf.reshape(non_empty_frames_idxs, [INPUT_SIZE, -1])\n\n # Mean Pool\n data = tf.experimental.numpy.nanmean(data, axis=1)\n non_empty_frames_idxs = tf.experimental.numpy.nanmean(non_empty_frames_idxs, axis=1)\n\n # Fill NaN Values With 0\n data = tf.where(tf.math.is_nan(data), 0.0, data)\n \n return data, non_empty_frames_idxs\n \npreprocess_layer = PreprocessLayer()","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:48:52.167839Z","iopub.execute_input":"2023-04-20T20:48:52.170496Z","iopub.status.idle":"2023-04-20T20:48:54.842283Z","shell.execute_reply.started":"2023-04-20T20:48:52.170453Z","shell.execute_reply":"2023-04-20T20:48:54.841228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Interpolate NaN Values","metadata":{}},{"cell_type":"markdown","source":"Interpolate NaN Values是指使用插值法填充数据中的NaN值。NaN是计算机科学中数值数据类型的一类值，表示未定义或不可表示的值。NaN是Not a Number的缩写，理解为不是一个数值。在计算机中，NaN通常用于表示无效的或未定义的操作结果，如0/0、∞-∞等1。","metadata":{}},{"cell_type":"code","source":"\"\"\"\n face: 0:468\n left_hand: 468:489\n pose: 489:522\n right_hand: 522:544\n 从file_path get data\n 第一行代码调用了load_relevant_data_subset函数，从文件路径中加载原始数据。\n 第二行代码调用了preprocess_layer函数，该函数使用Tensorflow处理数据。\n 最后返回处理后的数据。 \n\"\"\"\ndef get_data(file_path):\n # Load Raw Data\n data = load_relevant_data_subset(file_path)\n # Process Data Using Tensorflow\n data = preprocess_layer(data)\n \n return data","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:48:54.844024Z","iopub.execute_input":"2023-04-20T20:48:54.844403Z","iopub.status.idle":"2023-04-20T20:48:54.851163Z","shell.execute_reply.started":"2023-04-20T20:48:54.844361Z","shell.execute_reply":"2023-04-20T20:48:54.850163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Dataset","metadata":{}},{"cell_type":"code","source":"# Get the full dataset\ndef preprocess_data():\n # Create arrays to save data\n X = np.zeros([N_SAMPLES, INPUT_SIZE, N_COLS, N_DIMS], dtype=np.float32)\n y = np.zeros([N_SAMPLES], dtype=np.int32)\n NON_EMPTY_FRAME_IDXS = np.full([N_SAMPLES, INPUT_SIZE], -1, dtype=np.float32)\n\n # Fill X/y\n for row_idx, (file_path, sign_ord) in enumerate(tqdm(train[['file_path', 'sign_ord']].values)):\n # Log message every 5000 samples\n if row_idx % 5000 == 0:\n print(f'Generated {row_idx}/{N_SAMPLES}')\n\n data, non_empty_frame_idxs = get_data(file_path)\n X[row_idx] = data\n y[row_idx] = sign_ord\n NON_EMPTY_FRAME_IDXS[row_idx] = non_empty_frame_idxs\n # Sanity check, data should not contain NaN values\n if np.isnan(data).sum() > 0:\n print(row_idx)\n return data\n\n # Save X/y\n np.save('X.npy', X)\n np.save('y.npy', y)\n np.save('NON_EMPTY_FRAME_IDXS.npy', NON_EMPTY_FRAME_IDXS)\n \n # Save Validation\n splitter = GroupShuffleSplit(test_size=0.10, n_splits=2, random_state=SEED)\n PARTICIPANT_IDS = train['participant_id'].values\n train_idxs, val_idxs = next(splitter.split(X, y, groups=PARTICIPANT_IDS))\n\n # Save Train\n X_train = X[train_idxs]\n NON_EMPTY_FRAME_IDXS_TRAIN = NON_EMPTY_FRAME_IDXS[train_idxs]\n y_train = y[train_idxs]\n np.save('X_train.npy', X_train)\n np.save('y_train.npy', y_train)\n np.save('NON_EMPTY_FRAME_IDXS_TRAIN.npy', NON_EMPTY_FRAME_IDXS_TRAIN)\n # Save Validation\n X_val = X[val_idxs]\n NON_EMPTY_FRAME_IDXS_VAL = NON_EMPTY_FRAME_IDXS[val_idxs]\n y_val = y[val_idxs]\n np.save('X_val.npy', X_val)\n np.save('y_val.npy', y_val)\n np.save('NON_EMPTY_FRAME_IDXS_VAL.npy', NON_EMPTY_FRAME_IDXS_VAL)\n # Split Statistics\n print(f'Patient ID Intersection Train/Val: {set(PARTICIPANT_IDS[train_idxs]).intersection(PARTICIPANT_IDS[val_idxs])}')\n print(f'X_train shape: {X_train.shape}, X_val shape: {X_val.shape}')\n print(f'y_train shape: {y_train.shape}, y_val shape: {y_val.shape}')","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:48:54.852707Z","iopub.execute_input":"2023-04-20T20:48:54.853388Z","iopub.status.idle":"2023-04-20T20:48:54.872678Z","shell.execute_reply.started":"2023-04-20T20:48:54.853350Z","shell.execute_reply":"2023-04-20T20:48:54.871437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Preprocess All Data From Scratch\nif PREPROCESS_DATA:\n preprocess_data()\n ROOT_DIR = '.'\nelse:\n ROOT_DIR = '/kaggle/input/gislr-dataset-public'\n \n# Load Data\nif USE_VAL:\n # Load Train\n X_train = np.load(f'{ROOT_DIR}/X_train.npy')\n y_train = np.load(f'{ROOT_DIR}/y_train.npy')\n NON_EMPTY_FRAME_IDXS_TRAIN = np.load(f'{ROOT_DIR}/NON_EMPTY_FRAME_IDXS_TRAIN.npy')\n # Load Val\n X_val = np.load(f'{ROOT_DIR}/X_val.npy')\n y_val = np.load(f'{ROOT_DIR}/y_val.npy')\n NON_EMPTY_FRAME_IDXS_VAL = np.load(f'{ROOT_DIR}/NON_EMPTY_FRAME_IDXS_VAL.npy')\n # Define validation Data\n validation_data = ({ 'frames': X_val, 'non_empty_frame_idxs': NON_EMPTY_FRAME_IDXS_VAL }, y_val)\nelse:\n X_train = np.load(f'{ROOT_DIR}/X.npy')\n y_train = np.load(f'{ROOT_DIR}/y.npy')\n NON_EMPTY_FRAME_IDXS_TRAIN = np.load(f'{ROOT_DIR}/NON_EMPTY_FRAME_IDXS.npy')\n validation_data = None\n\n# Train \nprint_shape_dtype([X_train, y_train, NON_EMPTY_FRAME_IDXS_TRAIN], ['X_train', 'y_train', 'NON_EMPTY_FRAME_IDXS_TRAIN'])\n# Val\nif USE_VAL:\n print_shape_dtype([X_val, y_val, NON_EMPTY_FRAME_IDXS_VAL], ['X_val', 'y_val', 'NON_EMPTY_FRAME_IDXS_VAL'])\n# Sanity Check\nprint(f'# NaN Values X_train: {np.isnan(X_train).sum()}')","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:48:54.874190Z","iopub.execute_input":"2023-04-20T20:48:54.874721Z","iopub.status.idle":"2023-04-20T20:49:29.582066Z","shell.execute_reply.started":"2023-04-20T20:48:54.874682Z","shell.execute_reply":"2023-04-20T20:49:29.580861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Class Count\ndisplay(pd.Series(y_train).value_counts().to_frame('Class Count').iloc[[0,1,2,3,4, -5,-4,-3,-2,-1]])","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:49:29.583810Z","iopub.execute_input":"2023-04-20T20:49:29.584216Z","iopub.status.idle":"2023-04-20T20:49:29.599558Z","shell.execute_reply.started":"2023-04-20T20:49:29.584176Z","shell.execute_reply":"2023-04-20T20:49:29.597868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Number Of Frames","metadata":{}},{"cell_type":"markdown","source":"生成一个关于数据集中样本的非空帧数的瀑布图（Waterfall Plot）","metadata":{}},{"cell_type":"code","source":"# Vast majority of samples fits has less than 32 non empty frames\nN_EMPTY_FRAMES = (NON_EMPTY_FRAME_IDXS_TRAIN != -1).sum(axis=1) \nN_EMPTY_FRAMES_WATERFALL = []\nfor n in tqdm(range(1,INPUT_SIZE+1)):\n N_EMPTY_FRAMES_WATERFALL.append(sum(N_EMPTY_FRAMES >= n) / len(NON_EMPTY_FRAME_IDXS_TRAIN) * 100)\n\nplt.figure(figsize=(18,10))\nplt.title('Waterfall Plot For Number Of Non Empty Frames')\npd.Series(N_EMPTY_FRAMES_WATERFALL).plot(kind='bar')\nplt.grid(axis='y')\nplt.xticks(np.arange(INPUT_SIZE), np.arange(1, INPUT_SIZE+1))\nplt.xlabel('Number of Non Empty Frames', size=16)\nplt.yticks(np.arange(0, 100+10, 10))\nplt.ylim(0, 100)\nplt.ylabel('Percentage of Samples With At Least N Non Empty Frames', size=16)\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:49:29.600992Z","iopub.execute_input":"2023-04-20T20:49:29.601927Z","iopub.status.idle":"2023-04-20T20:49:43.077901Z","shell.execute_reply.started":"2023-04-20T20:49:29.601871Z","shell.execute_reply":"2023-04-20T20:49:43.076907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Percentage of Frames Filled","metadata":{}},{"cell_type":"code","source":"# Percentage of frames filled, this is the maximum fill percentage of each landmark\nP_DATA_FILLED = (NON_EMPTY_FRAME_IDXS_TRAIN != -1).sum() / NON_EMPTY_FRAME_IDXS_TRAIN.size * 100\nprint(f'P_DATA_FILLED: {P_DATA_FILLED:.2f}%')","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:49:43.078970Z","iopub.execute_input":"2023-04-20T20:49:43.079295Z","iopub.status.idle":"2023-04-20T20:49:43.095822Z","shell.execute_reply.started":"2023-04-20T20:49:43.079253Z","shell.execute_reply":"2023-04-20T20:49:43.094542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Statistics - Lips","metadata":{}},{"cell_type":"code","source":"# Percentage of Lips Measurements\nP_LEFT_LIPS_MEASUREMENTS = (X_train[:,:,LIPS_IDXS] != 0).sum() / X_train[:,:,LIPS_IDXS].size / P_DATA_FILLED * 1e4\nprint(f'P_LEFT_LIPS_MEASUREMENTS: {P_LEFT_LIPS_MEASUREMENTS:.2f}%')","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:49:43.097845Z","iopub.execute_input":"2023-04-20T20:49:43.098234Z","iopub.status.idle":"2023-04-20T20:50:00.932541Z","shell.execute_reply.started":"2023-04-20T20:49:43.098197Z","shell.execute_reply":"2023-04-20T20:50:00.931216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"这段代码的作用是计算嘴唇的均值和标准差。它使用了numpy和matplotlib库。其中，np.transpose函数将X_train数组的维度从(样本数，时间步数，特征数)转换为(特征数，时间步数，样本数)，然后reshape函数将其转换为(特征数，时间步数*样本数)的形状。最后，对于每个特征和每个样本，计算非零元素的均值和标准差，并将结果存储在LIPS_MEAN_X、LIPS_MEAN_Y、LIPS_STD_X和LIPS_STD_Y数组中。这些数组最终被组合成LIPS_MEAN和LIPS_STD数组，并作为函数的输出返回。","metadata":{}},{"cell_type":"code","source":"def get_lips_mean_std():\n # LIPS\n LIPS_MEAN_X = np.zeros([LIPS_IDXS.size], dtype=np.float32)\n LIPS_MEAN_Y = np.zeros([LIPS_IDXS.size], dtype=np.float32)\n LIPS_STD_X = np.zeros([LIPS_IDXS.size], dtype=np.float32)\n LIPS_STD_Y = np.zeros([LIPS_IDXS.size], dtype=np.float32)\n\n fig, axes = plt.subplots(3, 1, figsize=(15, N_DIMS*6))\n\n for col, ll in enumerate(tqdm( np.transpose(X_train[:,:,LIPS_IDXS], [2,3,0,1]).reshape([LIPS_IDXS.size, N_DIMS, -1]) )):\n for dim, l in enumerate(ll):\n v = l[np.nonzero(l)]\n if dim == 0: # X\n LIPS_MEAN_X[col] = v.mean()\n LIPS_STD_X[col] = v.std()\n if dim == 1: # Y\n LIPS_MEAN_Y[col] = v.mean()\n LIPS_STD_Y[col] = v.std()\n\n axes[dim].boxplot(v, notch=False, showfliers=False, positions=[col], whis=[5,95])\n\n for ax, dim_name in zip(axes, DIM_NAMES):\n ax.set_title(f'Lips {dim_name.upper()} Dimension', size=24)\n ax.tick_params(axis='x', labelsize=8)\n ax.grid(axis='y')\n\n plt.subplots_adjust(hspace=0.50)\n plt.show()\n\n LIPS_MEAN = np.array([LIPS_MEAN_X, LIPS_MEAN_Y]).T\n LIPS_STD = np.array([LIPS_STD_X, LIPS_STD_Y]).T\n \n return LIPS_MEAN, LIPS_STD\n\nLIPS_MEAN, LIPS_STD = get_lips_mean_std()","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:50:00.934291Z","iopub.execute_input":"2023-04-20T20:50:00.935074Z","iopub.status.idle":"2023-04-20T20:50:20.089219Z","shell.execute_reply.started":"2023-04-20T20:50:00.935028Z","shell.execute_reply":"2023-04-20T20:50:20.088080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Statistics - Hands","metadata":{}},{"cell_type":"code","source":"# Verify Normalised to Left Hand Dominant\nP_LEFT_HAND_MEASUREMENTS = (X_train[:,:,LEFT_HAND_IDXS] != 0).sum() / X_train[:,:,LEFT_HAND_IDXS].size / P_DATA_FILLED * 1e4\n# P_RIGHT_HAND_MEASUREMENTS = (X_train[:,:,RIGHT_HAND_IDXS] != 0).sum() / X_train[:,:,RIGHT_HAND_IDXS].size / P_DATA_FILLED * 1e4\nprint(f'P_LEFT_HAND_MEASUREMENTS: {P_LEFT_HAND_MEASUREMENTS:.2f}%')","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:50:20.090698Z","iopub.execute_input":"2023-04-20T20:50:20.091190Z","iopub.status.idle":"2023-04-20T20:50:29.395979Z","shell.execute_reply.started":"2023-04-20T20:50:20.091148Z","shell.execute_reply":"2023-04-20T20:50:29.394701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_left_right_hand_mean_std():\n # LEFT HAND\n LEFT_HANDS_MEAN_X = np.zeros([LEFT_HAND_IDXS.size], dtype=np.float32)\n LEFT_HANDS_MEAN_Y = np.zeros([LEFT_HAND_IDXS.size], dtype=np.float32)\n LEFT_HANDS_STD_X = np.zeros([LEFT_HAND_IDXS.size], dtype=np.float32)\n LEFT_HANDS_STD_Y = np.zeros([LEFT_HAND_IDXS.size], dtype=np.float32)\n\n fig, axes = plt.subplots(3, 1, figsize=(15, N_DIMS*6))\n\n for col, ll in enumerate(tqdm( np.transpose(X_train[:,:,LEFT_HAND_IDXS], [2,3,0,1]).reshape([LEFT_HAND_IDXS.size, N_DIMS, -1]) )):\n for dim, l in enumerate(ll):\n v = l[np.nonzero(l)]\n if dim == 0: # X\n LEFT_HANDS_MEAN_X[col] = v.mean()\n LEFT_HANDS_STD_X[col] = v.std()\n if dim == 1: # Y\n LEFT_HANDS_MEAN_Y[col] = v.mean()\n LEFT_HANDS_STD_Y[col] = v.std()\n # Plot\n axes[dim].boxplot(v, notch=False, showfliers=False, positions=[col], whis=[5,95])\n\n for ax, dim_name in zip(axes, DIM_NAMES):\n ax.set_title(f'Hands {dim_name.upper()} Dimension', size=24)\n ax.tick_params(axis='x', labelsize=8)\n ax.grid(axis='y')\n\n plt.subplots_adjust(hspace=0.50)\n plt.show()\n\n LEFT_HANDS_MEAN = np.array([LEFT_HANDS_MEAN_X, LEFT_HANDS_MEAN_Y]).T\n LEFT_HANDS_STD = np.array([LEFT_HANDS_STD_X, LEFT_HANDS_STD_Y]).T\n \n return LEFT_HANDS_MEAN, LEFT_HANDS_STD\n\nLEFT_HANDS_MEAN, LEFT_HANDS_STD = get_left_right_hand_mean_std()","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:50:29.397728Z","iopub.execute_input":"2023-04-20T20:50:29.398463Z","iopub.status.idle":"2023-04-20T20:50:40.841827Z","shell.execute_reply.started":"2023-04-20T20:50:29.398420Z","shell.execute_reply":"2023-04-20T20:50:40.840833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Statistics - Pose","metadata":{}},{"cell_type":"code","source":"# Percentage of Lips Measurements\nP_POSE_MEASUREMENTS = (X_train[:,:,POSE_IDXS] != 0).sum() / X_train[:,:,POSE_IDXS].size / P_DATA_FILLED * 1e4\nprint(f'P_POSE_MEASUREMENTS: {P_POSE_MEASUREMENTS:.2f}%')","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:50:40.843320Z","iopub.execute_input":"2023-04-20T20:50:40.846165Z","iopub.status.idle":"2023-04-20T20:50:42.727113Z","shell.execute_reply.started":"2023-04-20T20:50:40.846123Z","shell.execute_reply":"2023-04-20T20:50:42.725857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_pose_mean_std():\n # POSE\n POSE_MEAN_X = np.zeros([POSE_IDXS.size], dtype=np.float32)\n POSE_MEAN_Y = np.zeros([POSE_IDXS.size], dtype=np.float32)\n POSE_STD_X = np.zeros([POSE_IDXS.size], dtype=np.float32)\n POSE_STD_Y = np.zeros([POSE_IDXS.size], dtype=np.float32)\n\n fig, axes = plt.subplots(3, 1, figsize=(15, N_DIMS*6))\n\n for col, ll in enumerate(tqdm( np.transpose(X_train[:,:,POSE_IDXS], [2,3,0,1]).reshape([POSE_IDXS.size, N_DIMS, -1]) )):\n for dim, l in enumerate(ll):\n v = l[np.nonzero(l)]\n if dim == 0: # X\n POSE_MEAN_X[col] = v.mean()\n POSE_STD_X[col] = v.std()\n if dim == 1: # Y\n POSE_MEAN_Y[col] = v.mean()\n POSE_STD_Y[col] = v.std()\n\n axes[dim].boxplot(v, notch=False, showfliers=False, positions=[col], whis=[5,95])\n\n for ax, dim_name in zip(axes, DIM_NAMES):\n ax.set_title(f'Pose {dim_name.upper()} Dimension', size=24)\n ax.tick_params(axis='x', labelsize=8)\n ax.grid(axis='y')\n\n plt.subplots_adjust(hspace=0.50)\n plt.show()\n\n POSE_MEAN = np.array([POSE_MEAN_X, POSE_MEAN_Y]).T\n POSE_STD = np.array([POSE_STD_X, POSE_STD_Y]).T\n \n return POSE_MEAN, POSE_STD\n\nPOSE_MEAN, POSE_STD = get_pose_mean_std()","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:50:42.734766Z","iopub.execute_input":"2023-04-20T20:50:42.735313Z","iopub.status.idle":"2023-04-20T20:50:45.544913Z","shell.execute_reply.started":"2023-04-20T20:50:42.735280Z","shell.execute_reply":"2023-04-20T20:50:45.543876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Samples","metadata":{}},{"cell_type":"code","source":"# Custom sampler to get a batch containing N times all signs\ndef get_train_batch_all_signs(X, y, NON_EMPTY_FRAME_IDXS, n=BATCH_ALL_SIGNS_N):\n # Arrays to store batch in\n X_batch = np.zeros([NUM_CLASSES*n, INPUT_SIZE, N_COLS, N_DIMS], dtype=np.float32)\n y_batch = np.arange(0, NUM_CLASSES, step=1/n, dtype=np.float32).astype(np.int64)\n non_empty_frame_idxs_batch = np.zeros([NUM_CLASSES*n, INPUT_SIZE], dtype=np.float32)\n \n # Dictionary mapping ordinally encoded sign to corresponding sample indices\n CLASS2IDXS = {}\n for i in range(NUM_CLASSES):\n CLASS2IDXS[i] = np.argwhere(y == i).squeeze().astype(np.int32)\n \n while True:\n # Fill batch arrays\n for i in range(NUM_CLASSES):\n idxs = np.random.choice(CLASS2IDXS[i], n)\n X_batch[i*n:(i+1)*n] = X[idxs]\n non_empty_frame_idxs_batch[i*n:(i+1)*n] = NON_EMPTY_FRAME_IDXS[idxs]\n \n yield { 'frames': X_batch, 'non_empty_frame_idxs': non_empty_frame_idxs_batch }, y_batch","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:50:45.546754Z","iopub.execute_input":"2023-04-20T20:50:45.547813Z","iopub.status.idle":"2023-04-20T20:50:45.556712Z","shell.execute_reply.started":"2023-04-20T20:50:45.547765Z","shell.execute_reply":"2023-04-20T20:50:45.555649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"这段代码定义了一个生成器函数 get_train_batch_all_signs，用于生成指定数量（n）的所有手语标记的训练批次。\n\n该函数采用手语数据集（X 和 y）、非空帧索引集（NON_EMPTY_FRAME_IDXS）和一批次中所有手语标记的数量（n）作为输入，并生成包含 NUM_CLASSES * n 个样本的训练批次。这个训练批次包含一个 frames 字典和一个 non_empty_frame_idxs 字典，用于存储样本的手语帧和相应的非空帧索引。y_batch 数组包含了所有手语标记的序号。\n\n函数的主要逻辑是循环遍历所有手语标记，选择每个标记中的 n 个样本，并将它们添加到批次数组中。生成器会不停地循环生成这些样本，以便模型可以在整个训练过程中不断获得训练数据。","metadata":{}},{"cell_type":"code","source":"dummy_dataset = get_train_batch_all_signs(X_train, y_train, NON_EMPTY_FRAME_IDXS_TRAIN)\nX_batch, y_batch = next(dummy_dataset)\n\nfor k, v in X_batch.items():\n print(f'{k} shape: {v.shape}, dtype: {v.dtype}')\n\n# Batch shape/dtype\nprint(f'y_batch shape: {y_batch.shape}, dtype: {y_batch.dtype}')\n# Verify each batch contains each sign exactly N times\ndisplay(pd.Series(y_batch).value_counts().to_frame('Counts'))","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:50:45.558212Z","iopub.execute_input":"2023-04-20T20:50:45.558875Z","iopub.status.idle":"2023-04-20T20:50:45.641176Z","shell.execute_reply.started":"2023-04-20T20:50:45.558838Z","shell.execute_reply":"2023-04-20T20:50:45.640131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"这段代码的作用是从 X_train 和 y_train 数据集中随机选择 BATCH_ALL_SIGNS_N 个样本，生成一个包含所有手语标记的训练批次。\n\n生成器函数 get_train_batch_all_signs 将创建一个无限循环的生成器，它会生成包含所有手语标记的训练批次。为了方便测试，dummy_dataset 是从这个生成器中获取的一个批次，它包含了 X_batch、y_batch 和 NON_EMPTY_FRAME_IDXS_TRAIN 字典，以及一个包含所有手语标记的数量的常量 BATCH_ALL_SIGNS_N。\n\n在上述代码中，X_batch 是一个字典，它包含了 frames 和 non_empty_frame_idxs 字典，它们的形状和数据类型被打印出来。此外，还打印了 y_batch 数组的形状和数据类型。最后，使用 pd.Series(y_batch).value_counts() 函数验证每个手语标记都被包含了 BATCH_ALL_SIGNS_N 次。","metadata":{}},{"cell_type":"markdown","source":"# Model Config","metadata":{}},{"cell_type":"markdown","source":"这段代码定义了一些常量和变量，用于机器学习模型。\n\n`LAYER_NORM_EPS` 是一个常量，用于设置层归一化的 epsilon 值。\n\n`LIPS_UNITS`、`HANDS_UNITS`、`POSE_UNITS` 和 `UNITS` 是变量，用于设置关键点的密集层单元数、最终嵌入和变换器嵌入大小。\n\n`NUM_BLOCKS` 和 `MLP_RATIO` 是变量，用于设置变换器块和 MLP 比率的数量。\n\n`EMBEDDING_DROPOUT`、`MLP_DROPOUT_RATIO` 和 `CLASSIFIER_DROPOUT_RATIO` 是变量，用于设置嵌入、MLP 和分类器的 dropout 比率。\n\n`INIT_HE_UNIFORM`、`INIT_GLOROT_UNIFORM` 和 `INIT_ZEROS` 是变量，用于设置权重的初始化器。\n\n`GELU` 是一个变量，用于设置激活函数。\n\n最后一行打印出了 `UNITS` 的值。","metadata":{}},{"cell_type":"code","source":"# Epsilon value for layer normalisation\n#epsilon 值在机器学习中，层归一化是一种归一化技术，用于在神经网络的每个层中标准化输入。这有助于加速训练并提高模型的准确性。\n#在层归一化中，epsilon 值是一个常量，用于设置层归一化的 epsilon 值。它是一个非常小的数，通常设置为 1e-5 或 1e-6。\n#它的作用是防止分母为零，从而避免数值计算不稳定。\n\n\nLAYER_NORM_EPS = 1e-6\n\n# Dense layer units for landmarks\nLIPS_UNITS = 384\nHANDS_UNITS = 384\nPOSE_UNITS = 384\n# final embedding and transformer embedding size\nUNITS = 512\n\n# Transformer\n#mlp_ratio代表第一个全连接层上升通道倍数；\nNUM_BLOCKS = 2\nMLP_RATIO = 4\n\n# Dropout\n#模型很重要的性质就是非线性，\n#同时为了模型泛化能力，需要加入随机正则，例如dropout(随机置一些输出为0,其实也是一种变相的随机非线性激活)\nEMBEDDING_DROPOUT = 0.00\nMLP_DROPOUT_RATIO = 0.30\nCLASSIFIER_DROPOUT_RATIO = 0.00\n\n# Initiailizers\nINIT_HE_UNIFORM = tf.keras.initializers.he_uniform\nINIT_GLOROT_UNIFORM = tf.keras.initializers.glorot_uniform\nINIT_ZEROS = tf.keras.initializers.constant(0.0)\n# Activations\nGELU = tf.keras.activations.gelu\n\nprint(f'UNITS: {UNITS}')","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:50:45.642707Z","iopub.execute_input":"2023-04-20T20:50:45.643360Z","iopub.status.idle":"2023-04-20T20:50:45.651732Z","shell.execute_reply.started":"2023-04-20T20:50:45.643319Z","shell.execute_reply":"2023-04-20T20:50:45.650702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transformer\n\nNeed to implement transformer from scratch as TFLite does not support the native TF implementation of MultiHeadAttention.由于TFLite（TensorFlow Lite，一种针对移动和嵌入式设备进行优化的TensorFlow版本）不支持TensorFlow原生的MultiHeadAttention层实现，因此需要从头开始实现Transformer模型（一种在自然语言处理任务中使用的神经网络架构），其中包括MultiHeadAttention层。由于MultiHeadAttention层是Transformer模型的关键组成部分，因此如果TFLite不支持其实现，则可能无法在TFLite中使用TensorFlow中预先存在的Transformer实现。因此，需要自己编写Transformer的实现代码，而不依赖于TensorFlow中的MultiHeadAttention层的实现。","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"Encoder：输入是单词的Embedding，再加上位置编码，然后进入一个统一的结构，这个结构可以循环很多次（N次），也就是说有很多层（N层）。每一层又可以分成Attention层和全连接层，再额外加了一些处理，比如Skip Connection，做跳跃连接，然后还加了Normalization层。其实它本身的模型还是很简单的。\nDecoder：第一次输入是前缀信息，之后的就是上一次产出的Embedding，加入位置编码，然后进入一个可以重复很多次的模块。该模块可以分成三块来看，第一块也是Attention层，第二块是cross Attention，不是Self-Attention，第三块是全连接层。也用了跳跃连接和Normalization。\n输出：最后的输出要通过Linear层（全连接层），再通过softmax做预测。","metadata":{}},{"cell_type":"markdown","source":"Encoder部分是N个相同结构的堆叠，每个结构中又可以细分为如下结构：\n\n1. 对输入 one-hot 编码的样本进行 embedding（词嵌入）\n2. 加入位置编码\n3. 引入多头机制的 Self-Attention\n4. 将 self-attention 的输入和输出相加（残差网络结构）\n5. Layer Normalization（层标准化），对所有时刻的数据进行标准化\n6. 前馈型神经网络（Feedforword）结构\n7. 将 Feedforword 的输入和输出相加（残差网络结构）\n8. Layer Normalization，对所有时刻的数据进行标准化\n9. 重复N层3-8的结构","metadata":{}},{"cell_type":"markdown","source":"Decoder部分同样也是N个相同结构的堆叠，每个结构中又可以细分为如下结构：\n\n1. 对输入 one-hot 编码的样本进行 embedding（词嵌入）\n2. 加入位置编码\n3. 引入多头机制的 Self-Attention\n4. 将 self-attention 的输入和输出相加（残差网络结构）\n5. Layer Normalization（层标准化），\n6. 对所有时刻的数据进行标准化将上一步得到的只作为value，并和编码器端得到 q和k进行Self-Attenton\n7. 将 self-attention 的输入和输出相加（残差网络结构）\n8. Layer Normalization（层标准化），对所有时刻的数据进行标准化\n9. 前馈型神经网络（Feedforword） 结构\n10. 将 Feedforword 的输入和输出相加（残差网络结构）\n11. Layer Normalization，对所有时刻的数据进行标准化\n12. 重复N层3-11的结构","metadata":{}},{"cell_type":"code","source":"# based on: https://stackoverflow.com/\n#questions/67342988/verifying-the-implementation-of-multihead-attention-in-transformer\n# replaced softmax with softmax layer to support masked softmax\n#scaled dot-product attention是Transformer模型中的一种Attention机制，它是一种计算Attention权重的方法。\n#在这种方法中，Query和Key的点积被除以一个缩放因子，然后通过softmax函数进行归一化处理，最后与Value相乘得到Attention输出\ndef scaled_dot_product(q,k,v, softmax, attention_mask):\n #calculates Q . K(transpose)\n qkt = tf.matmul(q,k,transpose_b=True)\n #caculates scaling factor\n dk = tf.math.sqrt(tf.cast(q.shape[-1],dtype=tf.float32))\n scaled_qkt = qkt/dk\n softmax = softmax(scaled_qkt, mask=attention_mask)\n \n z = tf.matmul(softmax,v)\n #shape: (m,Tx,depth), same shape as q,k,v\n return z\n\nclass MultiHeadAttention(tf.keras.layers.Layer):\n def __init__(self,d_model,num_of_heads):\n super(MultiHeadAttention,self).__init__()\n self.d_model = d_model\n self.num_of_heads = num_of_heads\n self.depth = d_model//num_of_heads\n self.wq = [tf.keras.layers.Dense(self.depth) for i in range(num_of_heads)]\n self.wk = [tf.keras.layers.Dense(self.depth) for i in range(num_of_heads)]\n self.wv = [tf.keras.layers.Dense(self.depth) for i in range(num_of_heads)]\n self.wo = tf.keras.layers.Dense(d_model)\n self.softmax = tf.keras.layers.Softmax()\n \n def call(self,x, attention_mask):\n \n multi_attn = []\n for i in range(self.num_of_heads):\n Q = self.wq[i](x)\n K = self.wk[i](x)\n V = self.wv[i](x)\n multi_attn.append(scaled_dot_product(Q,K,V, self.softmax, attention_mask))\n \n multi_head = tf.concat(multi_attn,axis=-1)\n multi_head_attention = self.wo(multi_head)\n return multi_head_attention","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:50:45.653367Z","iopub.execute_input":"2023-04-20T20:50:45.653727Z","iopub.status.idle":"2023-04-20T20:50:45.666901Z","shell.execute_reply.started":"2023-04-20T20:50:45.653691Z","shell.execute_reply":"2023-04-20T20:50:45.665918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"这段代码是一个MultiHeadAttention的实现。它将输入张量x分别通过多个Dense层进行线性变换，然后将变换后的张量分别作为Q,K,V传入scaled_dot_product函数中，计算出多头注意力机制的输出。最后将多头注意力机制的输出拼接起来，再通过一个Dense层进行线性变换，得到最终的输出multi_head_attention。scaled_dot_product函数是计算Q.K^T的函数，其中Q,K,V分别为query,key,value矩阵，attention_mask是用于掩码的张量。softmax函数是用于计算softmax值的函数。","metadata":{}},{"cell_type":"markdown","source":"# transformer核心架构","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"# Full Transformer\nclass Transformer(tf.keras.Model):\n def __init__(self, num_blocks):\n super(Transformer, self).__init__(name='transformer')\n self.num_blocks = num_blocks\n \n def build(self, input_shape):\n self.ln_1s = []\n self.mhas = []\n self.ln_2s = []\n self.mlps = []\n # Make Transformer Blocks\n for i in range(self.num_blocks):\n # Multi Head Attention\n self.mhas.append(MultiHeadAttention(UNITS, 8))\n # Multi Layer Perception\n self.mlps.append(tf.keras.Sequential([\n tf.keras.layers.Dense(UNITS * MLP_RATIO, activation=GELU, kernel_initializer=INIT_GLOROT_UNIFORM),\n tf.keras.layers.Dropout(MLP_DROPOUT_RATIO),\n tf.keras.layers.Dense(UNITS, kernel_initializer=INIT_HE_UNIFORM),\n ]))\n \n def call(self, x, attention_mask):\n # Iterate input over transformer blocks\n for mha, mlp in zip(self.mhas, self.mlps):\n x = x + mha(x, attention_mask)\n x = x + mlp(x)\n \n return x","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:50:45.670055Z","iopub.execute_input":"2023-04-20T20:50:45.670359Z","iopub.status.idle":"2023-04-20T20:50:45.681546Z","shell.execute_reply.started":"2023-04-20T20:50:45.670317Z","shell.execute_reply":"2023-04-20T20:50:45.680613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"这是一个Python的Transformer模型，它是一个继承自tf.keras.Model的类。它有一个构造函数，其中num_blocks是一个整数，表示Transformer块的数量。在build函数中，它创建了多个Multi Head Attention和Multi Layer Perception对象，并将它们存储在类变量中。在call函数中，它迭代输入数据并将其传递给每个Transformer块。每个块都包含一个Multi Head Attention和一个Multi Layer Perception层。这个模型的目的是为了实现自然语言处理任务，如机器翻译、文本摘要等。","metadata":{}},{"cell_type":"markdown","source":"# Landmark Embedding\n关键点嵌入\"，其中\"Landmark\"表示人脸的关键点，\"Embedding\"表示将这些关键点信息映射到低维向量空间的过程。因此，\"Landmark Embedding\"的中文意思可以理解为\"将人脸关键点信息嵌入到低维向量空间中\"。","metadata":{}},{"cell_type":"markdown","source":"Landmark Embedding是一种将人脸关键点信息转换为低维向量表示的方法。在人脸识别和人脸表情识别等任务中，Landmark Embedding通常用于提取人脸特征表示。\n\n具体来说，Landmark Embedding通过对人脸图像中的关键点坐标进行处理，将其映射到一个低维空间中的向量表示。这个向量表示可以包含关于人脸形状、姿态和表情等信息，可以用于比较不同人脸之间的相似性或差异性。相比于直接使用像素信息或高维特征向量表示，Landmark Embedding可以提高人脸识别和表情识别的准确度和鲁棒性。","metadata":{}},{"cell_type":"code","source":"class LandmarkEmbedding(tf.keras.Model):\n def __init__(self, units, name):\n super(LandmarkEmbedding, self).__init__(name=f'{name}_embedding')\n self.units = units\n \n def build(self, input_shape):\n # Embedding for missing landmark in frame, initizlied with zeros\n self.empty_embedding = self.add_weight(\n name=f'{self.name}_empty_embedding',\n shape=[self.units],\n initializer=INIT_ZEROS,\n )\n # Embedding\n self.dense = tf.keras.Sequential([\n tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_1', use_bias=False, kernel_initializer=INIT_GLOROT_UNIFORM),\n tf.keras.layers.Activation(GELU),\n tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_2', use_bias=False, kernel_initializer=INIT_HE_UNIFORM),\n ], name=f'{self.name}_dense')\n\n def call(self, x):\n return tf.where(\n # Checks whether landmark is missing in frame\n tf.reduce_sum(x, axis=2, keepdims=True) == 0,\n # If so, the empty embedding is used\n self.empty_embedding,\n # Otherwise the landmark data is embedded\n self.dense(x),\n )","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:50:45.682821Z","iopub.execute_input":"2023-04-20T20:50:45.683391Z","iopub.status.idle":"2023-04-20T20:50:45.697014Z","shell.execute_reply.started":"2023-04-20T20:50:45.683351Z","shell.execute_reply":"2023-04-20T20:50:45.695930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Embedding","metadata":{}},{"cell_type":"code","source":"class Embedding(tf.keras.Model):\n def __init__(self):\n super(Embedding, self).__init__()\n \n def get_diffs(self, l):\n S = l.shape[2]\n other = tf.expand_dims(l, 3)\n other = tf.repeat(other, S, axis=3)\n other = tf.transpose(other, [0,1,3,2])\n diffs = tf.expand_dims(l, 3) - other\n diffs = tf.reshape(diffs, [-1, INPUT_SIZE, S*S])\n return diffs\n\n def build(self, input_shape):\n # Positional Embedding, initialized with zeros\n self.positional_embedding = tf.keras.layers.Embedding(INPUT_SIZE+1, UNITS, embeddings_initializer=INIT_ZEROS)\n # Embedding layer for Landmarks\n self.lips_embedding = LandmarkEmbedding(LIPS_UNITS, 'lips')\n self.left_hand_embedding = LandmarkEmbedding(HANDS_UNITS, 'left_hand')\n self.pose_embedding = LandmarkEmbedding(POSE_UNITS, 'pose')\n # Landmark Weights\n self.landmark_weights = tf.Variable(tf.zeros([3], dtype=tf.float32), name='landmark_weights')\n # Fully Connected Layers for combined landmarks\n self.fc = tf.keras.Sequential([\n tf.keras.layers.Dense(UNITS, name='fully_connected_1', use_bias=False, kernel_initializer=INIT_GLOROT_UNIFORM),\n tf.keras.layers.Activation(GELU),\n tf.keras.layers.Dense(UNITS, name='fully_connected_2', use_bias=False, kernel_initializer=INIT_HE_UNIFORM),\n ], name='fc')\n\n\n def call(self, lips0, left_hand0, pose0, non_empty_frame_idxs, training=False):\n # Lips\n lips_embedding = self.lips_embedding(lips0)\n # Left Hand\n left_hand_embedding = self.left_hand_embedding(left_hand0)\n # Pose\n pose_embedding = self.pose_embedding(pose0)\n # Merge Embeddings of all landmarks with mean pooling\n x = tf.stack((\n lips_embedding, left_hand_embedding, pose_embedding,\n ), axis=3)\n x = x * tf.nn.softmax(self.landmark_weights)\n x = tf.reduce_sum(x, axis=3)\n # Fully Connected Layers\n x = self.fc(x)\n # Add Positional Embedding\n max_frame_idxs = tf.clip_by_value(\n tf.reduce_max(non_empty_frame_idxs, axis=1, keepdims=True),\n 1,\n np.PINF,\n )\n normalised_non_empty_frame_idxs = tf.where(\n tf.math.equal(non_empty_frame_idxs, -1.0),\n INPUT_SIZE,\n tf.cast(\n non_empty_frame_idxs / max_frame_idxs * INPUT_SIZE,\n tf.int32,\n ),\n )\n x = x + self.positional_embedding(normalised_non_empty_frame_idxs)\n \n return x","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:50:45.699462Z","iopub.execute_input":"2023-04-20T20:50:45.700502Z","iopub.status.idle":"2023-04-20T20:50:45.715289Z","shell.execute_reply.started":"2023-04-20T20:50:45.700476Z","shell.execute_reply":"2023-04-20T20:50:45.714357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Augmentation数据增强","metadata":{}},{"cell_type":"markdown","source":"add_noise方法的作用是将输入张量t中的所有0替换为0，将所有非0元素加上一个服从正态分布的随机噪声。这个噪声的标准差是noise_std，它是在类的构造函数中定义的。这个方法使用了TensorFlow的tf.where函数，它接受三个参数：一个布尔型张量、一个张量x和一个张量y。如果布尔型张量中的元素为True，则返回x中对应位置的元素；否则返回y中对应位置的元素。在这个方法中，如果t中的元素为0，则返回0；否则返回t加上一个服从正态分布的随机噪声。\n如果训练标志为True，则会在每个张量上添加噪声。","metadata":{}},{"cell_type":"code","source":"# Not used, adds random X/y translation to input on samples level\nclass Augmentation(tf.keras.layers.Layer):\n def __init__(self, noise_std):\n super(Augmentation, self).__init__()\n self.noise_std = noise_std\n \n def add_noise(self, t):\n B = tf.shape(t)[0]\n return tf.where(\n t == 0.0,\n 0.0,\n t + tf.random.normal([B,1,1,tf.shape(t)[3]], 0, self.noise_std),\n )\n \n def call(self, lips0, left_hand0, pose0, training=False):\n if training:\n # Lips\n lips0 = self.add_noise(lips0)\n # Left Hand\n left_hand0 = self.add_noise(left_hand0)\n # Pose\n pose0 = self.add_noise(pose0)\n \n return lips0, left_hand0, pose0","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:50:45.718352Z","iopub.execute_input":"2023-04-20T20:50:45.718747Z","iopub.status.idle":"2023-04-20T20:50:45.729076Z","shell.execute_reply.started":"2023-04-20T20:50:45.718720Z","shell.execute_reply":"2023-04-20T20:50:45.728073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Sparse Categorical Crossentropy With Label Smoothing","metadata":{}},{"cell_type":"code","source":"# source:: https://stackoverflow.com/questions/60689185/label-smoothing-for-sparse-categorical-crossentropy\ndef scce_with_ls(y_true, y_pred):\n # One Hot Encode Sparsely Encoded Target Sign\n y_true = tf.cast(y_true, tf.int32)\n y_true = tf.one_hot(y_true, NUM_CLASSES, axis=1)\n y_true = tf.squeeze(y_true, axis=2)\n # Categorical Crossentropy with native label smoothing support\n return tf.keras.losses.categorical_crossentropy(y_true, y_pred, label_smoothing=0.25)","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:50:45.731856Z","iopub.execute_input":"2023-04-20T20:50:45.732480Z","iopub.status.idle":"2023-04-20T20:50:45.741391Z","shell.execute_reply.started":"2023-04-20T20:50:45.732443Z","shell.execute_reply":"2023-04-20T20:50:45.740441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"markdown","source":"这段代码是一个 TensorFlow 模型的实现。\n它有两个输入，分别是 \"frames\" 和 \"non_empty_frame_idxs\"。\n在这个模型中，frames 是一个包含多个帧的视频数据，而 non_empty_frame_idxs 表示在 frames 中哪些帧是有内容的。\n在这段代码中，将通过遮盖（masking）操作来选择具有有效帧的位置，以便只对这些帧进行训练。\n此模型使用了 Transformer 架构，它可以通过将多个带有注意力机制的层组合在一起来处理输入。\n在这个模型中，每个帧都被嵌入到三个不同的表示中，分别为 LIPS、LEFT HAND 和 POSE，这些表示被用于构建 Transformer 的输入。\n在这个模型中，还实现了一些额外的技巧，如随机帧屏蔽、类别丢失（分类时丢失部分特征），以及标签平滑等。\n最后，这个模型还包括一个优化器和一些评估指标。优化器使用 AdamW，而评估指标包括稀疏分类精度、稀疏分类前 k 个的精度等。","metadata":{}},{"cell_type":"code","source":"def get_model():\n # Inputs\n frames = tf.keras.layers.Input([INPUT_SIZE, N_COLS, N_DIMS], dtype=tf.float32, name='frames')\n non_empty_frame_idxs = tf.keras.layers.Input([INPUT_SIZE], dtype=tf.float32, name='non_empty_frame_idxs')\n # Padding Mask\n mask0 = tf.cast(tf.math.not_equal(non_empty_frame_idxs, -1), tf.float32)\n mask0 = tf.expand_dims(mask0, axis=2)\n # Random Frame Masking\n mask = tf.where(\n (tf.random.uniform(tf.shape(mask0)) > 0.25) & tf.math.not_equal(mask0, 0.0),\n 1.0,\n 0.0,\n )\n # Correct Samples Which are all masked now...\n mask = tf.where(\n tf.math.equal(tf.reduce_sum(mask, axis=[1,2], keepdims=True), 0.0),\n mask0,\n mask,\n )\n \n \n \"\"\"\n left_hand: 468:489\n pose: 489:522\n right_hand: 522:543\n \"\"\"\n x = frames\n x = tf.slice(x, [0,0,0,0], [-1,INPUT_SIZE, N_COLS, 2])\n # LIPS\n lips = tf.slice(x, [0,0,LIPS_START,0], [-1,INPUT_SIZE, 40, 2])\n lips = tf.where(\n tf.math.equal(lips, 0.0),\n 0.0,\n (lips - LIPS_MEAN) / LIPS_STD,\n )\n # LEFT HAND\n left_hand = tf.slice(x, [0,0,40,0], [-1,INPUT_SIZE, 21, 2])\n left_hand = tf.where(\n tf.math.equal(left_hand, 0.0),\n 0.0,\n (left_hand - LEFT_HANDS_MEAN) / LEFT_HANDS_STD,\n )\n # POSE\n pose = tf.slice(x, [0,0,61,0], [-1,INPUT_SIZE, 5, 2])\n pose = tf.where(\n tf.math.equal(pose, 0.0),\n 0.0,\n (pose - POSE_MEAN) / POSE_STD,\n )\n \n # Flatten\n lips = tf.reshape(lips, [-1, INPUT_SIZE, 40*2])\n left_hand = tf.reshape(left_hand, [-1, INPUT_SIZE, 21*2])\n pose = tf.reshape(pose, [-1, INPUT_SIZE, 5*2])\n \n # Embedding\n x = Embedding()(lips, left_hand, pose, non_empty_frame_idxs)\n \n # Encoder Transformer Blocks\n x = Transformer(NUM_BLOCKS)(x, mask)\n \n # Pooling\n x = tf.reduce_sum(x * mask, axis=1) / tf.reduce_sum(mask, axis=1)\n # Classifier Dropout\n x = tf.keras.layers.Dropout(CLASSIFIER_DROPOUT_RATIO)(x)\n # Classification Layer\n x = tf.keras.layers.Dense(NUM_CLASSES, activation=tf.keras.activations.softmax, kernel_initializer=INIT_GLOROT_UNIFORM)(x)\n \n outputs = x\n \n # Create Tensorflow Model\n model = tf.keras.models.Model(inputs=[frames, non_empty_frame_idxs], outputs=outputs)\n \n # Sparse Categorical Cross Entropy With Label Smoothing\n loss = scce_with_ls\n #SGDW是一种优化器，它是基于SGD的，但是加入了动量的概念。\n #动量的作用是在更新参数时，不仅仅减去了当前迭代的梯度，还减去了前t-1迭代的梯度的加权和。\n #这样做的好处是可以让参数更新更加平滑，避免了在参数更新过程中出现震荡的情况。\n #optimizer = tfa.optimizers.SGDW(\n #learning_rate=lr, weight_decay=wd, momentum=0.9)\n #optimizer = tf.keras.optimizers.SGD(lr=0.001, momentum=0.0, nesterov=False) \n #optimizer = tfa.optimizers.SGDW(learning_rate=0.001, momentum=0.7, weight_decay=0.005)\n #Adam Optimizer with weight decay\n optimizer = tfa.optimizers.AdamW(learning_rate=1e-3, weight_decay=1e-5, clipnorm=1.0)\n #学习率为1e-3，权重衰减为1e-5，梯度裁剪阈值为1.0\n # TopK Metrics\n metrics = [\n tf.keras.metrics.SparseCategoricalAccuracy(name='acc'),\n tf.keras.metrics.SparseTopKCategoricalAccuracy(k=5, name='top_5_acc'),\n tf.keras.metrics.SparseTopKCategoricalAccuracy(k=10, name='top_10_acc'),\n ]\n \n model.compile(loss=loss, optimizer=optimizer, metrics=metrics)\n \n return model","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:50:45.744436Z","iopub.execute_input":"2023-04-20T20:50:45.744729Z","iopub.status.idle":"2023-04-20T20:50:45.762496Z","shell.execute_reply.started":"2023-04-20T20:50:45.744704Z","shell.execute_reply":"2023-04-20T20:50:45.761453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.backend.clear_session()\n\nmodel = get_model()","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:50:45.764031Z","iopub.execute_input":"2023-04-20T20:50:45.764360Z","iopub.status.idle":"2023-04-20T20:50:49.241064Z","shell.execute_reply.started":"2023-04-20T20:50:45.764325Z","shell.execute_reply":"2023-04-20T20:50:49.240014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot model summary\nmodel.summary(expand_nested=True)","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:50:49.242745Z","iopub.execute_input":"2023-04-20T20:50:49.243128Z","iopub.status.idle":"2023-04-20T20:50:49.416677Z","shell.execute_reply.started":"2023-04-20T20:50:49.243085Z","shell.execute_reply":"2023-04-20T20:50:49.415927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.utils.plot_model(model, show_shapes=True, show_dtype=True, show_layer_names=True, expand_nested=True, show_layer_activations=True)","metadata":{"_kg_hide-input":false,"scrolled":true,"execution":{"iopub.status.busy":"2023-04-20T20:50:49.417716Z","iopub.execute_input":"2023-04-20T20:50:49.418418Z","iopub.status.idle":"2023-04-20T20:50:50.385771Z","shell.execute_reply.started":"2023-04-20T20:50:49.418379Z","shell.execute_reply":"2023-04-20T20:50:50.384688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# No NaN Predictions","metadata":{}},{"cell_type":"code","source":"if not PREPROCESS_DATA and TRAIN_MODEL:\n y_pred = model.predict_on_batch(X_batch).flatten()\n\n print(f'# NaN Values In Prediction: {np.isnan(y_pred).sum()}')","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:50:50.387910Z","iopub.execute_input":"2023-04-20T20:50:50.388552Z","iopub.status.idle":"2023-04-20T20:50:54.470866Z","shell.execute_reply.started":"2023-04-20T20:50:50.388508Z","shell.execute_reply":"2023-04-20T20:50:54.469615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Weight Initialization","metadata":{}},{"cell_type":"code","source":"if not PREPROCESS_DATA and TRAIN_MODEL:\n plt.figure(figsize=(12,5))\n plt.title(f'Softmax Output Initialized Model | µ={y_pred.mean():.3f}, σ={y_pred.std():.3f}', pad=25)\n pd.Series(y_pred).plot(kind='hist', bins=128, label='Class Probability')\n plt.xlim(0, max(y_pred) * 1.1)\n plt.vlines([1 / NUM_CLASSES], 0, plt.ylim()[1], color='red', label=f'Random Guessing Baseline 1/NUM_CLASSES={1 / NUM_CLASSES:.3f}')\n plt.grid()\n plt.legend()\n plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:50:54.472457Z","iopub.execute_input":"2023-04-20T20:50:54.473531Z","iopub.status.idle":"2023-04-20T20:50:55.012915Z","shell.execute_reply.started":"2023-04-20T20:50:54.473485Z","shell.execute_reply":"2023-04-20T20:50:55.011825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Learning Rate Scheduler","metadata":{}},{"cell_type":"code","source":"def lrfn(current_step, num_warmup_steps, lr_max, num_cycles=0.50, num_training_steps=N_EPOCHS):\n \n if current_step < num_warmup_steps:\n if WARMUP_METHOD == 'log':\n return lr_max * 0.10 ** (num_warmup_steps - current_step)\n else:\n return lr_max * 2 ** -(num_warmup_steps - current_step)\n else:\n progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))\n\n return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))) * lr_max","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:50:55.014459Z","iopub.execute_input":"2023-04-20T20:50:55.017576Z","iopub.status.idle":"2023-04-20T20:50:55.024629Z","shell.execute_reply.started":"2023-04-20T20:50:55.017542Z","shell.execute_reply":"2023-04-20T20:50:55.023333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_lr_schedule(lr_schedule, epochs):\n fig = plt.figure(figsize=(20, 10))\n plt.plot([None] + lr_schedule + [None])\n # X Labels\n x = np.arange(1, epochs + 1)\n x_axis_labels = [i if epochs <= 40 or i % 5 == 0 or i == 1 else None for i in range(1, epochs + 1)]\n plt.xlim([1, epochs])\n plt.xticks(x, x_axis_labels) # set tick step to 1 and let x axis start at 1\n \n # Increase y-limit for better readability\n plt.ylim([0, max(lr_schedule) * 1.1])\n \n # Title\n schedule_info = f'start: {lr_schedule[0]:.1E}, max: {max(lr_schedule):.1E}, final: {lr_schedule[-1]:.1E}'\n plt.title(f'Step Learning Rate Schedule, {schedule_info}', size=18, pad=12)\n \n # Plot Learning Rates\n for x, val in enumerate(lr_schedule):\n if epochs <= 40 or x % 5 == 0 or x is epochs - 1:\n if x < len(lr_schedule) - 1:\n if lr_schedule[x - 1] < val:\n ha = 'right'\n else:\n ha = 'left'\n elif x == 0:\n ha = 'right'\n else:\n ha = 'left'\n plt.plot(x + 1, val, 'o', color='black');\n offset_y = (max(lr_schedule) - min(lr_schedule)) * 0.02\n plt.annotate(f'{val:.1E}', xy=(x + 1, val + offset_y), size=12, ha=ha)\n \n plt.xlabel('Epoch', size=16, labelpad=5)\n plt.ylabel('Learning Rate', size=16, labelpad=5)\n plt.grid()\n plt.show()\n\n# Learning rate for encoder\nLR_SCHEDULE = [lrfn(step, num_warmup_steps=N_WARMUP_EPOCHS, lr_max=LR_MAX, num_cycles=0.50) for step in range(N_EPOCHS)]\n# Plot Learning Rate Schedule\nplot_lr_schedule(LR_SCHEDULE, epochs=N_EPOCHS)\n# Learning Rate Callback\nlr_callback = tf.keras.callbacks.LearningRateScheduler(lambda step: LR_SCHEDULE[step], verbose=1)","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:50:55.026586Z","iopub.execute_input":"2023-04-20T20:50:55.027043Z","iopub.status.idle":"2023-04-20T20:50:55.770830Z","shell.execute_reply.started":"2023-04-20T20:50:55.027006Z","shell.execute_reply":"2023-04-20T20:50:55.769851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Weight Decay Callback","metadata":{}},{"cell_type":"code","source":"# Custom callback to update weight decay with learning rate\nclass WeightDecayCallback(tf.keras.callbacks.Callback):\n def __init__(self, wd_ratio=WD_RATIO):\n self.step_counter = 0\n self.wd_ratio = wd_ratio\n \n def on_epoch_begin(self, epoch, logs=None):\n model.optimizer.weight_decay = model.optimizer.learning_rate * self.wd_ratio\n print(f'learning rate: {model.optimizer.learning_rate.numpy():.2e}, weight decay: {model.optimizer.weight_decay.numpy():.2e}')","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:50:55.772405Z","iopub.execute_input":"2023-04-20T20:50:55.773565Z","iopub.status.idle":"2023-04-20T20:50:55.780410Z","shell.execute_reply.started":"2023-04-20T20:50:55.773521Z","shell.execute_reply":"2023-04-20T20:50:55.779413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Performance Benchmark","metadata":{}},{"cell_type":"code","source":"%%timeit -n 100\nif TRAIN_MODEL:\n # Verify model prediction is <<<100ms\n model.predict_on_batch({ 'frames': X_train[:1], 'non_empty_frame_idxs': NON_EMPTY_FRAME_IDXS_TRAIN[:1] })\n pass","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:50:55.781818Z","iopub.execute_input":"2023-04-20T20:50:55.782256Z","iopub.status.idle":"2023-04-20T20:51:13.880600Z","shell.execute_reply.started":"2023-04-20T20:50:55.782218Z","shell.execute_reply":"2023-04-20T20:51:13.879421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"if USE_VAL:\n # Verify Validation Dataset Covers All Signs\n print(f'# Unique Signs in Validation Set: {pd.Series(y_val).nunique()}')\n # Value Counts\n display(pd.Series(y_val).value_counts().to_frame('Count').iloc[[1,2,3,-3,-2,-1]])","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:51:13.882570Z","iopub.execute_input":"2023-04-20T20:51:13.882951Z","iopub.status.idle":"2023-04-20T20:51:13.888686Z","shell.execute_reply.started":"2023-04-20T20:51:13.882911Z","shell.execute_reply":"2023-04-20T20:51:13.887411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluate Initialzied Model","metadata":{}},{"cell_type":"code","source":"# Sanity Check\nif TRAIN_MODEL and USE_VAL:\n _ = model.evaluate(*validation_data, verbose=2)","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:51:13.890451Z","iopub.execute_input":"2023-04-20T20:51:13.891146Z","iopub.status.idle":"2023-04-20T20:51:13.899547Z","shell.execute_reply.started":"2023-04-20T20:51:13.891106Z","shell.execute_reply":"2023-04-20T20:51:13.898846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"if TRAIN_MODEL:\n # Clear all models in GPU\n tf.keras.backend.clear_session()\n\n # Get new fresh model\n model = get_model()\n \n # Sanity Check\n model.summary()\n\n # Actual Training\n history = model.fit(\n x=get_train_batch_all_signs(X_train, y_train, NON_EMPTY_FRAME_IDXS_TRAIN),\n steps_per_epoch=len(X_train) // (NUM_CLASSES * BATCH_ALL_SIGNS_N),\n epochs=N_EPOCHS,\n # Only used for validation data since training data is a generator\n #\"只用于验证数据，因为训练数据是生成器\"。\n #如果使用生成器来训练模型，则必须使用验证数据来评估模型的性能。\n #这是因为生成器在每个时期中都会生成新的数据，而不是将所有数据加载到内存中。\n #因此，无法在训练期间使用训练数据来评估模型的性能。相反，必须使用验证数据来评估模型的性能.\n batch_size=BATCH_SIZE,\n validation_data=validation_data,\n callbacks=[\n lr_callback,\n WeightDecayCallback(),\n ],\n verbose = 2,\n )","metadata":{"execution":{"iopub.status.busy":"2023-04-20T20:51:13.901022Z","iopub.execute_input":"2023-04-20T20:51:13.901700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"model.fit()是TensorFlow中的一种方法，用于在给定的数据集上训练模型。它接受多个参数，例如训练数据、验证数据、时期数、批量大小等。它通过使用优化器来最小化损失函数来训练模型。损失函数是衡量模型在预测输出方面表现如何的一种方法。优化器调整模型的权重以最小化此损失函数。在训练期间，model.fit()会打印出诸如损失和准确性之类的指标","metadata":{}},{"cell_type":"code","source":"# Save Model Weights\nmodel.save_weights('model.h5')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if USE_VAL:\n # Validation Predictions\n y_val_pred = model.predict({ 'frames': X_val, 'non_empty_frame_idxs': NON_EMPTY_FRAME_IDXS_VAL }, verbose=2).argmax(axis=1)\n # Label\n labels = [ORD2SIGN.get(i).replace(' ', '_') for i in range(NUM_CLASSES)]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Landmark Attention Weights","metadata":{}},{"cell_type":"code","source":"# Landmark Weights\nfor w in model.get_layer('embedding').weights:\n if 'landmark_weights' in w.name:\n weights = scipy.special.softmax(w)\n\nlandmarks = ['lips_embedding', 'left_hand_embedding', 'pose_embedding']\n\nfor w, lm in zip(weights, landmarks):\n print(f'{lm} weight: {(w*100):.1f}%')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Classification Report","metadata":{}},{"cell_type":"code","source":"def print_classification_report():\n # Classification report for all signs\n classification_report = sklearn.metrics.classification_report(\n y_val,\n y_val_pred,\n target_names=labels,\n output_dict=True,\n )\n # Round Data for better readability\n classification_report = pd.DataFrame(classification_report).T\n classification_report = classification_report.round(2)\n classification_report = classification_report.astype({\n 'support': np.uint16,\n })\n # Add signs\n classification_report['sign'] = [e if e in SIGN2ORD else -1 for e in classification_report.index]\n classification_report['sign_ord'] = classification_report['sign'].apply(SIGN2ORD.get).fillna(-1).astype(np.int16)\n # Sort on F1-score\n classification_report = pd.concat((\n classification_report.head(NUM_CLASSES).sort_values('f1-score', ascending=False),\n classification_report.tail(3),\n ))\n\n pd.options.display.max_rows = 999\n display(classification_report)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if USE_VAL:\n print_classification_report()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training History","metadata":{}},{"cell_type":"markdown","source":"画出训练曲线图","metadata":{}},{"cell_type":"code","source":"def plot_history_metric(metric, f_best=np.argmax, ylim=None, yscale=None, yticks=None):\n plt.figure(figsize=(20, 10))\n \n values = history.history[metric]\n N_EPOCHS = len(values)\n val = 'val' in ''.join(history.history.keys())\n # Epoch Ticks\n if N_EPOCHS <= 20:\n x = np.arange(1, N_EPOCHS + 1)\n else:\n x = [1, 5] + [10 + 5 * idx for idx in range((N_EPOCHS - 10) // 5 + 1)]\n\n x_ticks = np.arange(1, N_EPOCHS+1)\n\n # Validation\n if val:\n val_values = history.history[f'val_{metric}']\n val_argmin = f_best(val_values)\n plt.plot(x_ticks, val_values, label=f'val')\n\n # summarize history for accuracy\n plt.plot(x_ticks, values, label=f'train')\n argmin = f_best(values)\n plt.scatter(argmin + 1, values[argmin], color='red', s=75, marker='o', label=f'train_best')\n if val:\n plt.scatter(val_argmin + 1, val_values[val_argmin], color='purple', s=75, marker='o', label=f'val_best')\n\n plt.title(f'Model {metric}', fontsize=24, pad=10)\n plt.ylabel(metric, fontsize=20, labelpad=10)\n\n if ylim:\n plt.ylim(ylim)\n\n if yscale is not None:\n plt.yscale(yscale)\n \n if yticks is not None:\n plt.yticks(yticks, fontsize=16)\n\n plt.xlabel('epoch', fontsize=20, labelpad=10) \n plt.tick_params(axis='x', labelsize=8)\n plt.xticks(x, fontsize=16) # set tick step to 1 and let x axis start at 1\n plt.yticks(fontsize=16)\n \n plt.legend(prop={'size': 10})\n plt.grid()\n plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if TRAIN_MODEL:\n plot_history_metric('loss', f_best=np.argmin)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if TRAIN_MODEL:\n plot_history_metric('acc', ylim=[0,1], yticks=np.arange(0.0, 1.1, 0.1))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if TRAIN_MODEL:\n plot_history_metric('top_5_acc', ylim=[0,1], yticks=np.arange(0.0, 1.1, 0.1))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if TRAIN_MODEL:\n plot_history_metric('top_10_acc', ylim=[0,1], yticks=np.arange(0.0, 1.1, 0.1))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission\n\nSubmission code loosley based on [this notebook](https://www.kaggle.com/code/dschettler8845/gislr-learn-eda-baseline#baseline) by [Darien Schettler\n](https://www.kaggle.com/dschettler8845)","metadata":{}},{"cell_type":"code","source":"# TFLite model for submission\nclass TFLiteModel(tf.Module):\n def __init__(self, model):\n super(TFLiteModel, self).__init__()\n\n # Load the feature generation and main models\n self.preprocess_layer = preprocess_layer\n self.model = model\n \n @tf.function(input_signature=[tf.TensorSpec(shape=[None, N_ROWS, N_DIMS], dtype=tf.float32, name='inputs')])\n def __call__(self, inputs):\n # Preprocess Data\n x, non_empty_frame_idxs = self.preprocess_layer(inputs)\n # Add Batch Dimension\n x = tf.expand_dims(x, axis=0)\n non_empty_frame_idxs = tf.expand_dims(non_empty_frame_idxs, axis=0)\n # Make Prediction\n outputs = self.model({ 'frames': x, 'non_empty_frame_idxs': non_empty_frame_idxs })\n # Squeeze Output 1x250 -> 250\n outputs = tf.squeeze(outputs, axis=0)\n\n # Return a dictionary with the output tensor\n return {'outputs': outputs}\n\n# Define TF Lite Model\ntflite_keras_model = TFLiteModel(model)\n\n# Sanity Check\ndemo_raw_data = load_relevant_data_subset(train['file_path'].values[5])\nprint(f'demo_raw_data shape: {demo_raw_data.shape}, dtype: {demo_raw_data.dtype}')\ndemo_output = tflite_keras_model(demo_raw_data)[\"outputs\"]\nprint(f'demo_output shape: {demo_output.shape}, dtype: {demo_output.dtype}')\ndemo_prediction = demo_output.numpy().argmax()\nprint(f'demo_prediction: {demo_prediction}, correct: {train.iloc[0][\"sign_ord\"]}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create Model Converter\nkeras_model_converter = tf.lite.TFLiteConverter.from_keras_model(tflite_keras_model)\n# Convert Model\ntflite_model = keras_model_converter.convert()\n# Write Model\nwith open('/kaggle/working/model.tflite', 'wb') as f:\n f.write(tflite_model)\n \n# Zip Model\n!zip submission.zip /kaggle/working/model.tflite","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Verify TFLite model can be loaded and used for prediction\n!pip install tflite-runtime\nimport tflite_runtime.interpreter as tflite\n\ninterpreter = tflite.Interpreter(\"/kaggle/working/model.tflite\")\nfound_signatures = list(interpreter.get_signature_list().keys())\nprediction_fn = interpreter.get_signature_runner(\"serving_default\")\n\noutput = prediction_fn(inputs=demo_raw_data)\nsign = output['outputs'].argmax()\n\nprint(\"PRED : \", ORD2SIGN.get(sign), f'[{sign}]')\nprint(\"TRUE : \", train.sign.values[0], f'[{train.sign_ord.values[0]}]')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"兄弟们能不能点个赞，球球了\n\n认可就是一种动力\n\n我会接着尽量完善注释的","metadata":{}},{"cell_type":"markdown","source":"# 最后，给大家表演一个后空翻\n# ጿ ኈ ቼ ዽ ጿ ኈ ቼ ዽ ጿ ኈ ቼ ዽ ጿ ኈ ቼ ዽ ጿ ኈ ቼ\n# (￣▽￣) ~*(￣▽￣)／","metadata":{}}]}