{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":46105,"databundleVersionId":5087314,"sourceType":"competition"},{"sourceId":5315518,"sourceType":"datasetVersion","datasetId":3036481}],"dockerImageVersionId":30408,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook will first do simple Exploratory Data Analysis (EDA) of the dataset. And then train the model for sign classification.\n\nThe data analysis and visualization/animation is based on notebooks here: \n * https://www.kaggle.com/code/danielpeshkov/animated-data-visualization\n * https://www.kaggle.com/code/dschettler8845/gislr-learn-eda-baseline\n * https://www.kaggle.com/code/ted0071/gislr-visualization\n\nThis notebook's model is based on this one: https://www.kaggle.com/code/markwijkhuizen/gislr-tf-data-processing-transformer-training with edits:\n * Learn data preprocessing\n * Reproduce the results\n * twist the models and/or features\n \n LSTM baseline with original notebook\n\n\n0.6545733\n\nPublic Score\n0.5624196","metadata":{}},{"cell_type":"code","source":"!pip install -q flatbuffers 2> /dev/null\n!pip install -q mediapipe 2> /dev/null\n","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:58:18.596983Z","iopub.execute_input":"2024-02-25T18:58:18.597680Z","iopub.status.idle":"2024-02-25T18:58:45.181449Z","shell.execute_reply.started":"2024-02-25T18:58:18.597648Z","shell.execute_reply":"2024-02-25T18:58:45.180275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Animation Libraries Used\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom matplotlib.animation import FuncAnimation\nfrom IPython.display import HTML\n\n## Data analysis libraries used\n\n\n\n## Model training libraies\n\nimport numpy as np\n\nimport tensorflow as tf\nimport tensorflow_addons as tfa\n\nimport matplotlib as mpl\nimport seaborn as sn\n\nfrom tqdm.notebook import tqdm\nfrom sklearn.model_selection import train_test_split, GroupShuffleSplit \n\nimport os\nimport json\nimport glob\nimport sys\n\nimport math\nimport gc\nimport sys\nimport sklearn\nimport scipy\n\n\nimport cv2\nimport matplotlib.pyplot as plt\nfrom matplotlib import animation\nfrom pathlib import Path\nimport IPython\nfrom IPython.display import display\nfrom IPython.display import HTML\n\nimport mediapipe as mp\nfrom mediapipe.framework.formats import landmark_pb2\n\n\nprint(f'Tensorflow V{tf.__version__}')\nprint(f'Keras V{tf.keras.__version__}')\nprint(f'Python V{sys.version}')\n\nprint(f' V{sys.version}')\n","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:58:45.183702Z","iopub.execute_input":"2024-02-25T18:58:45.184021Z","iopub.status.idle":"2024-02-25T18:58:53.527911Z","shell.execute_reply.started":"2024-02-25T18:58:45.183988Z","shell.execute_reply":"2024-02-25T18:58:53.526796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Part 1: Load Data and Simple Statistics and Visulization","metadata":{}},{"cell_type":"code","source":"DATA_DIR = \"/kaggle/input/asl-signs\"\n# LANDMARK_FILES_DIR = os.path.join(DATA_DIR, \"train_landmark_files\")\nTRAIN_FILE = os.path.join(DATA_DIR, \"train.csv\")\nLABEL_MAP = json.load(open(os.path.join(DATA_DIR, \"sign_to_prediction_index_map.json\"), \"r\"))\n\nprint(TRAIN_FILE)\n# print(LABEL_MAP)","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:58:53.529266Z","iopub.execute_input":"2024-02-25T18:58:53.530002Z","iopub.status.idle":"2024-02-25T18:58:53.540010Z","shell.execute_reply.started":"2024-02-25T18:58:53.529966Z","shell.execute_reply":"2024-02-25T18:58:53.538994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1.1 Training set","metadata":{}},{"cell_type":"code","source":"# how many training example (videos of signs)\ntrain_df = pd.read_csv(TRAIN_FILE)\nprint(train_df.info())\ndisplay(train_df.head())\n","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:58:53.544101Z","iopub.execute_input":"2024-02-25T18:58:53.544684Z","iopub.status.idle":"2024-02-25T18:58:53.771292Z","shell.execute_reply.started":"2024-02-25T18:58:53.544653Z","shell.execute_reply":"2024-02-25T18:58:53.770251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sign (label) statistics - how many labels in the training set\nprint(train_df[\"sign\"].describe().to_frame().T)\nsign_count_map = train_df[\"sign\"].value_counts().to_dict()\nprint(\"\\t1. Number Of Unique Signs -->\", len(sign_count_map))\nprint(\"\\t2. Average Number of Rows Per Sign -->\", np.array(list(sign_count_map.values())).mean())\nprint(\"\\t3. Standard Deviation in Counts Per Sign -->\", np.array(list(sign_count_map.values())).std())\nprint(\"\\t4. Minimum Number of Examples For One Sign -->\", np.array(list(sign_count_map.values())).min())\nprint(\"\\t5. Maximum Number of Examples For One Sign -->\", np.array(list(sign_count_map.values())).max())\n\n# print(\"\\n\\n... SIGN COLUMN HISTOGRAM:\\n\")\n# fig = px.histogram(train_df, y=train_df[\"sign\"], color=\"sign\", orientation=\"h\", height=5000,\n# labels={\"y\":\"Sign (label)\", \"count\":\"Total Row Count\"}, title=\"Row Counts by Sign (label)\",\n# category_orders={\"sign\": train_df[\"sign\"].value_counts().index}\n# )\n# fig.update_yaxes(title_text=\"Total Row Count\")\n# fig.update_layout(showlegend=False)\n# fig.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:58:53.772588Z","iopub.execute_input":"2024-02-25T18:58:53.772900Z","iopub.status.idle":"2024-02-25T18:58:53.816176Z","shell.execute_reply.started":"2024-02-25T18:58:53.772870Z","shell.execute_reply":"2024-02-25T18:58:53.815072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# training example from how many participants\n\nprint(\"\\n... BASICS OF THE PARTICIPANT ID COLUMN:\\n\")\nprint(train_df[\"participant_id\"].astype(str).describe().to_frame().T)\n\nparticipant_count_map = train_df[\"participant_id\"].value_counts().to_dict()\nprint(\"\\t1. Number of Unique Participants -->\", len(participant_count_map))\nprint(\"\\t2. Average Number of Rows Per Participant -->\", np.array(list(participant_count_map.values())).mean())\nprint(\"\\t3. Standard Deviation in Counts Per Participant -->\", np.array(list(participant_count_map.values())).std())\nprint(\"\\t4. Minimum Number of Examples For One Participant -->\", np.array(list(participant_count_map.values())).min())\nprint(\"\\t5. Maximum Number of Examples For One Participant -->\", np.array(list(participant_count_map.values())).max())\n\n# print(\"\\n\\n... PARTICIPANT ID COLUMN HISTOGRAM:\\n\")\n# fig = px.histogram(\n# train_df, x=train_df[\"participant_id\"].astype(str), color=\"participant_id\",\n# labels={\"x\":\"Participant ID\", \"count\":\"Total Row Count\"}, title=\"Row Counts by Participant ID\",\n# category_orders={\"participant_id\": train_df[\"participant_id\"].value_counts().index}\n# )\n# fig.update_yaxes(title_text=\"Total Row Count\")\n# fig.update_layout(showlegend=False)\n# fig.show()\n\n# print(\"\\n... GOING FORWARD WE SET THIS COLUMN TO BE A STRING\")\n# train_df[\"participant_id\"] = train_df[\"participant_id\"].astype(str)\n# subsample_train_df[\"participant_id\"] = subsample_train_df[\"participant_id\"].astype(str)","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:58:53.817541Z","iopub.execute_input":"2024-02-25T18:58:53.818037Z","iopub.status.idle":"2024-02-25T18:58:53.956128Z","shell.execute_reply.started":"2024-02-25T18:58:53.817997Z","shell.execute_reply":"2024-02-25T18:58:53.954989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1.2 Sample Video (sequence of frames) - visulize landmarks of the sequence of frames","metadata":{}},{"cell_type":"code","source":"# first sign video frame file as sample to see the frame data \n# it has face, left\nsample_sign_file_name = train_df.iloc[0]['path']\nsample_sign_df = pd.read_parquet(os.path.join(DATA_DIR, sample_sign_file_name))\nprint(\"sign frames for sign \", train_df.iloc[0]['sign'] )\nprint(sample_sign_df)","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:58:53.957559Z","iopub.execute_input":"2024-02-25T18:58:53.958304Z","iopub.status.idle":"2024-02-25T18:58:54.088709Z","shell.execute_reply.started":"2024-02-25T18:58:53.958261Z","shell.execute_reply":"2024-02-25T18:58:54.087648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(sample_sign_df['frame'].nunique()) # number of frames in this video of \"blow\" sign\nsample_sign_df[sample_sign_df['frame'] == 20]['type'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:58:54.090112Z","iopub.execute_input":"2024-02-25T18:58:54.090591Z","iopub.status.idle":"2024-02-25T18:58:54.104072Z","shell.execute_reply.started":"2024-02-25T18:58:54.090550Z","shell.execute_reply":"2024-02-25T18:58:54.103113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Hand Landmarks\n​\n
\n
\n \n
\n
\n​\n[Source](https://developers.google.com/mediapipe/solutions/vision/hand_landmarker#get_started)\n\n\n\n## Pose Landmarks\n\n
\n
\n \n
\n
\n\n[Source](https://google.github.io/mediapipe/solutions/pose.html#pose-landmark-model-blazepose-ghum-3d)","metadata":{}},{"cell_type":"code","source":"# choose features - landmarks to use for classification\n# Google mediapipe landmarks index: 468 face landmarks, 21 landmarks for each hand, and \n# https://developers.google.com/mediapipe/solutions/vision/pose_landmarker#:~:text=The%20MediaPipe%20Pose%20Landmarker%20task,with%20single%20images%20or%20video.\n# The pose consists of a total of 33 landmark points with indices from 0 to 32. \n# pose : 0-10 are assigned to the face, 11-22 to the upper body, and 23-32 to the lower body.\n\n# https://medium.com/@hotakoma/mediapipe-landmark-face-hand-pose-sequence-number-list-view-778364d6c414\n# here in the sign video, each frame has 543 landmards: \n# face: 0:468\n# left_hand: 468:489\n# pose: 489:522\n# right_hand: 522:544\n\n# More resource: https://www.youtube.com/watch?v=hV5S4iQhNkI\n\n# sample_sign_df[sample_sign_df['type'] == \"face\"].landmark_index\nsample_sign_df[(sample_sign_df['frame'] == 20) & (sample_sign_df['type'] == 'pose')].landmark_index\n# sample_sign_df[(sample_sign_df['frame'] == 20) & (sample_sign_df['type'] == 'left_hand')].landmark_index\n# sample_sign_df[(sample_sign_df['frame'] == 20) & (sample_sign_df['type'] == 'right_hand')].landmark_index\n","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:58:54.105198Z","iopub.execute_input":"2024-02-25T18:58:54.105907Z","iopub.status.idle":"2024-02-25T18:58:54.118506Z","shell.execute_reply.started":"2024-02-25T18:58:54.105875Z","shell.execute_reply":"2024-02-25T18:58:54.117548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"thank_you_sign_file = train_df[train_df[\"sign\"] == \"thankyou\"].iloc[1].path\npath_to_sign = thank_you_sign_file\nsign = pd.read_parquet(os.path.join(DATA_DIR, path_to_sign))\nsign.y = sign.y * -1","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:58:54.124020Z","iopub.execute_input":"2024-02-25T18:58:54.124598Z","iopub.status.idle":"2024-02-25T18:58:54.178947Z","shell.execute_reply.started":"2024-02-25T18:58:54.124567Z","shell.execute_reply":"2024-02-25T18:58:54.178133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_hand_points(hand):\n x = [[hand.iloc[0].x, hand.iloc[1].x, hand.iloc[2].x, hand.iloc[3].x, hand.iloc[4].x], # Thumb\n [hand.iloc[5].x, hand.iloc[6].x, hand.iloc[7].x, hand.iloc[8].x], # Index\n [hand.iloc[9].x, hand.iloc[10].x, hand.iloc[11].x, hand.iloc[12].x], \n [hand.iloc[13].x, hand.iloc[14].x, hand.iloc[15].x, hand.iloc[16].x], \n [hand.iloc[17].x, hand.iloc[18].x, hand.iloc[19].x, hand.iloc[20].x], \n [hand.iloc[0].x, hand.iloc[5].x, hand.iloc[9].x, hand.iloc[13].x, hand.iloc[17].x, hand.iloc[0].x]]\n\n y = [[hand.iloc[0].y, hand.iloc[1].y, hand.iloc[2].y, hand.iloc[3].y, hand.iloc[4].y], #Thumb\n [hand.iloc[5].y, hand.iloc[6].y, hand.iloc[7].y, hand.iloc[8].y], # Index\n [hand.iloc[9].y, hand.iloc[10].y, hand.iloc[11].y, hand.iloc[12].y], \n [hand.iloc[13].y, hand.iloc[14].y, hand.iloc[15].y, hand.iloc[16].y], \n [hand.iloc[17].y, hand.iloc[18].y, hand.iloc[19].y, hand.iloc[20].y], \n [hand.iloc[0].y, hand.iloc[5].y, hand.iloc[9].y, hand.iloc[13].y, hand.iloc[17].y, hand.iloc[0].y]] \n return x, y\n\ndef get_pose_points(pose):\n x = [[pose.iloc[8].x, pose.iloc[6].x, pose.iloc[5].x, pose.iloc[4].x, pose.iloc[0].x, pose.iloc[1].x, pose.iloc[2].x, pose.iloc[3].x, pose.iloc[7].x], \n [pose.iloc[10].x, pose.iloc[9].x], \n [pose.iloc[22].x, pose.iloc[16].x, pose.iloc[20].x, pose.iloc[18].x, pose.iloc[16].x, pose.iloc[14].x, pose.iloc[12].x, \n pose.iloc[11].x, pose.iloc[13].x, pose.iloc[15].x, pose.iloc[17].x, pose.iloc[19].x, pose.iloc[15].x, pose.iloc[21].x], \n [pose.iloc[12].x, pose.iloc[24].x, pose.iloc[26].x, pose.iloc[28].x, pose.iloc[30].x, pose.iloc[32].x, pose.iloc[28].x], \n [pose.iloc[11].x, pose.iloc[23].x, pose.iloc[25].x, pose.iloc[27].x, pose.iloc[29].x, pose.iloc[31].x, pose.iloc[27].x], \n [pose.iloc[24].x, pose.iloc[23].x]\n ]\n\n y = [[pose.iloc[8].y, pose.iloc[6].y, pose.iloc[5].y, pose.iloc[4].y, pose.iloc[0].y, pose.iloc[1].y, pose.iloc[2].y, pose.iloc[3].y, pose.iloc[7].y], \n [pose.iloc[10].y, pose.iloc[9].y], \n [pose.iloc[22].y, pose.iloc[16].y, pose.iloc[20].y, pose.iloc[18].y, pose.iloc[16].y, pose.iloc[14].y, pose.iloc[12].y, \n pose.iloc[11].y, pose.iloc[13].y, pose.iloc[15].y, pose.iloc[17].y, pose.iloc[19].y, pose.iloc[15].y, pose.iloc[21].y], \n [pose.iloc[12].y, pose.iloc[24].y, pose.iloc[26].y, pose.iloc[28].y, pose.iloc[30].y, pose.iloc[32].y, pose.iloc[28].y], \n [pose.iloc[11].y, pose.iloc[23].y, pose.iloc[25].y, pose.iloc[27].y, pose.iloc[29].y, pose.iloc[31].y, pose.iloc[27].y], \n [pose.iloc[24].y, pose.iloc[23].y]\n ]\n return x, y\n\n","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:58:54.180341Z","iopub.execute_input":"2024-02-25T18:58:54.180637Z","iopub.status.idle":"2024-02-25T18:58:54.214490Z","shell.execute_reply.started":"2024-02-25T18:58:54.180609Z","shell.execute_reply":"2024-02-25T18:58:54.213460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def animation_frame(f):\n frame = sign[sign.frame==f]\n left = frame[frame.type=='left_hand']\n right = frame[frame.type=='right_hand']\n pose = frame[frame.type=='pose']\n face = frame[frame.type=='face'][['x', 'y']].values\n lx, ly = get_hand_points(left)\n rx, ry = get_hand_points(right)\n px, py = get_pose_points(pose)\n ax.clear()\n ax.plot(face[:,0], face[:,1], '.')\n for i in range(len(lx)):\n ax.plot(lx[i], ly[i])\n for i in range(len(rx)):\n ax.plot(rx[i], ry[i])\n for i in range(len(px)):\n ax.plot(px[i], py[i])\n plt.xlim(xmin, xmax)\n plt.ylim(ymin, ymax)\n \nprint(f\"The sign being shown here is: {train_df[train_df.path==f'{path_to_sign}'].sign.values[0]}\")\n\n## These values set the limits on the graph to stabilize the video\nxmin = sign.x.min() - 0.2\nxmax = sign.x.max() + 0.2\nymin = sign.y.min() - 0.2\nymax = sign.y.max() + 0.2\n\nfig, ax = plt.subplots()\nl, = ax.plot([], [])\nfun_animation = FuncAnimation(fig, func=animation_frame, frames=sign.frame.unique())\n\nHTML(fun_animation.to_html5_video())","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:58:54.215684Z","iopub.execute_input":"2024-02-25T18:58:54.216050Z","iopub.status.idle":"2024-02-25T18:59:00.286168Z","shell.execute_reply.started":"2024-02-25T18:58:54.216010Z","shell.execute_reply":"2024-02-25T18:59:00.285019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 3D visualization \n# https://www.kaggle.com/code/ted0071/gislr-visualization\n","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:59:00.287640Z","iopub.execute_input":"2024-02-25T18:59:00.288002Z","iopub.status.idle":"2024-02-25T18:59:00.292399Z","shell.execute_reply.started":"2024-02-25T18:59:00.287969Z","shell.execute_reply":"2024-02-25T18:59:00.291406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nmpl.rcParams['axes.spines.left'] = False\nmpl.rcParams['axes.spines.right'] = False\nmpl.rcParams['axes.spines.top'] = False\nmpl.rcParams['axes.spines.bottom'] = False","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:59:00.293752Z","iopub.execute_input":"2024-02-25T18:59:00.294151Z","iopub.status.idle":"2024-02-25T18:59:00.302693Z","shell.execute_reply.started":"2024-02-25T18:59:00.294106Z","shell.execute_reply":"2024-02-25T18:59:00.301773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Cfg:\n RANDOM_STATE = 2023\n INPUT_ROOT = Path('/kaggle/input/asl-signs/')\n OUTPUT_ROOT = Path('kaggle/working')\n INDEX_MAP_FILE = INPUT_ROOT / 'sign_to_prediction_index_map.json'\n TRAN_FILE = INPUT_ROOT / 'train.csv'\n INDEX = 'sequence_id'\n ROW_ID = 'row_id'","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:59:00.303997Z","iopub.execute_input":"2024-02-25T18:59:00.304441Z","iopub.status.idle":"2024-02-25T18:59:00.312923Z","shell.execute_reply.started":"2024-02-25T18:59:00.304401Z","shell.execute_reply":"2024-02-25T18:59:00.311966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'cv2 version: {cv2.__version__}')\nprint(f'MediaPipe version: {mp.__version__}')\nprint(f'IPython version: {IPython.__version__}')\n","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:59:00.314203Z","iopub.execute_input":"2024-02-25T18:59:00.314490Z","iopub.status.idle":"2024-02-25T18:59:00.327373Z","shell.execute_reply.started":"2024-02-25T18:59:00.314459Z","shell.execute_reply":"2024-02-25T18:59:00.326282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_index_map(file_path=Cfg.INDEX_MAP_FILE):\n \"\"\"Reads the sign to predict as json file.\"\"\"\n with open(file_path, \"r\") as f:\n result = json.load(f)\n return result \n\ndef read_train(file_path=Cfg.TRAN_FILE):\n \"\"\"Reads the train csv as pandas data frame.\"\"\"\n return pd.read_csv(file_path).set_index(Cfg.INDEX)\n\ndef read_landmark_data_by_path(file_path, input_root=Cfg.INPUT_ROOT):\n \"\"\"Reads landmak data by the given file path.\"\"\"\n data = pd.read_parquet(input_root / file_path)\n return data.set_index(Cfg.ROW_ID)\n\ndef read_landmark_data_by_id(sequence_id, train_data):\n \"\"\"Reads the landmark data by the given sequence id.\"\"\"\n file_path = train_data.loc[sequence_id]['path']\n return read_landmark_data_by_path(file_path)\n\n\n\nmp_drawing = mp.solutions.drawing_utils\nmp_hands = mp.solutions.hands\nmp_face_mesh = mp.solutions.face_mesh\nmp_pose = mp.solutions.pose\n\ndef get_random_sequence_id(train_data):\n idx = np.random.randint(0, len(train_data))\n return train_data.index[idx]\n\n# def create_blank_image(height, width):\n# return np.zeros((height, width, 3), np.uint8)\n\ndef create_blank_image(height, width):\n return np.ones((height, width, 3), np.uint8)\n\ndef draw_landmarks(\n data, \n image, \n frame_id, \n landmark_type, \n connection_type, \n landmark_color=(255, 0, 0), \n connection_color=(0, 20, 255), \n thickness=1, \n circle_radius=1\n):\n \"\"\"Draws landmarks\"\"\"\n df = data.groupby(['frame', 'type']).get_group((frame_id, landmark_type))\n landmarks = [landmark_pb2.NormalizedLandmark(x=lm.x, y=lm.y, z=lm.z) for idx, lm in df.iterrows()]\n landmark_list = landmark_pb2.NormalizedLandmarkList(landmark = landmarks)\n\n mp_drawing.draw_landmarks(\n image=image,\n landmark_list=landmark_list, \n connections=connection_type,\n landmark_drawing_spec=mp_drawing.DrawingSpec(\n color=landmark_color, \n thickness=thickness, \n circle_radius=circle_radius),\n connection_drawing_spec=mp_drawing.DrawingSpec(\n color=connection_color, \n thickness=thickness, \n circle_radius=circle_radius))\n return image\n\ndef draw_left_hand(data, image, frame_id):\n return draw_landmarks(\n data, \n image, \n frame_id, \n landmark_type='left_hand', \n connection_type=mp_hands.HAND_CONNECTIONS,\n landmark_color=(255, 0, 0),\n# connection_color=(0, 20, 255),\n connection_color=(0, 255, 0),\n thickness=3, \n circle_radius=3)\n\ndef draw_right_hand(data, image, frame_id):\n return draw_landmarks(\n data, \n image, \n frame_id, \n landmark_type='right_hand', \n connection_type=mp_hands.HAND_CONNECTIONS,\n landmark_color=(255, 0, 0),\n# connection_color=(0, 20, 255),\n connection_color=(0, 255, 0),\n thickness=3, \n circle_radius=3)\n\ndef draw_face(data, image, frame_id):\n return draw_landmarks(\n data, \n image, \n frame_id, \n landmark_type='face', \n connection_type=mp_face_mesh.FACEMESH_TESSELATION,\n landmark_color=(255, 255, 255),\n connection_color=(0, 255, 0)) \n \ndef draw_pose(data, image, frame_id):\n return draw_landmarks(\n data, \n image, \n frame_id, \n landmark_type='pose', \n connection_type=mp_pose.POSE_CONNECTIONS,\n# landmark_color=(255, 255, 255),\n# connection_color=(255, 0, 0),\n landmark_color=(255, 0, 0),\n connection_color=(0, 255, 0),\n thickness=2, \n circle_radius=2)\n\ndef create_frame(data, frame_id, height=1000, width=1000):\n image = create_blank_image(height, width) \n\n draw_pose(data, image, frame_id) \n draw_left_hand(data, image, frame_id) \n draw_right_hand(data, image, frame_id) \n draw_face(data, image, frame_id)\n \n return image","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:59:00.328983Z","iopub.execute_input":"2024-02-25T18:59:00.329619Z","iopub.status.idle":"2024-02-25T18:59:00.354603Z","shell.execute_reply.started":"2024-02-25T18:59:00.329581Z","shell.execute_reply":"2024-02-25T18:59:00.353681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = read_train()\n\n\nheight = 800\nwidth = 600\n\nsequence_id = 1000106739\ndata = read_landmark_data_by_id(sequence_id, train_data)\n\nframe_id = data['frame'][0]\n\n_, ax = plt.subplots(1, 1, figsize=(4, 4))\nimage = draw_right_hand(data, image=create_blank_image(height, width), frame_id=frame_id)\n \nax.imshow(image)\nax.axis('off')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:59:00.355587Z","iopub.execute_input":"2024-02-25T18:59:00.355902Z","iopub.status.idle":"2024-02-25T18:59:00.672552Z","shell.execute_reply.started":"2024-02-25T18:59:00.355874Z","shell.execute_reply":"2024-02-25T18:59:00.670960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, ax = plt.subplots(1, 1, figsize=(4, 4))\nimage = draw_face(data, image=create_blank_image(height, width), frame_id=frame_id)\n \nax.imshow(image)\nax.axis('off')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:59:00.674780Z","iopub.execute_input":"2024-02-25T18:59:00.675822Z","iopub.status.idle":"2024-02-25T18:59:00.907526Z","shell.execute_reply.started":"2024-02-25T18:59:00.675762Z","shell.execute_reply":"2024-02-25T18:59:00.905905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, ax = plt.subplots(1, 1, figsize=(4, 4))\nimage = draw_pose(data, image=create_blank_image(height, width), frame_id=frame_id)\n \nax.imshow(image)\nax.axis('off')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:59:00.909968Z","iopub.execute_input":"2024-02-25T18:59:00.911204Z","iopub.status.idle":"2024-02-25T18:59:01.049318Z","shell.execute_reply.started":"2024-02-25T18:59:00.911138Z","shell.execute_reply":"2024-02-25T18:59:01.048542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = read_train()\n\ndef read_landmark_data_by_sign(sign_string, train_data):\n \"\"\"Reads the landmark data by the given sequence id.\"\"\"\n file_path = train_data[train_data['sign'] == sign_string].iloc[0]['path']\n return read_landmark_data_by_path(file_path)\n\nheight = 800\nwidth = 600\n\n\ndata = read_landmark_data_by_sign(\"thankyou\", train_data)\nframe_id = data['frame'][0]\n\n_, ax = plt.subplots(1, 1, figsize=(4, 4))\n\nimage = create_frame(data, frame_id=frame_id)\n \nax.imshow(image)\nax.axis('off')\nplt.savefig('/kaggle/working/thankyou_plot.png', dpi=300)\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:59:01.050441Z","iopub.execute_input":"2024-02-25T18:59:01.050906Z","iopub.status.idle":"2024-02-25T18:59:01.830065Z","shell.execute_reply.started":"2024-02-25T18:59:01.050876Z","shell.execute_reply":"2024-02-25T18:59:01.828985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_frames(sequence_id, train_data, height=800, width=800):\n data = read_landmark_data_by_id(sequence_id, train_data)\n frame_ids = data['frame'].unique()\n images = [create_frame(data, frame_id=fid, height=height, width=width) for fid in frame_ids]\n return np.array(images)\n\n\ndef create_animation(images, fig, ax):\n ax.axis('off')\n \n ims = []\n for img in images:\n im = ax.imshow(img, animated=True)\n ims.append([im])\n \n func_animation = mpl.animation.ArtistAnimation(\n fig, \n ims, \n interval=100, \n blit=True,\n repeat_delay=1000)\n\n return func_animation\n\ndef get_sign_by_id(sequence_id, train_data):\n return train_data.loc[sequence_id]['sign']\n\ndef play_animation(sequence_id, train_data, height, width, figsize=(4, 4)):\n frames = create_frames(sequence_id, train_data, height=height, width=width)\n sign = get_sign_by_id(sequence_id, train_data)\n \n fig, ax = plt.subplots(1, 1, figsize=figsize)\n anim = create_animation(frames, fig, ax)\n ax.set_title(f'Sign: {sign}')\n \n video = anim.to_html5_video()\n html = IPython.display.HTML(video)\n IPython.display.display(html)\n plt.close()\n\n","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:59:01.831822Z","iopub.execute_input":"2024-02-25T18:59:01.832141Z","iopub.status.idle":"2024-02-25T18:59:01.843793Z","shell.execute_reply.started":"2024-02-25T18:59:01.832109Z","shell.execute_reply":"2024-02-25T18:59:01.842796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sequence_id = get_random_sequence_id(train_data)\nplay_animation(sequence_id, train_data, height=height, width=width)","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:59:01.845368Z","iopub.execute_input":"2024-02-25T18:59:01.845843Z","iopub.status.idle":"2024-02-25T18:59:04.458637Z","shell.execute_reply.started":"2024-02-25T18:59:01.845804Z","shell.execute_reply":"2024-02-25T18:59:04.457513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = train_data[train_data['sign'] == 'happy']\nsequence_id = get_random_sequence_id(data)\n\nplay_animation(sequence_id, train_data, height=height, width=width)","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:59:04.460509Z","iopub.execute_input":"2024-02-25T18:59:04.461028Z","iopub.status.idle":"2024-02-25T18:59:14.668705Z","shell.execute_reply.started":"2024-02-25T18:59:04.460981Z","shell.execute_reply":"2024-02-25T18:59:14.667524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = train_data[train_data['sign'] == 'thankyou']\nsequence_id = get_random_sequence_id(data)\n\nplay_animation(sequence_id, train_data, height=height, width=width)\n\n# _, ax = plt.subplots(1, 1, figsize=(4, 4))\n# image = create_frame(data, frame_id=frame_id)\n \n# ax.imshow(image)\n# ax.axis('off')\n\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:59:14.670554Z","iopub.execute_input":"2024-02-25T18:59:14.671398Z","iopub.status.idle":"2024-02-25T18:59:15.797052Z","shell.execute_reply.started":"2024-02-25T18:59:14.671353Z","shell.execute_reply":"2024-02-25T18:59:15.795808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_gif(sequence_id, train_data, height, width, figsize=(4, 4)):\n frames = create_frames(sequence_id, train_data, height=height, width=width)\n sign = get_sign_by_id(sequence_id, train_data)\n \n fig, ax = plt.subplots(1, 1, figsize=figsize)\n anim = create_animation(frames, fig, ax)\n # Save the animation as a GIF\n anim.save('sign.gif', writer='imagemagick')\n plt.close()\n ","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:59:15.798748Z","iopub.execute_input":"2024-02-25T18:59:15.799546Z","iopub.status.idle":"2024-02-25T18:59:15.806622Z","shell.execute_reply.started":"2024-02-25T18:59:15.799508Z","shell.execute_reply":"2024-02-25T18:59:15.805403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = train_data[train_data['sign'] == 'thankyou']\nsequence_id = get_random_sequence_id(data)\n\ncreate_gif(sequence_id, train_data, height=height, width=width)\n# Display the GIF\nIPython.display.Image(filename='/kaggle/working/sign.gif')","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:59:15.808031Z","iopub.execute_input":"2024-02-25T18:59:15.808382Z","iopub.status.idle":"2024-02-25T18:59:21.593536Z","shell.execute_reply.started":"2024-02-25T18:59:15.808344Z","shell.execute_reply":"2024-02-25T18:59:21.592276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_each_frame(sequence_id, train_data, height, width, figsize=(4, 4)):\n frames = create_frames(sequence_id, train_data, height=height, width=width)\n fig, ax = plt.subplots(figsize=figsize)\n for i, frame in enumerate(frames):\n ax.clear() # Clear the previous frame\n ax.imshow(frame)\n ax.axis('off') # Hide the axis\n plt.savefig(f'/kaggle/working/frame_{i:03d}.png')\n plt.close(fig) # Close the figure after saving all frames","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:59:21.604073Z","iopub.execute_input":"2024-02-25T18:59:21.604508Z","iopub.status.idle":"2024-02-25T18:59:21.611769Z","shell.execute_reply.started":"2024-02-25T18:59:21.604469Z","shell.execute_reply":"2024-02-25T18:59:21.610664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !rm /kaggle/working/frame_*\n# !rm /kaggle/working/frames.zip\n# # !rm /kaggle/working/kaggle_working_files.zip","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:59:21.612901Z","iopub.execute_input":"2024-02-25T18:59:21.613194Z","iopub.status.idle":"2024-02-25T18:59:23.564112Z","shell.execute_reply.started":"2024-02-25T18:59:21.613166Z","shell.execute_reply":"2024-02-25T18:59:23.562800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data = train_data[train_data['sign'] == 'thankyou']\n# sequence_id = get_random_sequence_id(data)\n# save_each_frame(sequence_id, train_data, height=height, width=width)","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:59:23.566149Z","iopub.execute_input":"2024-02-25T18:59:23.567154Z","iopub.status.idle":"2024-02-25T18:59:27.448393Z","shell.execute_reply.started":"2024-02-25T18:59:23.567101Z","shell.execute_reply":"2024-02-25T18:59:27.447514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport zipfile\nimport fnmatch\n\n# Define the directory containing the frames\ndir_to_zip = '/kaggle/working'\n\n# Define the output ZIP file name\nzip_file_path = '/kaggle/working/frames.zip'\n\n# Create a ZIP file\nwith zipfile.ZipFile(zip_file_path, 'w', zipfile.ZIP_DEFLATED) as zipf:\n # Walk through the directory\n for root, dirs, files in os.walk(dir_to_zip):\n for file in fnmatch.filter(files, 'frame_*.png'):\n # Create a relative path for files to keep the directory structure\n relative_path = os.path.relpath(os.path.join(root, file), os.path.join(dir_to_zip, '..'))\n zipf.write(os.path.join(root, file), relative_path)\n\nprint(f\"Only 'frame_*.png' files in '{dir_to_zip}' are zipped into '{zip_file_path}'\")\n","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:59:27.449777Z","iopub.execute_input":"2024-02-25T18:59:27.450486Z","iopub.status.idle":"2024-02-25T18:59:27.484404Z","shell.execute_reply.started":"2024-02-25T18:59:27.450442Z","shell.execute_reply":"2024-02-25T18:59:27.483478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Part 2: Feature Engineering and Model Training\nThis notebook's model is based on this one: https://www.kaggle.com/code/markwijkhuizen/gislr-tf-data-processing-transformer-training with edits:\n * Learn data preprocessing\n * Reproduce the results\n * twist the models and/or features\n\n**Data Processing (from the original author's)**\n\nOnly lips, hands and arm pose coordinates are used.\n\nA custom Tensorflow layer handles the data processing. In short, it filters all frames without coordinates for the hands and downsamples the input to 32 frames if it is too long.\n\n**Model**\n\nA transformer based model is used. The embedding layer makes an ambedding per landmark(lips/left hand/right hand/arm pose) and merges these embedding with fully connected layers. The transformer consists of just 2 blocks with a simple mean pooling and fully connected layers for classification.","metadata":{}},{"cell_type":"markdown","source":"# Plot Config","metadata":{}},{"cell_type":"code","source":"# MatplotLib Global Settings\nmpl.rcParams.update(mpl.rcParamsDefault)\nmpl.rcParams['xtick.labelsize'] = 16\nmpl.rcParams['ytick.labelsize'] = 16\nmpl.rcParams['axes.labelsize'] = 18\nmpl.rcParams['axes.titlesize'] = 24","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:59:27.485548Z","iopub.execute_input":"2024-02-25T18:59:27.485924Z","iopub.status.idle":"2024-02-25T18:59:27.493002Z","shell.execute_reply.started":"2024-02-25T18:59:27.485883Z","shell.execute_reply":"2024-02-25T18:59:27.492020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"# If True, processing data from scratch\n# If False, loads preprocessed data\nPREPROCESS_DATA = False\nTRAIN_MODEL = True\n# True: use 10% of participants as validation set\n# False: use all data for training -> gives better LB result\nUSE_VAL = True\n\nN_ROWS = 543\nN_DIMS = 3\nDIM_NAMES = ['x', 'y', 'z']\nSEED = 42\nNUM_CLASSES = 250\nIS_INTERACTIVE = os.environ['KAGGLE_KERNEL_RUN_TYPE'] == 'Interactive'\nVERBOSE = 1 if IS_INTERACTIVE else 2\n\nINPUT_SIZE = 64\n\nBATCH_ALL_SIGNS_N = 4\nBATCH_SIZE = 256\nN_EPOCHS = 100\nLR_MAX = 1e-3\nN_WARMUP_EPOCHS = 0\nWD_RATIO = 0.05\nMASK_VAL = 4237","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:59:27.494349Z","iopub.execute_input":"2024-02-25T18:59:27.494694Z","iopub.status.idle":"2024-02-25T18:59:27.504391Z","shell.execute_reply.started":"2024-02-25T18:59:27.494657Z","shell.execute_reply":"2024-02-25T18:59:27.503519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utils","metadata":{}},{"cell_type":"code","source":"# Prints Shape and Dtype For List Of Variables\ndef print_shape_dtype(l, names):\n for e, n in zip(l, names):\n print(f'{n} shape: {e.shape}, dtype: {e.dtype}')","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:59:27.505446Z","iopub.execute_input":"2024-02-25T18:59:27.505724Z","iopub.status.idle":"2024-02-25T18:59:27.515075Z","shell.execute_reply.started":"2024-02-25T18:59:27.505696Z","shell.execute_reply":"2024-02-25T18:59:27.514166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"# Read Training Data\nif IS_INTERACTIVE or not PREPROCESS_DATA:\n train = pd.read_csv('/kaggle/input/asl-signs/train.csv').sample(5000, random_state=SEED)\nelse:\n train = pd.read_csv('/kaggle/input/asl-signs/train.csv')\n\nN_SAMPLES = len(train)\nprint(f'N_SAMPLES: {N_SAMPLES}')","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:59:27.516323Z","iopub.execute_input":"2024-02-25T18:59:27.516672Z","iopub.status.idle":"2024-02-25T18:59:27.650928Z","shell.execute_reply.started":"2024-02-25T18:59:27.516644Z","shell.execute_reply":"2024-02-25T18:59:27.649765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Add File Path","metadata":{}},{"cell_type":"code","source":"# Get complete file path to file\ndef get_file_path(path):\n return f'/kaggle/input/asl-signs/{path}'\n\ntrain['file_path'] = train['path'].apply(get_file_path)","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:59:27.652068Z","iopub.execute_input":"2024-02-25T18:59:27.652359Z","iopub.status.idle":"2024-02-25T18:59:27.660643Z","shell.execute_reply.started":"2024-02-25T18:59:27.652327Z","shell.execute_reply":"2024-02-25T18:59:27.659634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ordinally Encode Sign","metadata":{}},{"cell_type":"code","source":"# Add ordinally Encoded Sign (assign number to each sign name)\ntrain['sign_ord'] = train['sign'].astype('category').cat.codes\n\n# Dictionaries to translate sign <-> ordinal encoded sign\nSIGN2ORD = train[['sign', 'sign_ord']].set_index('sign').squeeze().to_dict()\nORD2SIGN = train[['sign_ord', 'sign']].set_index('sign_ord').squeeze().to_dict()","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:59:27.661931Z","iopub.execute_input":"2024-02-25T18:59:27.662302Z","iopub.status.idle":"2024-02-25T18:59:27.691032Z","shell.execute_reply.started":"2024-02-25T18:59:27.662264Z","shell.execute_reply":"2024-02-25T18:59:27.690209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train.head())\ndisplay(train.info())","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:59:27.692158Z","iopub.execute_input":"2024-02-25T18:59:27.692514Z","iopub.status.idle":"2024-02-25T18:59:27.987900Z","shell.execute_reply.started":"2024-02-25T18:59:27.692476Z","shell.execute_reply":"2024-02-25T18:59:27.986773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Video Statistics","metadata":{}},{"cell_type":"markdown","source":"# Landmark Indices","metadata":{}},{"cell_type":"code","source":"USE_TYPES = ['left_hand', 'pose', 'right_hand']\nSTART_IDX = 468\n\n# 40 landmarks for lips\nLIPS_IDXS0 = np.array([\n 61, 185, 40, 39, 37, 0, 267, 269, 270, 409,\n 291, 146, 91, 181, 84, 17, 314, 405, 321, 375,\n 78, 191, 80, 81, 82, 13, 312, 311, 310, 415,\n 95, 88, 178, 87, 14, 317, 402, 318, 324, 308,\n ]) \n# Landmark indices in original data\nLEFT_HAND_IDXS0 = np.arange(468,489) # 21\nRIGHT_HAND_IDXS0 = np.arange(522,543) # 21\nLEFT_POSE_IDXS0 = np.array([502, 504, 506, 508, 510])\nRIGHT_POSE_IDXS0 = np.array([503, 505, 507, 509, 511])\nLANDMARK_IDXS_LEFT_DOMINANT0 = np.concatenate((LIPS_IDXS0, LEFT_HAND_IDXS0, LEFT_POSE_IDXS0)) # 40+21+5 = 66 landmarks\nLANDMARK_IDXS_RIGHT_DOMINANT0 = np.concatenate((LIPS_IDXS0, RIGHT_HAND_IDXS0, RIGHT_POSE_IDXS0)) # 40+21+5 = 66 landmarks\nHAND_IDXS0 = np.concatenate((LEFT_HAND_IDXS0, RIGHT_HAND_IDXS0), axis=0) # both hands 42 landmarks\nN_COLS = LANDMARK_IDXS_LEFT_DOMINANT0.size\n\n# Landmark indices in processed data\n# index of lips landmarks in LEFT_DOMINANT0 0-39\nLIPS_IDXS = np.argwhere(np.isin(LANDMARK_IDXS_LEFT_DOMINANT0, LIPS_IDXS0)).squeeze() \n\n# index of left hand landmarks in LEFT_DOMINANT0 40-60\nLEFT_HAND_IDXS = np.argwhere(np.isin(LANDMARK_IDXS_LEFT_DOMINANT0, LEFT_HAND_IDXS0)).squeeze()\n\n# index of right hand landmarks in LEFT_DOMINANT0 empty []\nRIGHT_HAND_IDXS = np.argwhere(np.isin(LANDMARK_IDXS_LEFT_DOMINANT0, RIGHT_HAND_IDXS0)).squeeze()\n\n# index of both hands landmarks in LEFT_DOMINANT0 40-60 (actually only left hand)\nHAND_IDXS = np.argwhere(np.isin(LANDMARK_IDXS_LEFT_DOMINANT0, HAND_IDXS0)).squeeze()\n\n# index of left pose landmarks in LEFT_DOMINANT0 61-65\nPOSE_IDXS = np.argwhere(np.isin(LANDMARK_IDXS_LEFT_DOMINANT0, LEFT_POSE_IDXS0)).squeeze()\n\nprint(f'# HAND_IDXS: {len(HAND_IDXS)}, N_COLS: {N_COLS}')","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:59:27.989210Z","iopub.execute_input":"2024-02-25T18:59:27.989591Z","iopub.status.idle":"2024-02-25T18:59:28.004413Z","shell.execute_reply.started":"2024-02-25T18:59:27.989558Z","shell.execute_reply":"2024-02-25T18:59:28.003295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# only left dominant landmarks used lips + left hand + left pose\n\nLIPS_START = 0\nLEFT_HAND_START = LIPS_IDXS.size\nRIGHT_HAND_START = LEFT_HAND_START + LEFT_HAND_IDXS.size\nPOSE_START = RIGHT_HAND_START + RIGHT_HAND_IDXS.size\n\nprint(f'LIPS_START: {LIPS_START}, LEFT_HAND_START: {LEFT_HAND_START}, RIGHT_HAND_START: {RIGHT_HAND_START}, POSE_START: {POSE_START}')","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:59:28.005775Z","iopub.execute_input":"2024-02-25T18:59:28.006149Z","iopub.status.idle":"2024-02-25T18:59:28.015608Z","shell.execute_reply.started":"2024-02-25T18:59:28.006120Z","shell.execute_reply":"2024-02-25T18:59:28.014706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Process Data Tensorflow","metadata":{}},{"cell_type":"code","source":"# Source: https://www.kaggle.com/competitions/asl-signs/overview/evaluation\nROWS_PER_FRAME = 543 # number of landmarks per frame\n\ndef load_relevant_data_subset(pq_path):\n data_columns = ['x', 'y', 'z']\n data = pd.read_parquet(pq_path, columns=data_columns)\n n_frames = int(len(data) / ROWS_PER_FRAME)\n data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n return data.astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:59:28.017488Z","iopub.execute_input":"2024-02-25T18:59:28.017965Z","iopub.status.idle":"2024-02-25T18:59:28.031480Z","shell.execute_reply.started":"2024-02-25T18:59:28.017926Z","shell.execute_reply":"2024-02-25T18:59:28.030095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n Tensorflow layer to process data in TFLite\n Data needs to be processed in the model itself, so we can not use Python\n\"\"\" \nclass PreprocessLayer(tf.keras.layers.Layer):\n def __init__(self):\n super(PreprocessLayer, self).__init__()\n normalisation_correction = tf.constant([\n # Add 0.50 to left hand (original right hand) and substract 0.50 of right hand (original left hand)\n [0] * len(LIPS_IDXS) + [0.50] * len(LEFT_HAND_IDXS) + [0.50] * len(POSE_IDXS),\n # Y coordinates stay intact\n [0] * len(LANDMARK_IDXS_LEFT_DOMINANT0),\n # Z coordinates stay intact\n [0] * len(LANDMARK_IDXS_LEFT_DOMINANT0),\n ],\n dtype=tf.float32,\n )\n self.normalisation_correction = tf.transpose(normalisation_correction, [1,0])\n \n def pad_edge(self, t, repeats, side):\n if side == 'LEFT':\n return tf.concat((tf.repeat(t[:1], repeats=repeats, axis=0), t), axis=0)\n elif side == 'RIGHT':\n return tf.concat((t, tf.repeat(t[-1:], repeats=repeats, axis=0)), axis=0)\n \n @tf.function(\n input_signature=(tf.TensorSpec(shape=[None,N_ROWS,N_DIMS], dtype=tf.float32),),\n )\n def call(self, data0):\n # Number of Frames in Video\n N_FRAMES0 = tf.shape(data0)[0]\n \n # Find dominant hand by comparing summed absolute coordinates\n left_hand_sum = tf.math.reduce_sum(tf.where(tf.math.is_nan(tf.gather(data0, LEFT_HAND_IDXS0, axis=1)), 0, 1))\n right_hand_sum = tf.math.reduce_sum(tf.where(tf.math.is_nan(tf.gather(data0, RIGHT_HAND_IDXS0, axis=1)), 0, 1))\n left_dominant = left_hand_sum >= right_hand_sum\n \n # Count non NaN Hand values in each frame for the dominant hand\n if left_dominant:\n frames_hands_non_nan_sum = tf.math.reduce_sum(\n tf.where(tf.math.is_nan(tf.gather(data0, LEFT_HAND_IDXS0, axis=1)), 0, 1),\n axis=[1, 2],\n )\n else:\n frames_hands_non_nan_sum = tf.math.reduce_sum(\n tf.where(tf.math.is_nan(tf.gather(data0, RIGHT_HAND_IDXS0, axis=1)), 0, 1),\n axis=[1, 2],\n )\n \n # Find frames indices with coordinates of dominant hand\n non_empty_frames_idxs = tf.where(frames_hands_non_nan_sum > 0)\n non_empty_frames_idxs = tf.squeeze(non_empty_frames_idxs, axis=1)\n # Filter frames\n data = tf.gather(data0, non_empty_frames_idxs, axis=0)\n \n # Cast Indices in float32 to be compatible with Tensorflow Lite\n non_empty_frames_idxs = tf.cast(non_empty_frames_idxs, tf.float32)\n # Normalize to start with 0\n non_empty_frames_idxs -= tf.reduce_min(non_empty_frames_idxs)\n \n # Number of Frames in Filtered Video\n N_FRAMES = tf.shape(data)[0]\n \n # Gather Relevant Landmark Columns\n if left_dominant:\n data = tf.gather(data, LANDMARK_IDXS_LEFT_DOMINANT0, axis=1)\n else:\n data = tf.gather(data, LANDMARK_IDXS_RIGHT_DOMINANT0, axis=1)\n data = (\n self.normalisation_correction + (\n (data - self.normalisation_correction) * tf.where(self.normalisation_correction != 0, -1.0, 1.0))\n )\n \n # Video fits in INPUT_SIZE\n if N_FRAMES < INPUT_SIZE:\n # Pad With -1 to indicate padding\n non_empty_frames_idxs = tf.pad(non_empty_frames_idxs, [[0, INPUT_SIZE-N_FRAMES]], constant_values=-1)\n # Pad Data With Zeros\n data = tf.pad(data, [[0, INPUT_SIZE-N_FRAMES], [0,0], [0,0]], constant_values=0)\n # Fill NaN Values With 0\n data = tf.where(tf.math.is_nan(data), 0.0, data)\n return data, non_empty_frames_idxs\n # Video needs to be downsampled to INPUT_SIZE\n else:\n # Repeat\n if N_FRAMES < INPUT_SIZE**2:\n repeats = tf.math.floordiv(INPUT_SIZE * INPUT_SIZE, N_FRAMES0)\n data = tf.repeat(data, repeats=repeats, axis=0)\n non_empty_frames_idxs = tf.repeat(non_empty_frames_idxs, repeats=repeats, axis=0)\n\n # Pad To Multiple Of Input Size\n pool_size = tf.math.floordiv(len(data), INPUT_SIZE)\n if tf.math.mod(len(data), INPUT_SIZE) > 0:\n pool_size += 1\n\n if pool_size == 1:\n pad_size = (pool_size * INPUT_SIZE) - len(data)\n else:\n pad_size = (pool_size * INPUT_SIZE) % len(data)\n\n # Pad Start/End with Start/End value\n pad_left = tf.math.floordiv(pad_size, 2) + tf.math.floordiv(INPUT_SIZE, 2)\n pad_right = tf.math.floordiv(pad_size, 2) + tf.math.floordiv(INPUT_SIZE, 2)\n if tf.math.mod(pad_size, 2) > 0:\n pad_right += 1\n\n # Pad By Concatenating Left/Right Edge Values\n data = self.pad_edge(data, pad_left, 'LEFT')\n data = self.pad_edge(data, pad_right, 'RIGHT')\n\n # Pad Non Empty Frame Indices\n non_empty_frames_idxs = self.pad_edge(non_empty_frames_idxs, pad_left, 'LEFT')\n non_empty_frames_idxs = self.pad_edge(non_empty_frames_idxs, pad_right, 'RIGHT')\n\n # Reshape to Mean Pool\n data = tf.reshape(data, [INPUT_SIZE, -1, N_COLS, N_DIMS])\n non_empty_frames_idxs = tf.reshape(non_empty_frames_idxs, [INPUT_SIZE, -1])\n\n # Mean Pool\n data = tf.experimental.numpy.nanmean(data, axis=1)\n non_empty_frames_idxs = tf.experimental.numpy.nanmean(non_empty_frames_idxs, axis=1)\n\n # Fill NaN Values With 0\n data = tf.where(tf.math.is_nan(data), 0.0, data)\n \n return data, non_empty_frames_idxs\n \npreprocess_layer = PreprocessLayer()","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:59:28.033037Z","iopub.execute_input":"2024-02-25T18:59:28.033747Z","iopub.status.idle":"2024-02-25T18:59:30.323161Z","shell.execute_reply.started":"2024-02-25T18:59:28.033694Z","shell.execute_reply":"2024-02-25T18:59:30.322152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Interpolate NaN Values","metadata":{}},{"cell_type":"code","source":"\"\"\"\n face: 0:468\n left_hand: 468:489\n pose: 489:522\n right_hand: 522:544\n \n\"\"\"\ndef get_data(file_path):\n # Load Raw Data\n data = load_relevant_data_subset(file_path)\n # Process Data Using Tensorflow\n data = preprocess_layer(data)\n \n return data","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:59:30.324259Z","iopub.execute_input":"2024-02-25T18:59:30.325229Z","iopub.status.idle":"2024-02-25T18:59:30.331867Z","shell.execute_reply.started":"2024-02-25T18:59:30.325188Z","shell.execute_reply":"2024-02-25T18:59:30.330651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Dataset","metadata":{}},{"cell_type":"code","source":"# Get the full dataset\ndef preprocess_data():\n # Create arrays to save data\n X = np.zeros([N_SAMPLES, INPUT_SIZE, N_COLS, N_DIMS], dtype=np.float32)\n y = np.zeros([N_SAMPLES], dtype=np.int32)\n NON_EMPTY_FRAME_IDXS = np.full([N_SAMPLES, INPUT_SIZE], -1, dtype=np.float32)\n\n # Fill X/y\n for row_idx, (file_path, sign_ord) in enumerate(tqdm(train[['file_path', 'sign_ord']].values)):\n # Log message every 5000 samples\n if row_idx % 5000 == 0:\n print(f'Generated {row_idx}/{N_SAMPLES}')\n\n data, non_empty_frame_idxs = get_data(file_path)\n X[row_idx] = data\n y[row_idx] = sign_ord\n NON_EMPTY_FRAME_IDXS[row_idx] = non_empty_frame_idxs\n # Sanity check, data should not contain NaN values\n if np.isnan(data).sum() > 0:\n print(row_idx)\n return data\n\n # Save X/y\n np.save('X.npy', X)\n np.save('y.npy', y)\n np.save('NON_EMPTY_FRAME_IDXS.npy', NON_EMPTY_FRAME_IDXS)\n \n # Save Validation\n splitter = GroupShuffleSplit(test_size=0.10, n_splits=2, random_state=SEED)\n PARTICIPANT_IDS = train['participant_id'].values\n train_idxs, val_idxs = next(splitter.split(X, y, groups=PARTICIPANT_IDS))\n\n # Save Train\n X_train = X[train_idxs]\n NON_EMPTY_FRAME_IDXS_TRAIN = NON_EMPTY_FRAME_IDXS[train_idxs]\n y_train = y[train_idxs]\n np.save('X_train.npy', X_train)\n np.save('y_train.npy', y_train)\n np.save('NON_EMPTY_FRAME_IDXS_TRAIN.npy', NON_EMPTY_FRAME_IDXS_TRAIN)\n # Save Validation\n X_val = X[val_idxs]\n NON_EMPTY_FRAME_IDXS_VAL = NON_EMPTY_FRAME_IDXS[val_idxs]\n y_val = y[val_idxs]\n np.save('X_val.npy', X_val)\n np.save('y_val.npy', y_val)\n np.save('NON_EMPTY_FRAME_IDXS_VAL.npy', NON_EMPTY_FRAME_IDXS_VAL)\n # Split Statistics\n print(f'Patient ID Intersection Train/Val: {set(PARTICIPANT_IDS[train_idxs]).intersection(PARTICIPANT_IDS[val_idxs])}')\n print(f'X_train shape: {X_train.shape}, X_val shape: {X_val.shape}')\n print(f'y_train shape: {y_train.shape}, y_val shape: {y_val.shape}')","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:59:30.333602Z","iopub.execute_input":"2024-02-25T18:59:30.334190Z","iopub.status.idle":"2024-02-25T18:59:30.349751Z","shell.execute_reply.started":"2024-02-25T18:59:30.334158Z","shell.execute_reply":"2024-02-25T18:59:30.348762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Preprocess All Data From Scratch\nif PREPROCESS_DATA:\n preprocess_data()\n ROOT_DIR = '.'\nelse:\n ROOT_DIR = '/kaggle/input/gislr-dataset-public'\n \n# Load Data\nif USE_VAL:\n # Load Train\n X_train = np.load(f'{ROOT_DIR}/X_train.npy')\n y_train = np.load(f'{ROOT_DIR}/y_train.npy')\n NON_EMPTY_FRAME_IDXS_TRAIN = np.load(f'{ROOT_DIR}/NON_EMPTY_FRAME_IDXS_TRAIN.npy')\n # Load Val\n X_val = np.load(f'{ROOT_DIR}/X_val.npy')\n y_val = np.load(f'{ROOT_DIR}/y_val.npy')\n NON_EMPTY_FRAME_IDXS_VAL = np.load(f'{ROOT_DIR}/NON_EMPTY_FRAME_IDXS_VAL.npy')\n # Define validation Data\n validation_data = ({ 'frames': X_val, 'non_empty_frame_idxs': NON_EMPTY_FRAME_IDXS_VAL }, y_val)\nelse:\n X_train = np.load(f'{ROOT_DIR}/X.npy')\n y_train = np.load(f'{ROOT_DIR}/y.npy')\n NON_EMPTY_FRAME_IDXS_TRAIN = np.load(f'{ROOT_DIR}/NON_EMPTY_FRAME_IDXS.npy')\n validation_data = None\n\n# Train \nprint_shape_dtype([X_train, y_train, NON_EMPTY_FRAME_IDXS_TRAIN], ['X_train', 'y_train', 'NON_EMPTY_FRAME_IDXS_TRAIN'])\n# Val\nif USE_VAL:\n print_shape_dtype([X_val, y_val, NON_EMPTY_FRAME_IDXS_VAL], ['X_val', 'y_val', 'NON_EMPTY_FRAME_IDXS_VAL'])\n# Sanity Check\nprint(f'# NaN Values X_train: {np.isnan(X_train).sum()}')","metadata":{"execution":{"iopub.status.busy":"2024-02-25T18:59:30.353214Z","iopub.execute_input":"2024-02-25T18:59:30.353496Z","iopub.status.idle":"2024-02-25T19:00:20.556264Z","shell.execute_reply.started":"2024-02-25T18:59:30.353469Z","shell.execute_reply":"2024-02-25T19:00:20.555019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Class Count\ndisplay(pd.Series(y_train).value_counts().to_frame('Class Count').iloc[[0,1,2,3,4, -5,-4,-3,-2,-1]])","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:00:20.557564Z","iopub.execute_input":"2024-02-25T19:00:20.557911Z","iopub.status.idle":"2024-02-25T19:00:20.570128Z","shell.execute_reply.started":"2024-02-25T19:00:20.557879Z","shell.execute_reply":"2024-02-25T19:00:20.569139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Number Of Frames","metadata":{}},{"cell_type":"code","source":"# Vast majority of samples fits has less than 32 non empty frames\nN_EMPTY_FRAMES = (NON_EMPTY_FRAME_IDXS_TRAIN != -1).sum(axis=1) \nN_EMPTY_FRAMES_WATERFALL = []\nfor n in tqdm(range(1,INPUT_SIZE+1)):\n N_EMPTY_FRAMES_WATERFALL.append(sum(N_EMPTY_FRAMES >= n) / len(NON_EMPTY_FRAME_IDXS_TRAIN) * 100)\n\nplt.figure(figsize=(18,10))\nplt.title('Waterfall Plot For Number Of Non Empty Frames')\npd.Series(N_EMPTY_FRAMES_WATERFALL).plot(kind='bar')\nplt.grid(axis='y')\nplt.xticks(np.arange(INPUT_SIZE), np.arange(1, INPUT_SIZE+1))\nplt.xlabel('Number of Non Empty Frames', size=16)\nplt.yticks(np.arange(0, 100+10, 10))\nplt.ylim(0, 100)\nplt.ylabel('Percentage of Samples With At Least N Non Empty Frames', size=16)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:00:20.571633Z","iopub.execute_input":"2024-02-25T19:00:20.572344Z","iopub.status.idle":"2024-02-25T19:00:39.037148Z","shell.execute_reply.started":"2024-02-25T19:00:20.572303Z","shell.execute_reply":"2024-02-25T19:00:39.036061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Percentage of Frames Filled","metadata":{}},{"cell_type":"code","source":"# Percentage of frames filled, this is the maximum fill percentage of each landmark\nP_DATA_FILLED = (NON_EMPTY_FRAME_IDXS_TRAIN != -1).sum() / NON_EMPTY_FRAME_IDXS_TRAIN.size * 100\nprint(f'P_DATA_FILLED: {P_DATA_FILLED:.2f}%')","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:00:39.038521Z","iopub.execute_input":"2024-02-25T19:00:39.038883Z","iopub.status.idle":"2024-02-25T19:00:39.051213Z","shell.execute_reply.started":"2024-02-25T19:00:39.038850Z","shell.execute_reply":"2024-02-25T19:00:39.050218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Statistics - Lips","metadata":{}},{"cell_type":"code","source":"# Percentage of Lips Measurements\nP_LEFT_LIPS_MEASUREMENTS = (X_train[:,:,LIPS_IDXS] != 0).sum() / X_train[:,:,LIPS_IDXS].size / P_DATA_FILLED * 1e4\nprint(f'P_LEFT_LIPS_MEASUREMENTS: {P_LEFT_LIPS_MEASUREMENTS:.2f}%')","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:00:39.052809Z","iopub.execute_input":"2024-02-25T19:00:39.053229Z","iopub.status.idle":"2024-02-25T19:00:51.429076Z","shell.execute_reply.started":"2024-02-25T19:00:39.053198Z","shell.execute_reply":"2024-02-25T19:00:51.427961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_lips_mean_std():\n # LIPS\n LIPS_MEAN_X = np.zeros([LIPS_IDXS.size], dtype=np.float32)\n LIPS_MEAN_Y = np.zeros([LIPS_IDXS.size], dtype=np.float32)\n LIPS_STD_X = np.zeros([LIPS_IDXS.size], dtype=np.float32)\n LIPS_STD_Y = np.zeros([LIPS_IDXS.size], dtype=np.float32)\n\n fig, axes = plt.subplots(3, 1, figsize=(15, N_DIMS*6))\n\n for col, ll in enumerate(tqdm( np.transpose(X_train[:,:,LIPS_IDXS], [2,3,0,1]).reshape([LIPS_IDXS.size, N_DIMS, -1]) )):\n for dim, l in enumerate(ll):\n v = l[np.nonzero(l)]\n if dim == 0: # X\n LIPS_MEAN_X[col] = v.mean()\n LIPS_STD_X[col] = v.std()\n if dim == 1: # Y\n LIPS_MEAN_Y[col] = v.mean()\n LIPS_STD_Y[col] = v.std()\n\n axes[dim].boxplot(v, notch=False, showfliers=False, positions=[col], whis=[5,95])\n\n for ax, dim_name in zip(axes, DIM_NAMES):\n ax.set_title(f'Lips {dim_name.upper()} Dimension', size=24)\n ax.tick_params(axis='x', labelsize=8)\n ax.grid(axis='y')\n\n plt.subplots_adjust(hspace=0.50)\n plt.show()\n\n LIPS_MEAN = np.array([LIPS_MEAN_X, LIPS_MEAN_Y]).T\n LIPS_STD = np.array([LIPS_STD_X, LIPS_STD_Y]).T\n \n return LIPS_MEAN, LIPS_STD\n\nLIPS_MEAN, LIPS_STD = get_lips_mean_std()","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:00:51.430431Z","iopub.execute_input":"2024-02-25T19:00:51.430875Z","iopub.status.idle":"2024-02-25T19:01:09.625378Z","shell.execute_reply.started":"2024-02-25T19:00:51.430841Z","shell.execute_reply":"2024-02-25T19:01:09.624328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Statistics - Hands","metadata":{}},{"cell_type":"code","source":"# Verify Normalised to Left Hand Dominant\nP_LEFT_HAND_MEASUREMENTS = (X_train[:,:,LEFT_HAND_IDXS] != 0).sum() / X_train[:,:,LEFT_HAND_IDXS].size / P_DATA_FILLED * 1e4\n# P_RIGHT_HAND_MEASUREMENTS = (X_train[:,:,RIGHT_HAND_IDXS] != 0).sum() / X_train[:,:,RIGHT_HAND_IDXS].size / P_DATA_FILLED * 1e4\nprint(f'P_LEFT_HAND_MEASUREMENTS: {P_LEFT_HAND_MEASUREMENTS:.2f}%')","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:01:09.627168Z","iopub.execute_input":"2024-02-25T19:01:09.627549Z","iopub.status.idle":"2024-02-25T19:01:16.159058Z","shell.execute_reply.started":"2024-02-25T19:01:09.627510Z","shell.execute_reply":"2024-02-25T19:01:16.157814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_left_right_hand_mean_std():\n # LEFT HAND\n LEFT_HANDS_MEAN_X = np.zeros([LEFT_HAND_IDXS.size], dtype=np.float32)\n LEFT_HANDS_MEAN_Y = np.zeros([LEFT_HAND_IDXS.size], dtype=np.float32)\n LEFT_HANDS_STD_X = np.zeros([LEFT_HAND_IDXS.size], dtype=np.float32)\n LEFT_HANDS_STD_Y = np.zeros([LEFT_HAND_IDXS.size], dtype=np.float32)\n\n fig, axes = plt.subplots(3, 1, figsize=(15, N_DIMS*6))\n\n for col, ll in enumerate(tqdm( np.transpose(X_train[:,:,LEFT_HAND_IDXS], [2,3,0,1]).reshape([LEFT_HAND_IDXS.size, N_DIMS, -1]) )):\n for dim, l in enumerate(ll):\n v = l[np.nonzero(l)]\n if dim == 0: # X\n LEFT_HANDS_MEAN_X[col] = v.mean()\n LEFT_HANDS_STD_X[col] = v.std()\n if dim == 1: # Y\n LEFT_HANDS_MEAN_Y[col] = v.mean()\n LEFT_HANDS_STD_Y[col] = v.std()\n # Plot\n axes[dim].boxplot(v, notch=False, showfliers=False, positions=[col], whis=[5,95])\n\n for ax, dim_name in zip(axes, DIM_NAMES):\n ax.set_title(f'Hands {dim_name.upper()} Dimension', size=24)\n ax.tick_params(axis='x', labelsize=8)\n ax.grid(axis='y')\n\n plt.subplots_adjust(hspace=0.50)\n plt.show()\n\n LEFT_HANDS_MEAN = np.array([LEFT_HANDS_MEAN_X, LEFT_HANDS_MEAN_Y]).T\n LEFT_HANDS_STD = np.array([LEFT_HANDS_STD_X, LEFT_HANDS_STD_Y]).T\n \n return LEFT_HANDS_MEAN, LEFT_HANDS_STD\n\nLEFT_HANDS_MEAN, LEFT_HANDS_STD = get_left_right_hand_mean_std()","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:01:16.160394Z","iopub.execute_input":"2024-02-25T19:01:16.160743Z","iopub.status.idle":"2024-02-25T19:01:26.363630Z","shell.execute_reply.started":"2024-02-25T19:01:16.160702Z","shell.execute_reply":"2024-02-25T19:01:26.362509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Statistics - Pose","metadata":{}},{"cell_type":"code","source":"# Percentage of Lips Measurements\nP_POSE_MEASUREMENTS = (X_train[:,:,POSE_IDXS] != 0).sum() / X_train[:,:,POSE_IDXS].size / P_DATA_FILLED * 1e4\nprint(f'P_POSE_MEASUREMENTS: {P_POSE_MEASUREMENTS:.2f}%')","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:01:26.365192Z","iopub.execute_input":"2024-02-25T19:01:26.366064Z","iopub.status.idle":"2024-02-25T19:01:27.959304Z","shell.execute_reply.started":"2024-02-25T19:01:26.366016Z","shell.execute_reply":"2024-02-25T19:01:27.958204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_pose_mean_std():\n # POSE\n POSE_MEAN_X = np.zeros([POSE_IDXS.size], dtype=np.float32)\n POSE_MEAN_Y = np.zeros([POSE_IDXS.size], dtype=np.float32)\n POSE_STD_X = np.zeros([POSE_IDXS.size], dtype=np.float32)\n POSE_STD_Y = np.zeros([POSE_IDXS.size], dtype=np.float32)\n\n fig, axes = plt.subplots(3, 1, figsize=(15, N_DIMS*6))\n\n for col, ll in enumerate(tqdm( np.transpose(X_train[:,:,POSE_IDXS], [2,3,0,1]).reshape([POSE_IDXS.size, N_DIMS, -1]) )):\n for dim, l in enumerate(ll):\n v = l[np.nonzero(l)]\n if dim == 0: # X\n POSE_MEAN_X[col] = v.mean()\n POSE_STD_X[col] = v.std()\n if dim == 1: # Y\n POSE_MEAN_Y[col] = v.mean()\n POSE_STD_Y[col] = v.std()\n\n axes[dim].boxplot(v, notch=False, showfliers=False, positions=[col], whis=[5,95])\n\n for ax, dim_name in zip(axes, DIM_NAMES):\n ax.set_title(f'Pose {dim_name.upper()} Dimension', size=24)\n ax.tick_params(axis='x', labelsize=8)\n ax.grid(axis='y')\n\n plt.subplots_adjust(hspace=0.50)\n plt.show()\n\n POSE_MEAN = np.array([POSE_MEAN_X, POSE_MEAN_Y]).T\n POSE_STD = np.array([POSE_STD_X, POSE_STD_Y]).T\n \n return POSE_MEAN, POSE_STD\n\nPOSE_MEAN, POSE_STD = get_pose_mean_std()","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:01:27.960870Z","iopub.execute_input":"2024-02-25T19:01:27.961353Z","iopub.status.idle":"2024-02-25T19:01:30.991030Z","shell.execute_reply.started":"2024-02-25T19:01:27.961311Z","shell.execute_reply":"2024-02-25T19:01:30.989968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Samples","metadata":{}},{"cell_type":"code","source":"# Custom sampler to get a batch containing N times all signs\ndef get_train_batch_all_signs(X, y, NON_EMPTY_FRAME_IDXS, n=BATCH_ALL_SIGNS_N):\n # Arrays to store batch in\n X_batch = np.zeros([NUM_CLASSES*n, INPUT_SIZE, N_COLS, N_DIMS], dtype=np.float32)\n y_batch = np.arange(0, NUM_CLASSES, step=1/n, dtype=np.float32).astype(np.int64)\n non_empty_frame_idxs_batch = np.zeros([NUM_CLASSES*n, INPUT_SIZE], dtype=np.float32)\n \n # Dictionary mapping ordinally encoded sign to corresponding sample indices\n CLASS2IDXS = {}\n for i in range(NUM_CLASSES):\n CLASS2IDXS[i] = np.argwhere(y == i).squeeze().astype(np.int32)\n \n while True:\n # Fill batch arrays\n for i in range(NUM_CLASSES):\n idxs = np.random.choice(CLASS2IDXS[i], n)\n X_batch[i*n:(i+1)*n] = X[idxs]\n non_empty_frame_idxs_batch[i*n:(i+1)*n] = NON_EMPTY_FRAME_IDXS[idxs]\n \n yield { 'frames': X_batch, 'non_empty_frame_idxs': non_empty_frame_idxs_batch }, y_batch","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:01:30.992382Z","iopub.execute_input":"2024-02-25T19:01:30.992704Z","iopub.status.idle":"2024-02-25T19:01:31.002795Z","shell.execute_reply.started":"2024-02-25T19:01:30.992673Z","shell.execute_reply":"2024-02-25T19:01:31.001615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dummy_dataset = get_train_batch_all_signs(X_train, y_train, NON_EMPTY_FRAME_IDXS_TRAIN)\nX_batch, y_batch = next(dummy_dataset)\n\nfor k, v in X_batch.items():\n print(f'{k} shape: {v.shape}, dtype: {v.dtype}')\n\n# Batch shape/dtype\nprint(f'y_batch shape: {y_batch.shape}, dtype: {y_batch.dtype}')\n# Verify each batch contains each sign exactly N times\ndisplay(pd.Series(y_batch).value_counts().to_frame('Counts'))","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:01:31.004342Z","iopub.execute_input":"2024-02-25T19:01:31.004699Z","iopub.status.idle":"2024-02-25T19:01:31.091377Z","shell.execute_reply.started":"2024-02-25T19:01:31.004666Z","shell.execute_reply":"2024-02-25T19:01:31.090379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Config","metadata":{}},{"cell_type":"code","source":"# Epsilon value for layer normalisation\nLAYER_NORM_EPS = 1e-6\n\n# Dense layer units for landmarks\nLIPS_UNITS = 384\nHANDS_UNITS = 384\nPOSE_UNITS = 384\n# final embedding and transformer embedding size\nUNITS = 512\n\n# Transformer\nNUM_BLOCKS = 2\nMLP_RATIO = 2\n\n# Dropout\nEMBEDDING_DROPOUT = 0.00\nMLP_DROPOUT_RATIO = 0.30\nCLASSIFIER_DROPOUT_RATIO = 0.10\n\n# Initiailizers\nINIT_HE_UNIFORM = tf.keras.initializers.he_uniform\nINIT_GLOROT_UNIFORM = tf.keras.initializers.glorot_uniform\nINIT_ZEROS = tf.keras.initializers.constant(0.0)\n# Activations\nGELU = tf.keras.activations.gelu\n\nprint(f'UNITS: {UNITS}')","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:01:31.092715Z","iopub.execute_input":"2024-02-25T19:01:31.093547Z","iopub.status.idle":"2024-02-25T19:01:31.101871Z","shell.execute_reply.started":"2024-02-25T19:01:31.093505Z","shell.execute_reply":"2024-02-25T19:01:31.100875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transformer\n\nNeed to implement transformer from scratch as TFLite does not support the native TF implementation of MultiHeadAttention.","metadata":{}},{"cell_type":"code","source":"# based on: https://stackoverflow.com/questions/67342988/verifying-the-implementation-of-multihead-attention-in-transformer\n# replaced softmax with softmax layer to support masked softmax\ndef scaled_dot_product(q,k,v, softmax, attention_mask):\n #calculates Q . K(transpose)\n qkt = tf.matmul(q,k,transpose_b=True)\n #caculates scaling factor\n dk = tf.math.sqrt(tf.cast(q.shape[-1],dtype=tf.float32))\n scaled_qkt = qkt/dk\n softmax = softmax(scaled_qkt, mask=attention_mask)\n \n z = tf.matmul(softmax,v)\n #shape: (m,Tx,depth), same shape as q,k,v\n return z\n\nclass MultiHeadAttention(tf.keras.layers.Layer):\n def __init__(self,d_model,num_of_heads):\n super(MultiHeadAttention,self).__init__()\n self.d_model = d_model\n self.num_of_heads = num_of_heads\n self.depth = d_model//num_of_heads\n self.wq = [tf.keras.layers.Dense(self.depth) for i in range(num_of_heads)]\n self.wk = [tf.keras.layers.Dense(self.depth) for i in range(num_of_heads)]\n self.wv = [tf.keras.layers.Dense(self.depth) for i in range(num_of_heads)]\n self.wo = tf.keras.layers.Dense(d_model)\n self.softmax = tf.keras.layers.Softmax()\n \n def call(self,x, attention_mask):\n \n multi_attn = []\n for i in range(self.num_of_heads):\n Q = self.wq[i](x)\n K = self.wk[i](x)\n V = self.wv[i](x)\n multi_attn.append(scaled_dot_product(Q,K,V, self.softmax, attention_mask))\n \n multi_head = tf.concat(multi_attn,axis=-1)\n multi_head_attention = self.wo(multi_head)\n return multi_head_attention","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:01:31.103300Z","iopub.execute_input":"2024-02-25T19:01:31.103773Z","iopub.status.idle":"2024-02-25T19:01:31.118399Z","shell.execute_reply.started":"2024-02-25T19:01:31.103716Z","shell.execute_reply":"2024-02-25T19:01:31.117276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Full Transformer\nclass Transformer(tf.keras.Model):\n def __init__(self, num_blocks):\n super(Transformer, self).__init__(name='transformer')\n self.num_blocks = num_blocks\n \n def build(self, input_shape):\n self.ln_1s = []\n self.mhas = []\n self.ln_2s = []\n self.mlps = []\n # Make Transformer Blocks\n for i in range(self.num_blocks):\n # Multi Head Attention\n self.mhas.append(MultiHeadAttention(UNITS, 8))\n # Multi Layer Perception\n self.mlps.append(tf.keras.Sequential([\n tf.keras.layers.Dense(UNITS * MLP_RATIO, activation=GELU, kernel_initializer=INIT_GLOROT_UNIFORM),\n tf.keras.layers.Dropout(MLP_DROPOUT_RATIO),\n tf.keras.layers.Dense(UNITS, kernel_initializer=INIT_HE_UNIFORM),\n ]))\n \n def call(self, x, attention_mask):\n # Iterate input over transformer blocks\n for mha, mlp in zip(self.mhas, self.mlps):\n x = x + mha(x, attention_mask)\n x = x + mlp(x)\n \n return x","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:01:31.119859Z","iopub.execute_input":"2024-02-25T19:01:31.120782Z","iopub.status.idle":"2024-02-25T19:01:31.132654Z","shell.execute_reply.started":"2024-02-25T19:01:31.120706Z","shell.execute_reply":"2024-02-25T19:01:31.131641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Landmark Embedding","metadata":{}},{"cell_type":"code","source":"class LandmarkEmbedding(tf.keras.Model):\n def __init__(self, units, name):\n super(LandmarkEmbedding, self).__init__(name=f'{name}_embedding')\n self.units = units\n \n def build(self, input_shape):\n # Embedding for missing landmark in frame, initizlied with zeros\n self.empty_embedding = self.add_weight(\n name=f'{self.name}_empty_embedding',\n shape=[self.units],\n initializer=INIT_ZEROS,\n )\n # Embedding\n self.dense = tf.keras.Sequential([\n tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_1', use_bias=False, kernel_initializer=INIT_GLOROT_UNIFORM),\n tf.keras.layers.Activation(GELU),\n tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_2', use_bias=False, kernel_initializer=INIT_HE_UNIFORM),\n ], name=f'{self.name}_dense')\n\n def call(self, x):\n return tf.where(\n # Checks whether landmark is missing in frame\n tf.reduce_sum(x, axis=2, keepdims=True) == 0,\n # If so, the empty embedding is used\n self.empty_embedding,\n # Otherwise the landmark data is embedded\n self.dense(x),\n )","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:01:31.133805Z","iopub.execute_input":"2024-02-25T19:01:31.134869Z","iopub.status.idle":"2024-02-25T19:01:31.146717Z","shell.execute_reply.started":"2024-02-25T19:01:31.134818Z","shell.execute_reply":"2024-02-25T19:01:31.145881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Embedding","metadata":{}},{"cell_type":"code","source":"class Embedding(tf.keras.Model):\n def __init__(self):\n super(Embedding, self).__init__()\n \n def get_diffs(self, l):\n S = l.shape[2]\n other = tf.expand_dims(l, 3)\n other = tf.repeat(other, S, axis=3)\n other = tf.transpose(other, [0,1,3,2])\n diffs = tf.expand_dims(l, 3) - other\n diffs = tf.reshape(diffs, [-1, INPUT_SIZE, S*S])\n return diffs\n\n def build(self, input_shape):\n # Positional Embedding, initialized with zeros\n self.positional_embedding = tf.keras.layers.Embedding(INPUT_SIZE+1, UNITS, embeddings_initializer=INIT_ZEROS)\n # Embedding layer for Landmarks\n self.lips_embedding = LandmarkEmbedding(LIPS_UNITS, 'lips')\n self.left_hand_embedding = LandmarkEmbedding(HANDS_UNITS, 'left_hand')\n self.pose_embedding = LandmarkEmbedding(POSE_UNITS, 'pose')\n # Landmark Weights\n self.landmark_weights = tf.Variable(tf.zeros([3], dtype=tf.float32), name='landmark_weights')\n # Fully Connected Layers for combined landmarks\n self.fc = tf.keras.Sequential([\n tf.keras.layers.Dense(UNITS, name='fully_connected_1', use_bias=False, kernel_initializer=INIT_GLOROT_UNIFORM),\n tf.keras.layers.Activation(GELU),\n tf.keras.layers.Dense(UNITS, name='fully_connected_2', use_bias=False, kernel_initializer=INIT_HE_UNIFORM),\n ], name='fc')\n\n\n def call(self, lips0, left_hand0, pose0, non_empty_frame_idxs, training=False):\n # Lips\n lips_embedding = self.lips_embedding(lips0)\n # Left Hand\n left_hand_embedding = self.left_hand_embedding(left_hand0)\n # Pose\n pose_embedding = self.pose_embedding(pose0)\n # Merge Embeddings of all landmarks with mean pooling\n x = tf.stack((\n lips_embedding, left_hand_embedding, pose_embedding,\n ), axis=3)\n x = x * tf.nn.softmax(self.landmark_weights)\n x = tf.reduce_sum(x, axis=3)\n # Fully Connected Layers\n x = self.fc(x)\n # Add Positional Embedding\n max_frame_idxs = tf.clip_by_value(\n tf.reduce_max(non_empty_frame_idxs, axis=1, keepdims=True),\n 1,\n np.PINF,\n )\n normalised_non_empty_frame_idxs = tf.where(\n tf.math.equal(non_empty_frame_idxs, -1.0),\n INPUT_SIZE,\n tf.cast(\n non_empty_frame_idxs / max_frame_idxs * INPUT_SIZE,\n tf.int32,\n ),\n )\n x = x + self.positional_embedding(normalised_non_empty_frame_idxs)\n \n return x","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:01:31.148146Z","iopub.execute_input":"2024-02-25T19:01:31.148432Z","iopub.status.idle":"2024-02-25T19:01:31.165373Z","shell.execute_reply.started":"2024-02-25T19:01:31.148404Z","shell.execute_reply":"2024-02-25T19:01:31.164316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Augmentation","metadata":{}},{"cell_type":"code","source":"# Not used, adds random X/y translation to input on samples level\nclass Augmentation(tf.keras.layers.Layer):\n def __init__(self, noise_std):\n super(Augmentation, self).__init__()\n self.noise_std = noise_std\n \n def add_noise(self, t):\n B = tf.shape(t)[0]\n return tf.where(\n t == 0.0,\n 0.0,\n t + tf.random.normal([B,1,1,tf.shape(t)[3]], 0, self.noise_std),\n )\n \n def call(self, lips0, left_hand0, pose0, training=False):\n if training:\n # Lips\n lips0 = self.add_noise(lips0)\n # Left Hand\n left_hand0 = self.add_noise(left_hand0)\n # Pose\n pose0 = self.add_noise(pose0)\n \n return lips0, left_hand0, pose0","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:01:31.166790Z","iopub.execute_input":"2024-02-25T19:01:31.167224Z","iopub.status.idle":"2024-02-25T19:01:31.179253Z","shell.execute_reply.started":"2024-02-25T19:01:31.167186Z","shell.execute_reply":"2024-02-25T19:01:31.178434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Sparse Categorical Crossentropy With Label Smoothing","metadata":{}},{"cell_type":"code","source":"# source:: https://stackoverflow.com/questions/60689185/label-smoothing-for-sparse-categorical-crossentropy\ndef scce_with_ls(y_true, y_pred):\n # One Hot Encode Sparsely Encoded Target Sign\n y_true = tf.cast(y_true, tf.int32)\n y_true = tf.one_hot(y_true, NUM_CLASSES, axis=1)\n y_true = tf.squeeze(y_true, axis=2)\n # Categorical Crossentropy with native label smoothing support\n return tf.keras.losses.categorical_crossentropy(y_true, y_pred, label_smoothing=0.25)","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:01:31.180496Z","iopub.execute_input":"2024-02-25T19:01:31.181089Z","iopub.status.idle":"2024-02-25T19:01:31.194071Z","shell.execute_reply.started":"2024-02-25T19:01:31.181058Z","shell.execute_reply":"2024-02-25T19:01:31.193058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"def get_model():\n # Inputs\n frames = tf.keras.layers.Input([INPUT_SIZE, N_COLS, N_DIMS], dtype=tf.float32, name='frames')\n non_empty_frame_idxs = tf.keras.layers.Input([INPUT_SIZE], dtype=tf.float32, name='non_empty_frame_idxs')\n # Padding Mask\n mask0 = tf.cast(tf.math.not_equal(non_empty_frame_idxs, -1), tf.float32)\n mask0 = tf.expand_dims(mask0, axis=2)\n # Random Frame Masking\n mask = tf.where(\n (tf.random.uniform(tf.shape(mask0)) > 0.25) & tf.math.not_equal(mask0, 0.0),\n 1.0,\n 0.0,\n )\n # Correct Samples Which are all masked now...\n mask = tf.where(\n tf.math.equal(tf.reduce_sum(mask, axis=[1,2], keepdims=True), 0.0),\n mask0,\n mask,\n )\n \n \n \"\"\"\n left_hand: 468:489\n pose: 489:522\n right_hand: 522:543\n \"\"\"\n x = frames\n x = tf.slice(x, [0,0,0,0], [-1,INPUT_SIZE, N_COLS, 2])\n # LIPS\n lips = tf.slice(x, [0,0,LIPS_START,0], [-1,INPUT_SIZE, 40, 2])\n lips = tf.where(\n tf.math.equal(lips, 0.0),\n 0.0,\n (lips - LIPS_MEAN) / LIPS_STD,\n )\n # LEFT HAND\n left_hand = tf.slice(x, [0,0,40,0], [-1,INPUT_SIZE, 21, 2])\n left_hand = tf.where(\n tf.math.equal(left_hand, 0.0),\n 0.0,\n (left_hand - LEFT_HANDS_MEAN) / LEFT_HANDS_STD,\n )\n # POSE\n pose = tf.slice(x, [0,0,61,0], [-1,INPUT_SIZE, 5, 2])\n pose = tf.where(\n tf.math.equal(pose, 0.0),\n 0.0,\n (pose - POSE_MEAN) / POSE_STD,\n )\n \n # Flatten\n lips = tf.reshape(lips, [-1, INPUT_SIZE, 40*2])\n left_hand = tf.reshape(left_hand, [-1, INPUT_SIZE, 21*2])\n pose = tf.reshape(pose, [-1, INPUT_SIZE, 5*2])\n \n # Embedding\n x = Embedding()(lips, left_hand, pose, non_empty_frame_idxs)\n \n # Encoder Transformer Blocks\n x = Transformer(NUM_BLOCKS)(x, mask)\n \n # Pooling\n x = tf.reduce_sum(x * mask, axis=1) / tf.reduce_sum(mask, axis=1)\n # Classifier Dropout\n x = tf.keras.layers.Dropout(CLASSIFIER_DROPOUT_RATIO)(x)\n # Classification Layer\n x = tf.keras.layers.Dense(NUM_CLASSES, activation=tf.keras.activations.softmax, kernel_initializer=INIT_GLOROT_UNIFORM)(x)\n \n outputs = x\n \n # Create Tensorflow Model\n model = tf.keras.models.Model(inputs=[frames, non_empty_frame_idxs], outputs=outputs)\n \n # Sparse Categorical Cross Entropy With Label Smoothing\n loss = scce_with_ls\n \n # Adam Optimizer with weight decay\n optimizer = tfa.optimizers.AdamW(learning_rate=1e-3, weight_decay=1e-5, clipnorm=1.0)\n \n # TopK Metrics\n metrics = [\n tf.keras.metrics.SparseCategoricalAccuracy(name='acc'),\n tf.keras.metrics.SparseTopKCategoricalAccuracy(k=5, name='top_5_acc'),\n tf.keras.metrics.SparseTopKCategoricalAccuracy(k=10, name='top_10_acc'),\n ]\n \n model.compile(loss=loss, optimizer=optimizer, metrics=metrics)\n \n return model","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:01:31.195313Z","iopub.execute_input":"2024-02-25T19:01:31.196087Z","iopub.status.idle":"2024-02-25T19:01:31.217108Z","shell.execute_reply.started":"2024-02-25T19:01:31.196057Z","shell.execute_reply":"2024-02-25T19:01:31.216186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.backend.clear_session()\n\nmodel = get_model()","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:01:31.218344Z","iopub.execute_input":"2024-02-25T19:01:31.218965Z","iopub.status.idle":"2024-02-25T19:01:33.609397Z","shell.execute_reply.started":"2024-02-25T19:01:31.218925Z","shell.execute_reply":"2024-02-25T19:01:33.608269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot model summary\nmodel.summary(expand_nested=True)","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:01:33.610591Z","iopub.execute_input":"2024-02-25T19:01:33.610914Z","iopub.status.idle":"2024-02-25T19:01:33.817381Z","shell.execute_reply.started":"2024-02-25T19:01:33.610883Z","shell.execute_reply":"2024-02-25T19:01:33.816359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.utils.plot_model(model, show_shapes=True, show_dtype=True, show_layer_names=True, expand_nested=True, show_layer_activations=True)","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:01:33.818976Z","iopub.execute_input":"2024-02-25T19:01:33.819246Z","iopub.status.idle":"2024-02-25T19:01:34.686131Z","shell.execute_reply.started":"2024-02-25T19:01:33.819219Z","shell.execute_reply":"2024-02-25T19:01:34.684971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# No NaN Predictions","metadata":{}},{"cell_type":"code","source":"if not PREPROCESS_DATA and TRAIN_MODEL:\n y_pred = model.predict_on_batch(X_batch).flatten()\n\n print(f'# NaN Values In Prediction: {np.isnan(y_pred).sum()}')","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:01:34.688144Z","iopub.execute_input":"2024-02-25T19:01:34.689124Z","iopub.status.idle":"2024-02-25T19:01:38.059908Z","shell.execute_reply.started":"2024-02-25T19:01:34.689078Z","shell.execute_reply":"2024-02-25T19:01:38.058629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Weight Initialization","metadata":{}},{"cell_type":"code","source":"if not PREPROCESS_DATA and TRAIN_MODEL:\n plt.figure(figsize=(12,5))\n plt.title(f'Softmax Output Initialized Model | µ={y_pred.mean():.3f}, σ={y_pred.std():.3f}', pad=25)\n pd.Series(y_pred).plot(kind='hist', bins=128, label='Class Probability')\n plt.xlim(0, max(y_pred) * 1.1)\n plt.vlines([1 / NUM_CLASSES], 0, plt.ylim()[1], color='red', label=f'Random Guessing Baseline 1/NUM_CLASSES={1 / NUM_CLASSES:.3f}')\n plt.grid()\n plt.legend()\n plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:01:38.061273Z","iopub.execute_input":"2024-02-25T19:01:38.061574Z","iopub.status.idle":"2024-02-25T19:01:38.712204Z","shell.execute_reply.started":"2024-02-25T19:01:38.061546Z","shell.execute_reply":"2024-02-25T19:01:38.711080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Learning Rate Scheduler","metadata":{}},{"cell_type":"code","source":"def lrfn(current_step, num_warmup_steps, lr_max, num_cycles=0.50, num_training_steps=N_EPOCHS):\n \n if current_step < num_warmup_steps:\n if WARMUP_METHOD == 'log':\n return lr_max * 0.10 ** (num_warmup_steps - current_step)\n else:\n return lr_max * 2 ** -(num_warmup_steps - current_step)\n else:\n progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))\n\n return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))) * lr_max","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:01:38.713835Z","iopub.execute_input":"2024-02-25T19:01:38.714576Z","iopub.status.idle":"2024-02-25T19:01:38.722249Z","shell.execute_reply.started":"2024-02-25T19:01:38.714532Z","shell.execute_reply":"2024-02-25T19:01:38.721166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_lr_schedule(lr_schedule, epochs):\n fig = plt.figure(figsize=(20, 10))\n plt.plot([None] + lr_schedule + [None])\n # X Labels\n x = np.arange(1, epochs + 1)\n x_axis_labels = [i if epochs <= 40 or i % 5 == 0 or i == 1 else None for i in range(1, epochs + 1)]\n plt.xlim([1, epochs])\n plt.xticks(x, x_axis_labels) # set tick step to 1 and let x axis start at 1\n \n # Increase y-limit for better readability\n plt.ylim([0, max(lr_schedule) * 1.1])\n \n # Title\n schedule_info = f'start: {lr_schedule[0]:.1E}, max: {max(lr_schedule):.1E}, final: {lr_schedule[-1]:.1E}'\n plt.title(f'Step Learning Rate Schedule, {schedule_info}', size=18, pad=12)\n \n # Plot Learning Rates\n for x, val in enumerate(lr_schedule):\n if epochs <= 40 or x % 5 == 0 or x is epochs - 1:\n if x < len(lr_schedule) - 1:\n if lr_schedule[x - 1] < val:\n ha = 'right'\n else:\n ha = 'left'\n elif x == 0:\n ha = 'right'\n else:\n ha = 'left'\n plt.plot(x + 1, val, 'o', color='black');\n offset_y = (max(lr_schedule) - min(lr_schedule)) * 0.02\n plt.annotate(f'{val:.1E}', xy=(x + 1, val + offset_y), size=12, ha=ha)\n \n plt.xlabel('Epoch', size=16, labelpad=5)\n plt.ylabel('Learning Rate', size=16, labelpad=5)\n plt.grid()\n plt.show()\n\n# Learning rate for encoder\nLR_SCHEDULE = [lrfn(step, num_warmup_steps=N_WARMUP_EPOCHS, lr_max=LR_MAX, num_cycles=0.50) for step in range(N_EPOCHS)]\n# Plot Learning Rate Schedule\nplot_lr_schedule(LR_SCHEDULE, epochs=N_EPOCHS)\n# Learning Rate Callback\nlr_callback = tf.keras.callbacks.LearningRateScheduler(lambda step: LR_SCHEDULE[step], verbose=1)","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:01:38.723642Z","iopub.execute_input":"2024-02-25T19:01:38.724289Z","iopub.status.idle":"2024-02-25T19:01:39.011561Z","shell.execute_reply.started":"2024-02-25T19:01:38.724249Z","shell.execute_reply":"2024-02-25T19:01:39.010382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Weight Decay Callback","metadata":{}},{"cell_type":"code","source":"# Custom callback to update weight decay with learning rate\nclass WeightDecayCallback(tf.keras.callbacks.Callback):\n def __init__(self, wd_ratio=WD_RATIO):\n self.step_counter = 0\n self.wd_ratio = wd_ratio\n \n def on_epoch_begin(self, epoch, logs=None):\n model.optimizer.weight_decay = model.optimizer.learning_rate * self.wd_ratio\n print(f'learning rate: {model.optimizer.learning_rate.numpy():.2e}, weight decay: {model.optimizer.weight_decay.numpy():.2e}')","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:01:39.012896Z","iopub.execute_input":"2024-02-25T19:01:39.013202Z","iopub.status.idle":"2024-02-25T19:01:39.022463Z","shell.execute_reply.started":"2024-02-25T19:01:39.013172Z","shell.execute_reply":"2024-02-25T19:01:39.021446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Performance Benchmark","metadata":{}},{"cell_type":"code","source":"%%timeit -n 100\nif TRAIN_MODEL:\n # Verify model prediction is <<<100ms\n model.predict_on_batch({ 'frames': X_train[:1], 'non_empty_frame_idxs': NON_EMPTY_FRAME_IDXS_TRAIN[:1] })\n pass","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:01:39.023958Z","iopub.execute_input":"2024-02-25T19:01:39.024267Z","iopub.status.idle":"2024-02-25T19:01:51.210783Z","shell.execute_reply.started":"2024-02-25T19:01:39.024237Z","shell.execute_reply":"2024-02-25T19:01:51.209701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"if USE_VAL:\n # Verify Validation Dataset Covers All Signs\n print(f'# Unique Signs in Validation Set: {pd.Series(y_val).nunique()}')\n # Value Counts\n display(pd.Series(y_val).value_counts().to_frame('Count').iloc[[1,2,3,-3,-2,-1]])","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:01:51.212107Z","iopub.execute_input":"2024-02-25T19:01:51.212400Z","iopub.status.idle":"2024-02-25T19:01:51.225579Z","shell.execute_reply.started":"2024-02-25T19:01:51.212373Z","shell.execute_reply":"2024-02-25T19:01:51.224417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluate Initialzied Model","metadata":{}},{"cell_type":"code","source":"# Sanity Check\nif TRAIN_MODEL and USE_VAL:\n _ = model.evaluate(*validation_data, verbose=2)","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:01:51.226859Z","iopub.execute_input":"2024-02-25T19:01:51.227176Z","iopub.status.idle":"2024-02-25T19:02:01.363103Z","shell.execute_reply.started":"2024-02-25T19:01:51.227146Z","shell.execute_reply":"2024-02-25T19:02:01.361939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"if TRAIN_MODEL:\n # Clear all models in GPU\n tf.keras.backend.clear_session()\n\n # Get new fresh model\n model = get_model()\n \n # Sanity Check\n model.summary()\n\n # Actual Training\n history = model.fit(\n x=get_train_batch_all_signs(X_train, y_train, NON_EMPTY_FRAME_IDXS_TRAIN),\n steps_per_epoch=len(X_train) // (NUM_CLASSES * BATCH_ALL_SIGNS_N),\n epochs=N_EPOCHS,\n # Only used for validation data since training data is a generator\n batch_size=BATCH_SIZE,\n validation_data=validation_data,\n callbacks=[\n lr_callback,\n WeightDecayCallback(),\n ],\n verbose = VERBOSE,\n )","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:02:01.366525Z","iopub.execute_input":"2024-02-25T19:02:01.367391Z","iopub.status.idle":"2024-02-25T19:03:05.309090Z","shell.execute_reply.started":"2024-02-25T19:02:01.367354Z","shell.execute_reply":"2024-02-25T19:03:05.308126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save Model Weights\nmodel.save_weights('model.h5')","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:03:05.310810Z","iopub.execute_input":"2024-02-25T19:03:05.311796Z","iopub.status.idle":"2024-02-25T19:03:05.468465Z","shell.execute_reply.started":"2024-02-25T19:03:05.311729Z","shell.execute_reply":"2024-02-25T19:03:05.467472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if USE_VAL:\n # Validation Predictions\n y_val_pred = model.predict({ 'frames': X_val, 'non_empty_frame_idxs': NON_EMPTY_FRAME_IDXS_VAL }, verbose=2).argmax(axis=1)\n # Label\n labels = [ORD2SIGN.get(i).replace(' ', '_') for i in range(NUM_CLASSES)]","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:03:05.469808Z","iopub.execute_input":"2024-02-25T19:03:05.470135Z","iopub.status.idle":"2024-02-25T19:03:14.739496Z","shell.execute_reply.started":"2024-02-25T19:03:05.470103Z","shell.execute_reply":"2024-02-25T19:03:14.738494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Landmark Attention Weights","metadata":{}},{"cell_type":"code","source":"# Landmark Weights\nfor w in model.get_layer('embedding').weights:\n if 'landmark_weights' in w.name:\n weights = scipy.special.softmax(w)\n\nlandmarks = ['lips_embedding', 'left_hand_embedding', 'pose_embedding']\n\nfor w, lm in zip(weights, landmarks):\n print(f'{lm} weight: {(w*100):.1f}%')","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:03:14.740811Z","iopub.execute_input":"2024-02-25T19:03:14.741153Z","iopub.status.idle":"2024-02-25T19:03:14.750329Z","shell.execute_reply.started":"2024-02-25T19:03:14.741121Z","shell.execute_reply":"2024-02-25T19:03:14.749192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Classification Report","metadata":{}},{"cell_type":"code","source":"def print_classification_report():\n # Classification report for all signs\n classification_report = sklearn.metrics.classification_report(\n y_val,\n y_val_pred,\n target_names=labels,\n output_dict=True,\n )\n # Round Data for better readability\n classification_report = pd.DataFrame(classification_report).T\n classification_report = classification_report.round(2)\n classification_report = classification_report.astype({\n 'support': np.uint16,\n })\n # Add signs\n classification_report['sign'] = [e if e in SIGN2ORD else -1 for e in classification_report.index]\n classification_report['sign_ord'] = classification_report['sign'].apply(SIGN2ORD.get).fillna(-1).astype(np.int16)\n # Sort on F1-score\n classification_report = pd.concat((\n classification_report.head(NUM_CLASSES).sort_values('f1-score', ascending=False),\n classification_report.tail(3),\n ))\n\n pd.options.display.max_rows = 999\n display(classification_report)","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:03:14.751793Z","iopub.execute_input":"2024-02-25T19:03:14.752209Z","iopub.status.idle":"2024-02-25T19:03:14.762623Z","shell.execute_reply.started":"2024-02-25T19:03:14.752170Z","shell.execute_reply":"2024-02-25T19:03:14.761688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if USE_VAL:\n print_classification_report()","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:03:14.763960Z","iopub.execute_input":"2024-02-25T19:03:14.764409Z","iopub.status.idle":"2024-02-25T19:03:14.944670Z","shell.execute_reply.started":"2024-02-25T19:03:14.764377Z","shell.execute_reply":"2024-02-25T19:03:14.943531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training History","metadata":{}},{"cell_type":"code","source":"def plot_history_metric(metric, f_best=np.argmax, ylim=None, yscale=None, yticks=None):\n plt.figure(figsize=(20, 10))\n \n values = history.history[metric]\n N_EPOCHS = len(values)\n val = 'val' in ''.join(history.history.keys())\n # Epoch Ticks\n if N_EPOCHS <= 20:\n x = np.arange(1, N_EPOCHS + 1)\n else:\n x = [1, 5] + [10 + 5 * idx for idx in range((N_EPOCHS - 10) // 5 + 1)]\n\n x_ticks = np.arange(1, N_EPOCHS+1)\n\n # Validation\n if val:\n val_values = history.history[f'val_{metric}']\n val_argmin = f_best(val_values)\n plt.plot(x_ticks, val_values, label=f'val')\n\n # summarize history for accuracy\n plt.plot(x_ticks, values, label=f'train')\n argmin = f_best(values)\n plt.scatter(argmin + 1, values[argmin], color='red', s=75, marker='o', label=f'train_best')\n if val:\n plt.scatter(val_argmin + 1, val_values[val_argmin], color='purple', s=75, marker='o', label=f'val_best')\n\n plt.title(f'Model {metric}', fontsize=24, pad=10)\n plt.ylabel(metric, fontsize=20, labelpad=10)\n\n if ylim:\n plt.ylim(ylim)\n\n if yscale is not None:\n plt.yscale(yscale)\n \n if yticks is not None:\n plt.yticks(yticks, fontsize=16)\n\n plt.xlabel('epoch', fontsize=20, labelpad=10) \n plt.tick_params(axis='x', labelsize=8)\n plt.xticks(x, fontsize=16) # set tick step to 1 and let x axis start at 1\n plt.yticks(fontsize=16)\n \n plt.legend(prop={'size': 10})\n plt.grid()\n plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:03:14.957082Z","iopub.execute_input":"2024-02-25T19:03:14.957882Z","iopub.status.idle":"2024-02-25T19:03:14.972007Z","shell.execute_reply.started":"2024-02-25T19:03:14.957849Z","shell.execute_reply":"2024-02-25T19:03:14.970844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if TRAIN_MODEL:\n plot_history_metric('loss', f_best=np.argmin)","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:03:14.973340Z","iopub.execute_input":"2024-02-25T19:03:14.973633Z","iopub.status.idle":"2024-02-25T19:03:15.236714Z","shell.execute_reply.started":"2024-02-25T19:03:14.973605Z","shell.execute_reply":"2024-02-25T19:03:15.235800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if TRAIN_MODEL:\n plot_history_metric('acc', ylim=[0,1], yticks=np.arange(0.0, 1.1, 0.1))","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:03:15.238104Z","iopub.execute_input":"2024-02-25T19:03:15.238480Z","iopub.status.idle":"2024-02-25T19:03:15.504442Z","shell.execute_reply.started":"2024-02-25T19:03:15.238436Z","shell.execute_reply":"2024-02-25T19:03:15.503412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if TRAIN_MODEL:\n plot_history_metric('top_5_acc', ylim=[0,1], yticks=np.arange(0.0, 1.1, 0.1))","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:03:15.505930Z","iopub.execute_input":"2024-02-25T19:03:15.506336Z","iopub.status.idle":"2024-02-25T19:03:15.841117Z","shell.execute_reply.started":"2024-02-25T19:03:15.506298Z","shell.execute_reply":"2024-02-25T19:03:15.840081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if TRAIN_MODEL:\n plot_history_metric('top_10_acc', ylim=[0,1], yticks=np.arange(0.0, 1.1, 0.1))","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:03:15.842573Z","iopub.execute_input":"2024-02-25T19:03:15.843063Z","iopub.status.idle":"2024-02-25T19:03:16.131953Z","shell.execute_reply.started":"2024-02-25T19:03:15.843021Z","shell.execute_reply":"2024-02-25T19:03:16.130927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission\n\nSubmission code loosley based on [this notebook](https://www.kaggle.com/code/dschettler8845/gislr-learn-eda-baseline#baseline) by [Darien Schettler\n](https://www.kaggle.com/dschettler8845)","metadata":{}},{"cell_type":"code","source":"# TFLite model for submission\nclass TFLiteModel(tf.Module):\n def __init__(self, model):\n super(TFLiteModel, self).__init__()\n\n # Load the feature generation and main models\n self.preprocess_layer = preprocess_layer\n self.model = model\n \n @tf.function(input_signature=[tf.TensorSpec(shape=[None, N_ROWS, N_DIMS], dtype=tf.float32, name='inputs')])\n def __call__(self, inputs):\n # Preprocess Data\n x, non_empty_frame_idxs = self.preprocess_layer(inputs)\n # Add Batch Dimension\n x = tf.expand_dims(x, axis=0)\n non_empty_frame_idxs = tf.expand_dims(non_empty_frame_idxs, axis=0)\n # Make Prediction\n outputs = self.model({ 'frames': x, 'non_empty_frame_idxs': non_empty_frame_idxs })\n # Squeeze Output 1x250 -> 250\n outputs = tf.squeeze(outputs, axis=0)\n\n # Return a dictionary with the output tensor\n return {'outputs': outputs}\n\n# Define TF Lite Model\ntflite_keras_model = TFLiteModel(model)\n\n# Sanity Check\ndemo_raw_data = load_relevant_data_subset(train['file_path'].values[5])\nprint(f'demo_raw_data shape: {demo_raw_data.shape}, dtype: {demo_raw_data.dtype}')\ndemo_output = tflite_keras_model(demo_raw_data)[\"outputs\"]\nprint(f'demo_output shape: {demo_output.shape}, dtype: {demo_output.dtype}')\ndemo_prediction = demo_output.numpy().argmax()\nprint(f'demo_prediction: {demo_prediction}, correct: {train.iloc[0][\"sign_ord\"]}')","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:03:16.133493Z","iopub.execute_input":"2024-02-25T19:03:16.133981Z","iopub.status.idle":"2024-02-25T19:03:18.492762Z","shell.execute_reply.started":"2024-02-25T19:03:16.133938Z","shell.execute_reply":"2024-02-25T19:03:18.491636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create Model Converter\nkeras_model_converter = tf.lite.TFLiteConverter.from_keras_model(tflite_keras_model)\n# Convert Model\ntflite_model = keras_model_converter.convert()\n# Write Model\nwith open('/kaggle/working/model.tflite', 'wb') as f:\n f.write(tflite_model)\n \n# Zip Model\n!zip submission.zip /kaggle/working/model.tflite","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:03:18.494125Z","iopub.execute_input":"2024-02-25T19:03:18.494442Z","iopub.status.idle":"2024-02-25T19:04:04.181661Z","shell.execute_reply.started":"2024-02-25T19:03:18.494412Z","shell.execute_reply":"2024-02-25T19:04:04.180105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Verify TFLite model can be loaded and used for prediction\n!pip install tflite-runtime\nimport tflite_runtime.interpreter as tflite\n\ninterpreter = tflite.Interpreter(\"/kaggle/working/model.tflite\")\nfound_signatures = list(interpreter.get_signature_list().keys())\nprediction_fn = interpreter.get_signature_runner(\"serving_default\")\n\noutput = prediction_fn(inputs=demo_raw_data)\nsign = output['outputs'].argmax()\n\nprint(\"PRED : \", ORD2SIGN.get(sign), f'[{sign}]')\nprint(\"TRUE : \", train.sign.values[0], f'[{train.sign_ord.values[0]}]')","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:04:04.184022Z","iopub.execute_input":"2024-02-25T19:04:04.184918Z","iopub.status.idle":"2024-02-25T19:04:16.510660Z","shell.execute_reply.started":"2024-02-25T19:04:04.184864Z","shell.execute_reply":"2024-02-25T19:04:16.509457Z"},"trusted":true},"execution_count":null,"outputs":[]}]}