{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"

Google - Isolated Sign Language Recognition - Visualization

\n

Enhance PopSign''s educational games for learning ASL

\n\n
\n
\n \n
\n
","metadata":{"execution":{"iopub.status.busy":"2023-03-11T19:39:14.387120Z","iopub.execute_input":"2023-03-11T19:39:14.388208Z","iopub.status.idle":"2023-03-11T19:39:14.399659Z","shell.execute_reply.started":"2023-03-11T19:39:14.388084Z","shell.execute_reply":"2023-03-11T19:39:14.397745Z"}}},{"cell_type":"markdown","source":"# Table of Contents #\n\n1. [Setup](#setup)\n1. [Visualization](#visualization)\n1. [Animation](#animation)","metadata":{"execution":{"iopub.status.busy":"2023-03-11T19:45:13.555322Z","iopub.execute_input":"2023-03-11T19:45:13.555861Z","iopub.status.idle":"2023-03-11T19:45:13.563656Z","shell.execute_reply.started":"2023-03-11T19:45:13.555816Z","shell.execute_reply":"2023-03-11T19:45:13.562041Z"}}},{"cell_type":"markdown","source":"___\n\n# Setup","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2023-03-11T19:26:42.796903Z","iopub.execute_input":"2023-03-11T19:26:42.797705Z","iopub.status.idle":"2023-03-11T19:26:42.809042Z","shell.execute_reply.started":"2023-03-11T19:26:42.797653Z","shell.execute_reply":"2023-03-11T19:26:42.807899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -q flatbuffers 2> /dev/null\n!pip install -q mediapipe 2> /dev/null","metadata":{"execution":{"iopub.status.busy":"2023-03-11T19:26:42.850013Z","iopub.execute_input":"2023-03-11T19:26:42.851207Z","iopub.status.idle":"2023-03-11T19:27:06.091916Z","shell.execute_reply.started":"2023-03-11T19:26:42.851134Z","shell.execute_reply":"2023-03-11T19:27:06.090252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2023-03-11T19:27:06.094850Z","iopub.execute_input":"2023-03-11T19:27:06.095287Z","iopub.status.idle":"2023-03-11T19:27:06.102831Z","shell.execute_reply.started":"2023-03-11T19:27:06.095246Z","shell.execute_reply":"2023-03-11T19:27:06.101686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\nimport json\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport mediapipe as mp\nimport matplotlib.pyplot as plt\n\nfrom matplotlib import animation\nfrom pathlib import Path\nimport IPython\nfrom IPython import display\nfrom IPython.display import HTML\n\nimport mediapipe as mp\nfrom mediapipe.framework.formats import landmark_pb2","metadata":{"execution":{"iopub.status.busy":"2023-03-11T19:27:06.103992Z","iopub.execute_input":"2023-03-11T19:27:06.104355Z","iopub.status.idle":"2023-03-11T19:27:06.782031Z","shell.execute_reply.started":"2023-03-11T19:27:06.104320Z","shell.execute_reply":"2023-03-11T19:27:06.780995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib as mpl\n\nmpl.rcParams['axes.spines.left'] = False\nmpl.rcParams['axes.spines.right'] = False\nmpl.rcParams['axes.spines.top'] = False\nmpl.rcParams['axes.spines.bottom'] = False","metadata":{"execution":{"iopub.status.busy":"2023-03-11T19:27:06.784405Z","iopub.execute_input":"2023-03-11T19:27:06.785617Z","iopub.status.idle":"2023-03-11T19:27:06.791584Z","shell.execute_reply.started":"2023-03-11T19:27:06.785571Z","shell.execute_reply":"2023-03-11T19:27:06.790212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Cfg:\n RANDOM_STATE = 2023\n INPUT_ROOT = Path('/kaggle/input/asl-signs/')\n OUTPUT_ROOT = Path('kaggle/working')\n INDEX_MAP_FILE = INPUT_ROOT / 'sign_to_prediction_index_map.json'\n TRAN_FILE = INPUT_ROOT / 'train.csv'\n INDEX = 'sequence_id'\n ROW_ID = 'row_id'","metadata":{"execution":{"iopub.status.busy":"2023-03-11T19:27:06.793616Z","iopub.execute_input":"2023-03-11T19:27:06.794110Z","iopub.status.idle":"2023-03-11T19:27:06.802659Z","shell.execute_reply.started":"2023-03-11T19:27:06.794058Z","shell.execute_reply":"2023-03-11T19:27:06.801590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'cv2 version: {cv2.__version__}')\nprint(f'MediaPipe version: {mp.__version__}')\nprint(f'IPython version: {IPython.__version__}')","metadata":{"execution":{"iopub.status.busy":"2023-03-11T19:27:06.804521Z","iopub.execute_input":"2023-03-11T19:27:06.805310Z","iopub.status.idle":"2023-03-11T19:27:06.813820Z","shell.execute_reply.started":"2023-03-11T19:27:06.805272Z","shell.execute_reply":"2023-03-11T19:27:06.812570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import data","metadata":{}},{"cell_type":"code","source":"def read_index_map(file_path=Cfg.INDEX_MAP_FILE):\n \"\"\"Reads the sign to predict as json file.\"\"\"\n with open(file_path, \"r\") as f:\n result = json.load(f)\n return result \n\ndef read_train(file_path=Cfg.TRAN_FILE):\n \"\"\"Reads the train csv as pandas data frame.\"\"\"\n return pd.read_csv(file_path).set_index(Cfg.INDEX)\n\ndef read_landmark_data_by_path(file_path, input_root=Cfg.INPUT_ROOT):\n \"\"\"Reads landmak data by the given file path.\"\"\"\n data = pd.read_parquet(input_root / file_path)\n return data.set_index(Cfg.ROW_ID)\n\ndef read_landmark_data_by_id(sequence_id, train_data):\n \"\"\"Reads the landmark data by the given sequence id.\"\"\"\n file_path = train_data.loc[sequence_id]['path']\n return read_landmark_data_by_path(file_path)","metadata":{"execution":{"iopub.status.busy":"2023-03-11T19:27:06.815759Z","iopub.execute_input":"2023-03-11T19:27:06.816127Z","iopub.status.idle":"2023-03-11T19:27:06.824442Z","shell.execute_reply.started":"2023-03-11T19:27:06.816092Z","shell.execute_reply":"2023-03-11T19:27:06.823230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = read_train()\ntrain_data.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-11T19:27:06.825840Z","iopub.execute_input":"2023-03-11T19:27:06.826749Z","iopub.status.idle":"2023-03-11T19:27:06.982614Z","shell.execute_reply.started":"2023-03-11T19:27:06.826710Z","shell.execute_reply":"2023-03-11T19:27:06.981404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"___\n\n# Visualization","metadata":{}},{"cell_type":"code","source":"mp_drawing = mp.solutions.drawing_utils\nmp_hands = mp.solutions.hands\nmp_face_mesh = mp.solutions.face_mesh\nmp_pose = mp.solutions.pose","metadata":{"execution":{"iopub.status.busy":"2023-03-11T19:27:07.026451Z","iopub.execute_input":"2023-03-11T19:27:07.026961Z","iopub.status.idle":"2023-03-11T19:27:07.033259Z","shell.execute_reply.started":"2023-03-11T19:27:07.026922Z","shell.execute_reply":"2023-03-11T19:27:07.031527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_random_sequence_id(train_data):\n idx = np.random.randint(0, len(train_data))\n return train_data.index[idx]","metadata":{"execution":{"iopub.status.busy":"2023-03-11T19:27:07.034689Z","iopub.execute_input":"2023-03-11T19:27:07.035085Z","iopub.status.idle":"2023-03-11T19:27:07.043888Z","shell.execute_reply.started":"2023-03-11T19:27:07.035043Z","shell.execute_reply":"2023-03-11T19:27:07.042823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_blank_image(height, width):\n return np.zeros((height, width, 3), np.uint8)\n\ndef draw_landmarks(\n data, \n image, \n frame_id, \n landmark_type, \n connection_type, \n landmark_color=(255, 0, 0), \n connection_color=(0, 20, 255), \n thickness=1, \n circle_radius=1\n):\n \"\"\"Draws landmarks\"\"\"\n df = data.groupby(['frame', 'type']).get_group((frame_id, landmark_type))\n landmarks = [landmark_pb2.NormalizedLandmark(x=lm.x, y=lm.y, z=lm.z) for idx, lm in df.iterrows()]\n landmark_list = landmark_pb2.NormalizedLandmarkList(landmark = landmarks)\n\n mp_drawing.draw_landmarks(\n image=image,\n landmark_list=landmark_list, \n connections=connection_type,\n landmark_drawing_spec=mp_drawing.DrawingSpec(\n color=landmark_color, \n thickness=thickness, \n circle_radius=circle_radius),\n connection_drawing_spec=mp_drawing.DrawingSpec(\n color=connection_color, \n thickness=thickness, \n circle_radius=circle_radius))\n return image\n\ndef draw_left_hand(data, image, frame_id):\n return draw_landmarks(\n data, \n image, \n frame_id, \n landmark_type='left_hand', \n connection_type=mp_hands.HAND_CONNECTIONS,\n landmark_color=(255, 0, 0),\n connection_color=(0, 20, 255), \n thickness=3, \n circle_radius=3)\n\ndef draw_right_hand(data, image, frame_id):\n return draw_landmarks(\n data, \n image, \n frame_id, \n landmark_type='right_hand', \n connection_type=mp_hands.HAND_CONNECTIONS,\n landmark_color=(255, 0, 0),\n connection_color=(0, 20, 255),\n thickness=3, \n circle_radius=3)\n\ndef draw_face(data, image, frame_id):\n return draw_landmarks(\n data, \n image, \n frame_id, \n landmark_type='face', \n connection_type=mp_face_mesh.FACEMESH_TESSELATION,\n landmark_color=(255, 255, 255),\n connection_color=(0, 255, 0)) \n \ndef draw_pose(data, image, frame_id):\n return draw_landmarks(\n data, \n image, \n frame_id, \n landmark_type='pose', \n connection_type=mp_pose.POSE_CONNECTIONS,\n landmark_color=(255, 255, 255),\n connection_color=(255, 0, 0),\n thickness=2, \n circle_radius=2)\n\ndef create_frame(data, frame_id, height=1000, width=1000):\n image = create_blank_image(height, width) \n\n draw_pose(data, image, frame_id) \n draw_left_hand(data, image, frame_id) \n draw_right_hand(data, image, frame_id) \n draw_face(data, image, frame_id)\n \n return image","metadata":{"execution":{"iopub.status.busy":"2023-03-11T19:27:07.045520Z","iopub.execute_input":"2023-03-11T19:27:07.046054Z","iopub.status.idle":"2023-03-11T19:27:07.063539Z","shell.execute_reply.started":"2023-03-11T19:27:07.046016Z","shell.execute_reply":"2023-03-11T19:27:07.062270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"height = 800\nwidth = 600\n\nsequence_id = 1000106739\ndata = read_landmark_data_by_id(sequence_id, train_data)\n\nframe_id = data['frame'][0]","metadata":{"execution":{"iopub.status.busy":"2023-03-11T19:27:07.065445Z","iopub.execute_input":"2023-03-11T19:27:07.066632Z","iopub.status.idle":"2023-03-11T19:27:07.109818Z","shell.execute_reply.started":"2023-03-11T19:27:07.066587Z","shell.execute_reply":"2023-03-11T19:27:07.108726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Hand Landmarks\n\n
\n
\n \n
\n
\n\n[Source](https://developers.google.com/mediapipe/solutions/vision/hand_landmarker#get_started)","metadata":{}},{"cell_type":"code","source":"_, ax = plt.subplots(1, 1, figsize=(4, 4))\nimage = draw_right_hand(data, image=create_blank_image(height, width), frame_id=frame_id)\n \nax.imshow(image)\nax.axis('off')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-11T19:27:07.111193Z","iopub.execute_input":"2023-03-11T19:27:07.111741Z","iopub.status.idle":"2023-03-11T19:27:07.295980Z","shell.execute_reply.started":"2023-03-11T19:27:07.111706Z","shell.execute_reply":"2023-03-11T19:27:07.294294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Face Landmarks","metadata":{}},{"cell_type":"code","source":"_, ax = plt.subplots(1, 1, figsize=(4, 4))\nimage = draw_face(data, image=create_blank_image(height, width), frame_id=frame_id)\n \nax.imshow(image)\nax.axis('off')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-11T19:27:07.298599Z","iopub.execute_input":"2023-03-11T19:27:07.299686Z","iopub.status.idle":"2023-03-11T19:27:07.524776Z","shell.execute_reply.started":"2023-03-11T19:27:07.299614Z","shell.execute_reply":"2023-03-11T19:27:07.522921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pose Landmarks\n\n
\n
\n \n
\n
\n\n[Source](https://google.github.io/mediapipe/solutions/pose.html#pose-landmark-model-blazepose-ghum-3d)","metadata":{}},{"cell_type":"code","source":"_, ax = plt.subplots(1, 1, figsize=(4, 4))\nimage = draw_pose(data, image=create_blank_image(height, width), frame_id=frame_id)\n \nax.imshow(image)\nax.axis('off')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-11T19:27:07.527412Z","iopub.execute_input":"2023-03-11T19:27:07.528532Z","iopub.status.idle":"2023-03-11T19:27:07.714542Z","shell.execute_reply.started":"2023-03-11T19:27:07.528458Z","shell.execute_reply":"2023-03-11T19:27:07.712782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"___\n\n# Animation","metadata":{}},{"cell_type":"code","source":"def create_frames(sequence_id, train_data, height=800, width=800):\n data = read_landmark_data_by_id(sequence_id, train_data)\n frame_ids = data['frame'].unique()\n images = [create_frame(data, frame_id=fid, height=height, width=width) for fid in frame_ids]\n return np.array(images)","metadata":{"execution":{"iopub.status.busy":"2023-03-11T19:27:07.717098Z","iopub.execute_input":"2023-03-11T19:27:07.718191Z","iopub.status.idle":"2023-03-11T19:27:07.727951Z","shell.execute_reply.started":"2023-03-11T19:27:07.718104Z","shell.execute_reply":"2023-03-11T19:27:07.726303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_animation(images, fig, ax):\n ax.axis('off')\n \n ims = []\n for img in images:\n im = ax.imshow(img, animated=True)\n ims.append([im])\n \n func_animation = animation.ArtistAnimation(\n fig, \n ims, \n interval=100, \n blit=True,\n repeat_delay=1000)\n\n return func_animation\n\ndef get_sign_by_id(sequence_id, train_data):\n return train_data.loc[sequence_id]['sign']\n\ndef play_animation(sequence_id, train_data, height, width, figsize=(4, 4)):\n frames = create_frames(sequence_id, train_data, height=height, width=width)\n sign = get_sign_by_id(sequence_id, train_data)\n \n fig, ax = plt.subplots(1, 1, figsize=figsize)\n anim = create_animation(frames, fig, ax)\n ax.set_title(f'Sign: {sign}')\n \n video = anim.to_html5_video()\n html = display.HTML(video)\n display.display(html)\n plt.close()","metadata":{"execution":{"iopub.status.busy":"2023-03-11T19:27:07.730586Z","iopub.execute_input":"2023-03-11T19:27:07.731661Z","iopub.status.idle":"2023-03-11T19:27:07.746718Z","shell.execute_reply.started":"2023-03-11T19:27:07.731592Z","shell.execute_reply":"2023-03-11T19:27:07.745854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sequence_id = get_random_sequence_id(train_data)\nplay_animation(sequence_id, train_data, height=height, width=width)","metadata":{"execution":{"iopub.status.busy":"2023-03-11T19:27:07.748068Z","iopub.execute_input":"2023-03-11T19:27:07.748628Z","iopub.status.idle":"2023-03-11T19:27:26.162926Z","shell.execute_reply.started":"2023-03-11T19:27:07.748593Z","shell.execute_reply":"2023-03-11T19:27:26.161609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Sign `happy`","metadata":{"execution":{"iopub.status.busy":"2023-03-11T19:17:26.072721Z","iopub.execute_input":"2023-03-11T19:17:26.073395Z","iopub.status.idle":"2023-03-11T19:17:26.080904Z","shell.execute_reply.started":"2023-03-11T19:17:26.073273Z","shell.execute_reply":"2023-03-11T19:17:26.079210Z"}}},{"cell_type":"code","source":"data = train_data[train_data['sign'] == 'happy']\nsequence_id = get_random_sequence_id(data)\n\nplay_animation(sequence_id, train_data, height=height, width=width)","metadata":{"execution":{"iopub.status.busy":"2023-03-11T19:27:26.164687Z","iopub.execute_input":"2023-03-11T19:27:26.165131Z","iopub.status.idle":"2023-03-11T19:27:30.639389Z","shell.execute_reply.started":"2023-03-11T19:27:26.165086Z","shell.execute_reply":"2023-03-11T19:27:30.637737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Sign `cow`","metadata":{}},{"cell_type":"code","source":"data = train_data[train_data['sign'] == 'cow']\nsequence_id = get_random_sequence_id(data)\n\nplay_animation(sequence_id, train_data, height=height, width=width)","metadata":{"execution":{"iopub.status.busy":"2023-03-11T19:27:30.641602Z","iopub.execute_input":"2023-03-11T19:27:30.642009Z","iopub.status.idle":"2023-03-11T19:27:34.479228Z","shell.execute_reply.started":"2023-03-11T19:27:30.641966Z","shell.execute_reply":"2023-03-11T19:27:34.477794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Sign `chocolate`","metadata":{}},{"cell_type":"code","source":"data = train_data[train_data['sign'] == 'chocolate']\nsequence_id = get_random_sequence_id(data)\n\nplay_animation(sequence_id, train_data, height=height, width=width)","metadata":{"execution":{"iopub.status.busy":"2023-03-11T19:27:34.481107Z","iopub.execute_input":"2023-03-11T19:27:34.481482Z","iopub.status.idle":"2023-03-11T19:27:38.228316Z","shell.execute_reply.started":"2023-03-11T19:27:34.481448Z","shell.execute_reply":"2023-03-11T19:27:38.226905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Sign `tiger`","metadata":{}},{"cell_type":"code","source":"data = train_data[train_data['sign'] == 'tiger']\nsequence_id = get_random_sequence_id(data)\n\nplay_animation(sequence_id, train_data, height=height, width=width)","metadata":{"execution":{"iopub.status.busy":"2023-03-11T19:27:38.230449Z","iopub.execute_input":"2023-03-11T19:27:38.231316Z","iopub.status.idle":"2023-03-11T19:27:43.523079Z","shell.execute_reply.started":"2023-03-11T19:27:38.231260Z","shell.execute_reply":"2023-03-11T19:27:43.521900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Sign `kitty`","metadata":{}},{"cell_type":"code","source":"data = train_data[train_data['sign'] == 'kitty']\nsequence_id = get_random_sequence_id(data)\n\nplay_animation(sequence_id, train_data, height=height, width=width)","metadata":{"execution":{"iopub.status.busy":"2023-03-11T19:27:43.524710Z","iopub.execute_input":"2023-03-11T19:27:43.525984Z","iopub.status.idle":"2023-03-11T19:28:12.726608Z","shell.execute_reply.started":"2023-03-11T19:27:43.525923Z","shell.execute_reply":"2023-03-11T19:28:12.725210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Sign `moon`","metadata":{}},{"cell_type":"code","source":"data = train_data[train_data['sign'] == 'moon']\nsequence_id = get_random_sequence_id(data)\n\nplay_animation(sequence_id, train_data, height=height, width=width)","metadata":{"execution":{"iopub.status.busy":"2023-03-11T19:28:12.728795Z","iopub.execute_input":"2023-03-11T19:28:12.729662Z","iopub.status.idle":"2023-03-11T19:28:23.550318Z","shell.execute_reply.started":"2023-03-11T19:28:12.729606Z","shell.execute_reply":"2023-03-11T19:28:23.548340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Thanks for reading. If this notebook was helpful for you, please vote for it.**","metadata":{}}]}