{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# The Purpose of This Notebook\n* My goal in this notebook is to show a basic visualization of the coordinate data given to us. \n* I find that a graph can be far easier to read than an array of numbers, and I hope it will be insightful to you as well. \n* In version 3 I have stabilized the displayed video to reduce jitter and make movement more clear. ","metadata":{}},{"cell_type":"code","source":"## Libraries Used\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom matplotlib.animation import FuncAnimation\nfrom IPython.display import HTML\n\ndir = '/kaggle/input/asl-signs'\ntrain = pd.read_csv(f'{dir}/train.csv')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-24T17:32:44.691403Z","iopub.execute_input":"2023-02-24T17:32:44.691878Z","iopub.status.idle":"2023-02-24T17:32:44.933767Z","shell.execute_reply.started":"2023-02-24T17:32:44.691784Z","shell.execute_reply":"2023-02-24T17:32:44.932785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### This is the file we will be looking at. Feel free to change the directory to any of the files available to visualize them as well.\n* The y values all seem to be inverted, so inverting them back shows the video right side up.","metadata":{}},{"cell_type":"code","source":"## Change this directory to any file\npath_to_sign = 'train_landmark_files/16069/1011655866.parquet'\nsign = pd.read_parquet(f'{dir}/{path_to_sign}')\nsign.y = sign.y * -1","metadata":{"execution":{"iopub.status.busy":"2023-02-24T17:32:46.119744Z","iopub.execute_input":"2023-02-24T17:32:46.120300Z","iopub.status.idle":"2023-02-24T17:32:46.272269Z","shell.execute_reply.started":"2023-02-24T17:32:46.120271Z","shell.execute_reply":"2023-02-24T17:32:46.271597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### These two functions return the coordinate values with connecting lines for hands and pose.","metadata":{}},{"cell_type":"code","source":"def get_hand_points(hand):\n    x = [[hand.iloc[0].x, hand.iloc[1].x, hand.iloc[2].x, hand.iloc[3].x, hand.iloc[4].x], # Thumb\n         [hand.iloc[5].x, hand.iloc[6].x, hand.iloc[7].x, hand.iloc[8].x], # Index\n         [hand.iloc[9].x, hand.iloc[10].x, hand.iloc[11].x, hand.iloc[12].x], \n         [hand.iloc[13].x, hand.iloc[14].x, hand.iloc[15].x, hand.iloc[16].x], \n         [hand.iloc[17].x, hand.iloc[18].x, hand.iloc[19].x, hand.iloc[20].x], \n         [hand.iloc[0].x, hand.iloc[5].x, hand.iloc[9].x, hand.iloc[13].x, hand.iloc[17].x, hand.iloc[0].x]]\n\n    y = [[hand.iloc[0].y, hand.iloc[1].y, hand.iloc[2].y, hand.iloc[3].y, hand.iloc[4].y],  #Thumb\n         [hand.iloc[5].y, hand.iloc[6].y, hand.iloc[7].y, hand.iloc[8].y], # Index\n         [hand.iloc[9].y, hand.iloc[10].y, hand.iloc[11].y, hand.iloc[12].y], \n         [hand.iloc[13].y, hand.iloc[14].y, hand.iloc[15].y, hand.iloc[16].y], \n         [hand.iloc[17].y, hand.iloc[18].y, hand.iloc[19].y, hand.iloc[20].y], \n         [hand.iloc[0].y, hand.iloc[5].y, hand.iloc[9].y, hand.iloc[13].y, hand.iloc[17].y, hand.iloc[0].y]] \n    return x, y\n\ndef get_pose_points(pose):\n    x = [[pose.iloc[8].x, pose.iloc[6].x, pose.iloc[5].x, pose.iloc[4].x, pose.iloc[0].x, pose.iloc[1].x, pose.iloc[2].x, pose.iloc[3].x, pose.iloc[7].x], \n         [pose.iloc[10].x, pose.iloc[9].x], \n         [pose.iloc[22].x, pose.iloc[16].x, pose.iloc[20].x, pose.iloc[18].x, pose.iloc[16].x, pose.iloc[14].x, pose.iloc[12].x, \n          pose.iloc[11].x, pose.iloc[13].x, pose.iloc[15].x, pose.iloc[17].x, pose.iloc[19].x, pose.iloc[15].x, pose.iloc[21].x], \n         [pose.iloc[12].x, pose.iloc[24].x, pose.iloc[26].x, pose.iloc[28].x, pose.iloc[30].x, pose.iloc[32].x, pose.iloc[28].x], \n         [pose.iloc[11].x, pose.iloc[23].x, pose.iloc[25].x, pose.iloc[27].x, pose.iloc[29].x, pose.iloc[31].x, pose.iloc[27].x], \n         [pose.iloc[24].x, pose.iloc[23].x]\n        ]\n\n    y = [[pose.iloc[8].y, pose.iloc[6].y, pose.iloc[5].y, pose.iloc[4].y, pose.iloc[0].y, pose.iloc[1].y, pose.iloc[2].y, pose.iloc[3].y, pose.iloc[7].y], \n         [pose.iloc[10].y, pose.iloc[9].y], \n         [pose.iloc[22].y, pose.iloc[16].y, pose.iloc[20].y, pose.iloc[18].y, pose.iloc[16].y, pose.iloc[14].y, pose.iloc[12].y, \n          pose.iloc[11].y, pose.iloc[13].y, pose.iloc[15].y, pose.iloc[17].y, pose.iloc[19].y, pose.iloc[15].y, pose.iloc[21].y], \n         [pose.iloc[12].y, pose.iloc[24].y, pose.iloc[26].y, pose.iloc[28].y, pose.iloc[30].y, pose.iloc[32].y, pose.iloc[28].y], \n         [pose.iloc[11].y, pose.iloc[23].y, pose.iloc[25].y, pose.iloc[27].y, pose.iloc[29].y, pose.iloc[31].y, pose.iloc[27].y], \n         [pose.iloc[24].y, pose.iloc[23].y]\n        ]\n    return x, y","metadata":{"execution":{"iopub.status.busy":"2023-02-24T17:32:47.610716Z","iopub.execute_input":"2023-02-24T17:32:47.611315Z","iopub.status.idle":"2023-02-24T17:32:47.633716Z","shell.execute_reply.started":"2023-02-24T17:32:47.611285Z","shell.execute_reply":"2023-02-24T17:32:47.632747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### This animation shows the entire body and uses all points available in each frame's data. \n* The points for hands and pose were connected using information from mediapipe. \n* The face is simply shown as dots on their respective coordinates. \n* Link to pose article: https://google.github.io/mediapipe/solutions/pose.html\n* Link to hands article: https://google.github.io/mediapipe/solutions/hands.html","metadata":{}},{"cell_type":"code","source":"def animation_frame(f):\n    frame = sign[sign.frame==f]\n    left = frame[frame.type=='left_hand']\n    right = frame[frame.type=='right_hand']\n    pose = frame[frame.type=='pose']\n    face = frame[frame.type=='face'][['x', 'y']].values\n    lx, ly = get_hand_points(left)\n    rx, ry = get_hand_points(right)\n    px, py = get_pose_points(pose)\n    ax.clear()\n    ax.plot(face[:,0], face[:,1], '.')\n    for i in range(len(lx)):\n        ax.plot(lx[i], ly[i])\n    for i in range(len(rx)):\n        ax.plot(rx[i], ry[i])\n    for i in range(len(px)):\n        ax.plot(px[i], py[i])\n    plt.xlim(xmin, xmax)\n    plt.ylim(ymin, ymax)\n        \nprint(f\"The sign being shown here is: {train[train.path==f'{path_to_sign}'].sign.values[0]}\")\n\n## These values set the limits on the graph to stabilize the video\nxmin = sign.x.min() - 0.2\nxmax = sign.x.max() + 0.2\nymin = sign.y.min() - 0.2\nymax = sign.y.max() + 0.2\n\nfig, ax = plt.subplots()\nl, = ax.plot([], [])\nanimation = FuncAnimation(fig, func=animation_frame, frames=sign.frame.unique())\n\nHTML(animation.to_html5_video())","metadata":{"execution":{"iopub.status.busy":"2023-02-24T17:32:50.644531Z","iopub.execute_input":"2023-02-24T17:32:50.644963Z","iopub.status.idle":"2023-02-24T17:32:53.236756Z","shell.execute_reply.started":"2023-02-24T17:32:50.644931Z","shell.execute_reply":"2023-02-24T17:32:53.235939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### This is another animation, this time just of the left hand. \n* Some frames are missing data, so these are dropped to make the video smooth. \n* Some files have more missing data than others, and you may want to keep all frames for continuity.\n* Reload the sign file if you wish to re-run the full body animation with all the data again. ","metadata":{}},{"cell_type":"code","source":"sign = sign[sign.type=='left_hand'].dropna()\ndef animation_frame(f):\n    frame = sign[sign.frame==f]\n    left = frame[frame.type=='left_hand']\n    lx, ly = get_hand_points(left)\n    ax.clear()\n    for i in range(len(lx)):\n        ax.plot(lx[i], ly[i])\n    plt.xlim(xmin, xmax)\n    plt.ylim(ymin, ymax)\n\n        \nprint(f\"The sign being shown here is: {train[train.path==f'{path_to_sign}'].sign.values[0]}\")\n\n## These values set the limits on the graph to stabilize the video\nxmin = sign.x.min() - 0.2\nxmax = sign.x.max() + 0.2\nymin = sign.y.min() - 0.2\nymax = sign.y.max() + 0.2\n\nfig, ax = plt.subplots()\nl, = ax.plot([], [])\nanimation = FuncAnimation(fig, func=animation_frame, frames=sign.frame.unique())\n\nHTML(animation.to_html5_video())","metadata":{"execution":{"iopub.status.busy":"2023-02-24T17:32:53.340851Z","iopub.execute_input":"2023-02-24T17:32:53.341181Z","iopub.status.idle":"2023-02-24T17:32:54.825878Z","shell.execute_reply.started":"2023-02-24T17:32:53.341157Z","shell.execute_reply":"2023-02-24T17:32:54.824929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}