{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# <h><center>🤟ASL Recognition Competition✌️👊</center></h>\n\n## ***The goal of this competition is to classify isolated American Sign Language (ASL) signs. You will create a TensorFlow Lite model trained on labeled landmark data extracted using the MediaPipe Holistic Solution.***\n\n## ***Your work may improve the ability of PopSign* to help relatives of deaf children learn basic signs and communicate better with their loved ones.***\n\n<img src='https://media.wired.com/photos/624764ed48046e8802c9c5b8/3:2/w_2400,h_1600,c_limit/Learn-ASL-Online-Gear-495674588.jpg'>\n\n## **Now, First I am exploring my self in eda & visualization of sign language using mediapipe landmarks**\n\n<img src='https://www.dummies.com/wp-content/uploads/321607.image0.jpg'>\n","metadata":{}},{"cell_type":"markdown","source":"# **Import Necessary library**","metadata":{}},{"cell_type":"code","source":"import os\nimport sys\nimport cv2\nimport imageio\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport seaborn as sns\nimport plotly.express as px\nfrom wordcloud import WordCloud\nimport matplotlib.pyplot as plt\nfrom matplotlib.pyplot import imshow\nimport matplotlib.animation as animation\nfrom matplotlib.animation import FuncAnimation\nfrom IPython.display import HTML","metadata":{"execution":{"iopub.status.busy":"2023-02-26T13:14:21.416712Z","iopub.execute_input":"2023-02-26T13:14:21.417079Z","iopub.status.idle":"2023-02-26T13:14:21.423561Z","shell.execute_reply.started":"2023-02-26T13:14:21.417049Z","shell.execute_reply":"2023-02-26T13:14:21.422202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install plotly","metadata":{"execution":{"iopub.status.busy":"2023-02-26T13:14:21.425841Z","iopub.execute_input":"2023-02-26T13:14:21.426569Z","iopub.status.idle":"2023-02-26T13:14:32.218536Z","shell.execute_reply.started":"2023-02-26T13:14:21.426511Z","shell.execute_reply":"2023-02-26T13:14:32.217095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Visualize the path and set your required**","metadata":{}},{"cell_type":"code","source":"!ls '/kaggle/input/asl-signs'","metadata":{"execution":{"iopub.status.busy":"2023-02-26T13:14:32.222047Z","iopub.execute_input":"2023-02-26T13:14:32.222446Z","iopub.status.idle":"2023-02-26T13:14:33.180889Z","shell.execute_reply.started":"2023-02-26T13:14:32.222403Z","shell.execute_reply":"2023-02-26T13:14:33.179621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dir = '/kaggle/input/asl-signs'\ntrain = pd.read_csv(f'{dir}/train.csv')\nto_prediction_index_map = dir+'sign_to_prediction_index_map.json'\nto_folder_with_parquet = dir+'train_landmark_files'","metadata":{"execution":{"iopub.status.busy":"2023-02-26T13:14:33.184685Z","iopub.execute_input":"2023-02-26T13:14:33.185055Z","iopub.status.idle":"2023-02-26T13:14:33.402641Z","shell.execute_reply.started":"2023-02-26T13:14:33.185025Z","shell.execute_reply":"2023-02-26T13:14:33.401584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Change this directory to any file\npath_to_sign = 'train_landmark_files/22343/1000638205.parquet'\nsign = pd.read_parquet(f'{dir}/{path_to_sign}')\nsign","metadata":{"execution":{"iopub.status.busy":"2023-02-26T13:14:33.404024Z","iopub.execute_input":"2023-02-26T13:14:33.404423Z","iopub.status.idle":"2023-02-26T13:14:33.519898Z","shell.execute_reply.started":"2023-02-26T13:14:33.404384Z","shell.execute_reply":"2023-02-26T13:14:33.518712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sign.y = sign.y * -1\nprint(\"sign.y column:\",sign.y)","metadata":{"execution":{"iopub.status.busy":"2023-02-26T13:14:33.521361Z","iopub.execute_input":"2023-02-26T13:14:33.522516Z","iopub.status.idle":"2023-02-26T13:14:33.532372Z","shell.execute_reply.started":"2023-02-26T13:14:33.522465Z","shell.execute_reply":"2023-02-26T13:14:33.531069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **How to know the hand keypoints**\n\n<img src='https://external-content.duckduckgo.com/iu/?u=https%3A%2F%2Fi2.wp.com%2Ftechvidvan.com%2Ftutorials%2Fwp-content%2Fuploads%2Fsites%2F2%2F2021%2F07%2Fhand-landmarks.jpg%3Fresize%3D1440%252C537%26ssl%3D1&f=1&nofb=1&ipt=95bc6d1d5f6357985346e3f383faa6ba98b3225bc9f0aace0534792c81e6a601&ipo=images'>","metadata":{}},{"cell_type":"code","source":"#connecting coordinates values with connecting lines for hands and pose\n\ndef get_hand_points(hand):\n    '''Get hand points only'''\n    x = [[hand.iloc[0].x, hand.iloc[1].x, hand.iloc[2].x, hand.iloc[3].x, hand.iloc[4].x], # Thumb\n         [hand.iloc[5].x, hand.iloc[6].x, hand.iloc[7].x, hand.iloc[8].x], # Index\n         [hand.iloc[9].x, hand.iloc[10].x, hand.iloc[11].x, hand.iloc[12].x], #Middle\n         [hand.iloc[13].x, hand.iloc[14].x, hand.iloc[15].x, hand.iloc[16].x], #Ring\n         [hand.iloc[17].x, hand.iloc[18].x, hand.iloc[19].x, hand.iloc[20].x], #Pinky\n         [hand.iloc[0].x, hand.iloc[5].x, hand.iloc[9].x, hand.iloc[13].x, hand.iloc[17].x, hand.iloc[0].x]] # remain part of hand (without finger parts)\n\n    y = [[hand.iloc[0].y, hand.iloc[1].y, hand.iloc[2].y, hand.iloc[3].y, hand.iloc[4].y],  #Thumb\n         [hand.iloc[5].y, hand.iloc[6].y, hand.iloc[7].y, hand.iloc[8].y], # Index\n         [hand.iloc[9].y, hand.iloc[10].y, hand.iloc[11].y, hand.iloc[12].y], \n         [hand.iloc[13].y, hand.iloc[14].y, hand.iloc[15].y, hand.iloc[16].y], \n         [hand.iloc[17].y, hand.iloc[18].y, hand.iloc[19].y, hand.iloc[20].y], \n         [hand.iloc[0].y, hand.iloc[5].y, hand.iloc[9].y, hand.iloc[13].y, hand.iloc[17].y, hand.iloc[0].y]] \n    return x, y","metadata":{"execution":{"iopub.status.busy":"2023-02-26T13:14:33.534074Z","iopub.execute_input":"2023-02-26T13:14:33.535056Z","iopub.status.idle":"2023-02-26T13:14:33.550597Z","shell.execute_reply.started":"2023-02-26T13:14:33.535016Z","shell.execute_reply":"2023-02-26T13:14:33.549717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Set the particular pose using this above keypoints**","metadata":{}},{"cell_type":"code","source":"def get_pose_points(pose):\n    \n    '''Get pose points in particular sign's '''\n    \n    x = [[pose.iloc[8].x, pose.iloc[6].x, pose.iloc[5].x, pose.iloc[4].x, pose.iloc[0].x, pose.iloc[1].x, pose.iloc[2].x, pose.iloc[3].x, pose.iloc[7].x], \n         [pose.iloc[10].x, pose.iloc[9].x], \n         [pose.iloc[22].x, pose.iloc[16].x, pose.iloc[20].x, pose.iloc[18].x, pose.iloc[16].x, pose.iloc[14].x, pose.iloc[12].x, \n          pose.iloc[11].x, pose.iloc[13].x, pose.iloc[15].x, pose.iloc[17].x, pose.iloc[19].x, pose.iloc[15].x, pose.iloc[21].x], \n         [pose.iloc[12].x, pose.iloc[24].x, pose.iloc[26].x, pose.iloc[28].x, pose.iloc[30].x, pose.iloc[32].x, pose.iloc[28].x], \n         [pose.iloc[11].x, pose.iloc[23].x, pose.iloc[25].x, pose.iloc[27].x, pose.iloc[29].x, pose.iloc[31].x, pose.iloc[27].x], \n         [pose.iloc[24].x, pose.iloc[23].x]\n        ]\n\n    y = [[pose.iloc[8].y, pose.iloc[6].y, pose.iloc[5].y, pose.iloc[4].y, pose.iloc[0].y, pose.iloc[1].y, pose.iloc[2].y, pose.iloc[3].y, pose.iloc[7].y], \n         [pose.iloc[10].y, pose.iloc[9].y], \n         [pose.iloc[22].y, pose.iloc[16].y, pose.iloc[20].y, pose.iloc[18].y, pose.iloc[16].y, pose.iloc[14].y, pose.iloc[12].y, \n          pose.iloc[11].y, pose.iloc[13].y, pose.iloc[15].y, pose.iloc[17].y, pose.iloc[19].y, pose.iloc[15].y, pose.iloc[21].y], \n         [pose.iloc[12].y, pose.iloc[24].y, pose.iloc[26].y, pose.iloc[28].y, pose.iloc[30].y, pose.iloc[32].y, pose.iloc[28].y], \n         [pose.iloc[11].y, pose.iloc[23].y, pose.iloc[25].y, pose.iloc[27].y, pose.iloc[29].y, pose.iloc[31].y, pose.iloc[27].y], \n         [pose.iloc[24].y, pose.iloc[23].y]\n        ]\n    return x, y\n","metadata":{"execution":{"iopub.status.busy":"2023-02-26T13:14:33.552148Z","iopub.execute_input":"2023-02-26T13:14:33.552839Z","iopub.status.idle":"2023-02-26T13:14:33.570320Z","shell.execute_reply.started":"2023-02-26T13:14:33.552793Z","shell.execute_reply":"2023-02-26T13:14:33.569502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Animation visualize - Very interesting**","metadata":{}},{"cell_type":"code","source":"def animation_frame(f):\n    frame = sign[sign.frame==f]\n    left = frame[frame.type=='left_hand']\n    right = frame[frame.type=='right_hand']\n    pose = frame[frame.type=='pose']\n    face = frame[frame.type=='face'][['x', 'y']].values\n    lx, ly = get_hand_points(left)\n    rx, ry = get_hand_points(right)\n    px, py = get_pose_points(pose)\n    ax.clear()\n    ax.plot(face[:,0], face[:,1], '.')\n    for i in range(len(lx)):\n        ax.plot(lx[i], ly[i])\n    for i in range(len(rx)):\n        ax.plot(rx[i], ry[i])\n    for i in range(len(px)):\n        ax.plot(px[i], py[i])\n    plt.xlim(xmin, xmax)\n    plt.ylim(ymin, ymax)\n        \nprint(f\"The sign being shown here is: {train[train.path==f'{path_to_sign}'].sign.values[0]}\")\n","metadata":{"execution":{"iopub.status.busy":"2023-02-26T13:14:33.573779Z","iopub.execute_input":"2023-02-26T13:14:33.574526Z","iopub.status.idle":"2023-02-26T13:14:33.592287Z","shell.execute_reply.started":"2023-02-26T13:14:33.574445Z","shell.execute_reply":"2023-02-26T13:14:33.591258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## These values set the limits on the graph to stabilize the video\nxmin = sign.x.min() - 0.2\nxmax = sign.x.max() + 0.2\nymin = sign.y.min() - 0.2\nymax = sign.y.max() + 0.2\n","metadata":{"execution":{"iopub.status.busy":"2023-02-26T13:14:33.593778Z","iopub.execute_input":"2023-02-26T13:14:33.594442Z","iopub.status.idle":"2023-02-26T13:14:33.604231Z","shell.execute_reply.started":"2023-02-26T13:14:33.594405Z","shell.execute_reply":"2023-02-26T13:14:33.603485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots()\nl, = ax.plot([], [])\nanimation = FuncAnimation(fig, func=animation_frame, frames=sign.frame.unique())\n\nHTML(animation.to_html5_video())\n","metadata":{"execution":{"iopub.status.busy":"2023-02-26T13:14:33.605646Z","iopub.execute_input":"2023-02-26T13:14:33.606266Z","iopub.status.idle":"2023-02-26T13:14:36.627136Z","shell.execute_reply.started":"2023-02-26T13:14:33.606230Z","shell.execute_reply":"2023-02-26T13:14:36.626057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Particular part of hand visualization**","metadata":{}},{"cell_type":"code","source":"# left hand\nsign = sign[sign.type=='left_hand'].dropna()\ndef animation_frame(f):\n    frame = sign[sign.frame==f]\n    left = frame[frame.type=='left_hand']\n    lx, ly = get_hand_points(left)\n    ax.clear()\n    for i in range(len(lx)):\n        ax.plot(lx[i], ly[i])\n    plt.xlim(xmin, xmax)\n    plt.ylim(ymin, ymax)\n\n        \nprint(f\"The sign being shown here is: {train[train.path==f'{path_to_sign}'].sign.values[0]}\")\n\n## These values set the limits on the graph to stabilize the video\nxmin = sign.x.min() - 0.2\nxmax = sign.x.max() + 0.2\nymin = sign.y.min() - 0.2\nymax = sign.y.max() + 0.2\n\nfig, ax = plt.subplots()\nl, = ax.plot([], [])\nanimation = FuncAnimation(fig, func=animation_frame, frames=sign.frame.unique())\n\nHTML(animation.to_html5_video())","metadata":{"execution":{"iopub.status.busy":"2023-02-26T13:14:36.631884Z","iopub.execute_input":"2023-02-26T13:14:36.634214Z","iopub.status.idle":"2023-02-26T13:14:37.892436Z","shell.execute_reply.started":"2023-02-26T13:14:36.634172Z","shell.execute_reply":"2023-02-26T13:14:37.891308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Different approach**","metadata":{}},{"cell_type":"code","source":"train.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-02-26T13:15:21.226095Z","iopub.execute_input":"2023-02-26T13:15:21.226732Z","iopub.status.idle":"2023-02-26T13:15:21.247749Z","shell.execute_reply.started":"2023-02-26T13:15:21.226682Z","shell.execute_reply":"2023-02-26T13:15:21.246766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(data=train, x='sign')\nplt.title('distribution of target variable')\nplt.xlabel('sign')\nplt.ylabel('count')\nplt.xticks([])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-26T13:16:38.975600Z","iopub.execute_input":"2023-02-26T13:16:38.975974Z","iopub.status.idle":"2023-02-26T13:16:39.990941Z","shell.execute_reply.started":"2023-02-26T13:16:38.975942Z","shell.execute_reply":"2023-02-26T13:16:39.988713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text = ''.join(train['sign'])\nwordcloud = WordCloud(width=600, height=300, background_color='white', colormap='tab20').generate(text)\nplt.figure(figsize=(10,8))\nplt.imshow(wordcloud, interpolation= 'bilinear')\nplt.axis('off')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-26T13:38:53.644114Z","iopub.execute_input":"2023-02-26T13:38:53.644629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sign = pd.read_parquet(f'{dir}/{path_to_sign}')\nsign","metadata":{"execution":{"iopub.status.busy":"2023-02-26T13:30:49.907755Z","iopub.execute_input":"2023-02-26T13:30:49.908814Z","iopub.status.idle":"2023-02-26T13:30:49.940642Z","shell.execute_reply.started":"2023-02-26T13:30:49.908761Z","shell.execute_reply":"2023-02-26T13:30:49.939350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for column in (\"frame\", \"type\"):\n    plt.subplots(figsize=(10, 5))\n    sns.countplot(x=sign[column]);\n    \n    plt.title(f\"Distribution by {column}\")","metadata":{"execution":{"iopub.status.busy":"2023-02-26T13:30:55.407201Z","iopub.execute_input":"2023-02-26T13:30:55.407791Z","iopub.status.idle":"2023-02-26T13:30:55.955657Z","shell.execute_reply.started":"2023-02-26T13:30:55.407748Z","shell.execute_reply":"2023-02-26T13:30:55.954606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"group_df = sign.groupby('frame')\ngroup_df","metadata":{"execution":{"iopub.status.busy":"2023-02-26T13:31:43.165813Z","iopub.execute_input":"2023-02-26T13:31:43.166752Z","iopub.status.idle":"2023-02-26T13:31:43.173938Z","shell.execute_reply.started":"2023-02-26T13:31:43.166698Z","shell.execute_reply":"2023-02-26T13:31:43.172893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\npx.scatter_3d(sign, x=\"x\", y=\"y\", z=\"z\", color=\"type\")","metadata":{"execution":{"iopub.status.busy":"2023-02-26T13:32:57.627019Z","iopub.execute_input":"2023-02-26T13:32:57.627431Z","iopub.status.idle":"2023-02-26T13:32:57.726832Z","shell.execute_reply.started":"2023-02-26T13:32:57.627384Z","shell.execute_reply":"2023-02-26T13:32:57.725867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def update_plot(frame_num):\n    plt.cla()\n\n    data = group_df.get_group(frame_num)\n\n    face_data = data[data['type'] == 'face']\n    left_hand_data = data[data['type'] == 'left_hand']\n    right_hand_data = data[data['type'] == 'right_hand']\n\n    plt.scatter(face_data['x'], face_data['y'], c=face_data['landmark_index'], cmap='gist_rainbow', vmin=0, vmax=67, s=100)\n    plt.title(f'Frame {frame_num}')\n\n    plt.scatter(left_hand_data['x'], left_hand_data['y'], c='blue', marker='o', s=100)\n\n    plt.scatter(right_hand_data['x'], right_hand_data['y'], c='red', marker='o', s=100)","metadata":{"execution":{"iopub.status.busy":"2023-02-26T13:36:07.066050Z","iopub.execute_input":"2023-02-26T13:36:07.066646Z","iopub.status.idle":"2023-02-26T13:36:07.074464Z","shell.execute_reply.started":"2023-02-26T13:36:07.066600Z","shell.execute_reply":"2023-02-26T13:36:07.073351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib.animation import FuncAnimation\nimport matplotlib.animation as animation\n\n\nfig = plt.figure()\nani = animation.FuncAnimation(fig, update_plot, frames=group_df.groups.keys(), repeat=True)\n\nani.save('animation.gif', writer='imagemagick')","metadata":{"execution":{"iopub.status.busy":"2023-02-26T13:36:18.116203Z","iopub.execute_input":"2023-02-26T13:36:18.116595Z","iopub.status.idle":"2023-02-26T13:36:24.563459Z","shell.execute_reply.started":"2023-02-26T13:36:18.116562Z","shell.execute_reply":"2023-02-26T13:36:24.562330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"animation = imageio.mimread(\"/kaggle/working/animation.gif\")\nfor frame in animation:\n    plt.imshow(frame)\n    plt.axis('off')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-26T13:37:44.516283Z","iopub.execute_input":"2023-02-26T13:37:44.517011Z","iopub.status.idle":"2023-02-26T13:37:47.578607Z","shell.execute_reply.started":"2023-02-26T13:37:44.516976Z","shell.execute_reply":"2023-02-26T13:37:47.577422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Full credits:\n\n1. https://www.kaggle.com/code/danielpeshkov/animated-data-visualization\n2. https://www.kaggle.com/code/asimple/eda-sign-language-recognition","metadata":{}}]}