{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-04T20:03:41.182812Z","iopub.execute_input":"2023-03-04T20:03:41.183886Z","iopub.status.idle":"2023-03-04T20:03:41.218962Z","shell.execute_reply.started":"2023-03-04T20:03:41.183750Z","shell.execute_reply":"2023-03-04T20:03:41.217650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Importing required libraries\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom mpl_toolkits.mplot3d import Axes3D","metadata":{"execution":{"iopub.status.busy":"2023-03-04T20:07:33.910964Z","iopub.execute_input":"2023-03-04T20:07:33.911502Z","iopub.status.idle":"2023-03-04T20:07:33.919430Z","shell.execute_reply.started":"2023-03-04T20:07:33.911450Z","shell.execute_reply":"2023-03-04T20:07:33.917921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/asl-signs/train.csv')\nprint(f'Training data shape is: {train_df.shape}')\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-04T20:03:42.494556Z","iopub.execute_input":"2023-03-04T20:03:42.495244Z","iopub.status.idle":"2023-03-04T20:03:42.737459Z","shell.execute_reply.started":"2023-03-04T20:03:42.495208Z","shell.execute_reply":"2023-03-04T20:03:42.736308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Participant_id","metadata":{}},{"cell_type":"code","source":"print(f\"Number of Unique Participants:                  {train_df['participant_id'].nunique()}\")\nprint(f\"Average Number of Rows Per Participant:         {train_df.groupby('participant_id').size().mean():.2f}\")\nprint(f\"Standard Deviation in Counts Per Participant:   {train_df.groupby('participant_id').size().std():.2f}\")\nprint(f\"Minimum Number of Examples For One Participant: {train_df.groupby('participant_id').size().min()}\")\nprint(f\"Maximum Number of Examples For One Participant  {train_df.groupby('participant_id').size().max()}\")","metadata":{"execution":{"iopub.status.busy":"2023-03-04T20:03:46.639169Z","iopub.execute_input":"2023-03-04T20:03:46.639601Z","iopub.status.idle":"2023-03-04T20:03:46.668099Z","shell.execute_reply.started":"2023-03-04T20:03:46.639566Z","shell.execute_reply":"2023-03-04T20:03:46.666726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Sequence_id","metadata":{"execution":{"iopub.status.busy":"2023-03-04T02:42:25.767840Z","iopub.execute_input":"2023-03-04T02:42:25.768295Z","iopub.status.idle":"2023-03-04T02:42:25.774057Z","shell.execute_reply.started":"2023-03-04T02:42:25.768253Z","shell.execute_reply":"2023-03-04T02:42:25.772759Z"}}},{"cell_type":"code","source":"train_df['sequence_id'].astype(str).describe()","metadata":{"execution":{"iopub.status.busy":"2023-03-04T20:03:48.449779Z","iopub.execute_input":"2023-03-04T20:03:48.451017Z","iopub.status.idle":"2023-03-04T20:03:48.582387Z","shell.execute_reply.started":"2023-03-04T20:03:48.450970Z","shell.execute_reply":"2023-03-04T20:03:48.580820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Each sequence id represents a unique video clip. \\\nSequence id is the same as the path in parquet file.","metadata":{}},{"cell_type":"markdown","source":"## Label","metadata":{}},{"cell_type":"code","source":"print(f\"Number of unique label: {train_df['sign'].nunique()}\")","metadata":{"execution":{"iopub.status.busy":"2023-03-04T20:03:50.668368Z","iopub.execute_input":"2023-03-04T20:03:50.668772Z","iopub.status.idle":"2023-03-04T20:03:50.682412Z","shell.execute_reply.started":"2023-03-04T20:03:50.668738Z","shell.execute_reply":"2023-03-04T20:03:50.681080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_counts = train_df.sign.value_counts().to_frame().reset_index()\nlabel_counts.columns = ['label','count']\nplt.figure(figsize=(8,38))\n#plt.barh('label', 'count', data=label_counts, height=0.5)\nsns.barplot(y=label_counts['label'], x=label_counts['count'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-04T20:03:51.303849Z","iopub.execute_input":"2023-03-04T20:03:51.304327Z","iopub.status.idle":"2023-03-04T20:03:54.902223Z","shell.execute_reply.started":"2023-03-04T20:03:51.304285Z","shell.execute_reply":"2023-03-04T20:03:54.901239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It's pretty balanced","metadata":{}},{"cell_type":"markdown","source":"## Take a look at one sequence","metadata":{}},{"cell_type":"code","source":"sample = pd.read_parquet('/kaggle/input/asl-signs/train_landmark_files/16069/100015657.parquet')\nprint(sample.shape)\nsample.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-03-04T20:04:05.339530Z","iopub.execute_input":"2023-03-04T20:04:05.340027Z","iopub.status.idle":"2023-03-04T20:04:05.523362Z","shell.execute_reply.started":"2023-03-04T20:04:05.339979Z","shell.execute_reply":"2023-03-04T20:04:05.522332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample.describe()","metadata":{"execution":{"iopub.status.busy":"2023-03-04T20:04:06.895115Z","iopub.execute_input":"2023-03-04T20:04:06.896073Z","iopub.status.idle":"2023-03-04T20:04:06.945394Z","shell.execute_reply.started":"2023-03-04T20:04:06.896018Z","shell.execute_reply":"2023-03-04T20:04:06.944000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"There are {sample.frame.max()-sample.frame.min()} frames in this sequence.\")","metadata":{"execution":{"iopub.status.busy":"2023-03-04T20:04:08.666018Z","iopub.execute_input":"2023-03-04T20:04:08.666466Z","iopub.status.idle":"2023-03-04T20:04:08.673905Z","shell.execute_reply.started":"2023-03-04T20:04:08.666427Z","shell.execute_reply":"2023-03-04T20:04:08.672540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample.type.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-03-04T20:04:11.844834Z","iopub.execute_input":"2023-03-04T20:04:11.845605Z","iopub.status.idle":"2023-03-04T20:04:11.857236Z","shell.execute_reply.started":"2023-03-04T20:04:11.845560Z","shell.execute_reply":"2023-03-04T20:04:11.855799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are 4 parts of body being captured.","metadata":{}},{"cell_type":"markdown","source":"## Visualization","metadata":{}},{"cell_type":"markdown","source":"### Hand","metadata":{}},{"cell_type":"code","source":"edges = [(0,1),(1,2),(2,3),(3,4),(0,5),(0,17),(5,6),(6,7),(7,8),(5,9),(9,10),(10,11),(11,12),\n         (9,13),(13,14),(14,15),(15,16),(13,17),(17,18),(18,19),(19,20)]\n\ndef plot_frame(df, frame_id, ax):\n    df = df[df.frame == frame_id].sort_values(['landmark_index'])\n    x = list(df.x)\n    y = list(df.y)\n    \n    ax.scatter(df.x, df.y, color='dodgerblue')\n    for i in range(len(x)):\n        ax.text(x[i], y[i], str(i))\n        \n    for edge in edges:\n        ax.plot([x[edge[0]], x[edge[1]]], [y[edge[0]], y[edge[1]]], color='salmon')\n        ax.set_xlabel(f\"Frame {frame_id}\")\n        ax.set_xticks([])\n        ax.set_yticks([])\n        ax.set_xticklabels([])\n        ax.set_yticklabels([])\n\n    \ndef plot_frame_seq(df, frame_range, n_frames):\n    frames = np.linspace(frame_range[0],frame_range[1],n_frames, dtype = int, endpoint=True)\n    fig, ax = plt.subplots(n_frames, 1, figsize=(6,30))\n    for i in range(n_frames):\n        plot_frame(df, frames[i], ax[i])\n        \n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-04T20:04:52.132958Z","iopub.execute_input":"2023-03-04T20:04:52.133415Z","iopub.status.idle":"2023-03-04T20:04:52.149127Z","shell.execute_reply.started":"2023-03-04T20:04:52.133370Z","shell.execute_reply":"2023-03-04T20:04:52.147895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Left hand\")\nplot_frame_seq(sample[sample.type=='left_hand'], (178,186), 5)","metadata":{"execution":{"iopub.status.busy":"2023-03-04T20:04:52.343569Z","iopub.execute_input":"2023-03-04T20:04:52.344293Z","iopub.status.idle":"2023-03-04T20:04:53.199409Z","shell.execute_reply.started":"2023-03-04T20:04:52.344214Z","shell.execute_reply":"2023-03-04T20:04:53.198132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample[sample.type=='right_hand'].x.sum()","metadata":{"execution":{"iopub.status.busy":"2023-03-04T20:04:53.201456Z","iopub.execute_input":"2023-03-04T20:04:53.202492Z","iopub.status.idle":"2023-03-04T20:04:53.220914Z","shell.execute_reply.started":"2023-03-04T20:04:53.202444Z","shell.execute_reply":"2023-03-04T20:04:53.219710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"No right hand movement in this sequence.","metadata":{}},{"cell_type":"markdown","source":"### Pose","metadata":{}},{"cell_type":"code","source":"def add_init_c(start, end, hand):\n    return (\n        pd.concat([hand['x'][start:start+1], hand['x'][end[0]:end[1]]]), \n        pd.concat([hand['y'][start:start+1], hand['y'][end[0]:end[1]]]), \n        pd.concat([hand['z'][start:start+1], hand['z'][end[0]:end[1]]])\n        )\n\ndef plot_pose(pose, td):\n    fig, ax = plt.subplots()\n    fig.set_size_inches(6, 6)\n\n    if td:\n        ax = Axes3D(fig, auto_add_to_figure=False)\n        fig.add_axes(ax)\n\n    ind = [[0, [1, 4]], [3, [7, 8]], [0, [4, 7]], [6, [8, 9]], [9, [10, 11]], [11, [12, 13]], [12, [14, 15]], [14, [16, 17]], \n           [16, [22, 23]], [16, [18, 19]], [16, [20, 21]], [18, [20, 21]], [11, [13, 14]], [13, [15, 16]], [15, [21, 22]], [15, [19, 20]],\n           [15, [17, 18]], [17, [19, 20]], [12, [24, 25]], [24, [26, 27]], [26, [28, 29]], [28, [30, 31]], [30, [32, 33]], [28, [32, 33]], \n           [11, [23, 24]], [23, [25, 26]], [25, [27, 28]], [27, [29, 30]], [29, [31, 32]], [27, [31, 32]], [23, [24, 25]]]\n\n    for i, k in ind: \n        x, y, z = add_init_c(i, k, pose)\n        if td:\n            ax.plot(x, -1*y, z)\n        else:\n            ax.plot(x, -1*y)","metadata":{"execution":{"iopub.status.busy":"2023-03-04T20:06:14.313345Z","iopub.execute_input":"2023-03-04T20:06:14.313820Z","iopub.status.idle":"2023-03-04T20:06:14.329862Z","shell.execute_reply.started":"2023-03-04T20:06:14.313778Z","shell.execute_reply":"2023-03-04T20:06:14.328508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_pose(sample.loc[sample.type=='pose'][:33], True)","metadata":{"execution":{"iopub.status.busy":"2023-03-04T20:13:39.167900Z","iopub.execute_input":"2023-03-04T20:13:39.168440Z","iopub.status.idle":"2023-03-04T20:13:39.757183Z","shell.execute_reply.started":"2023-03-04T20:13:39.168372Z","shell.execute_reply":"2023-03-04T20:13:39.755841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Pose in 5 consecutive frames**","metadata":{}},{"cell_type":"code","source":"plot_pose(sample.loc[sample.type=='pose'][:33], False)\nplot_pose(sample.loc[sample.type=='pose'][33: 66], False)\nplot_pose(sample.loc[sample.type=='pose'][66: 99], False)\nplot_pose(sample.loc[sample.type=='pose'][99: 132], False)\nplot_pose(sample.loc[sample.type=='pose'][132: 165], False)","metadata":{"execution":{"iopub.status.busy":"2023-03-04T20:15:05.981123Z","iopub.execute_input":"2023-03-04T20:15:05.981545Z","iopub.status.idle":"2023-03-04T20:15:07.393530Z","shell.execute_reply.started":"2023-03-04T20:15:05.981506Z","shell.execute_reply":"2023-03-04T20:15:07.392136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"First frame in this sequence:\")\nplot_pose(sample.loc[sample.type=='pose'][: 33], False)\n\nprint(\"Middle frame in this sequence:\")\nplot_pose(sample.loc[sample.type=='pose'][33*52: 33*53], False)\n\nprint(\"Last frame in this sequence:\")\nplot_pose(sample.loc[sample.type=='pose'][33*104: 33*106], False)","metadata":{"execution":{"iopub.status.busy":"2023-03-04T20:27:41.634288Z","iopub.execute_input":"2023-03-04T20:27:41.634701Z","iopub.status.idle":"2023-03-04T20:27:42.335583Z","shell.execute_reply.started":"2023-03-04T20:27:41.634666Z","shell.execute_reply":"2023-03-04T20:27:42.334307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Pose rarely changes in each sequence.","metadata":{}},{"cell_type":"markdown","source":"### Face","metadata":{}},{"cell_type":"code","source":"def plot_face(face, td):\n    fig, ax = plt.subplots()\n    fig.set_size_inches(6, 6)\n\n    if td:\n        ax = Axes3D(fig, auto_add_to_figure=False)\n        fig.add_axes(ax)\n\n    if td:\n        ax.scatter(face['x'], -1*face['y'], face['z'])\n    else:\n        ax.scatter(face['x'], -1*face['y'])","metadata":{"execution":{"iopub.status.busy":"2023-03-04T20:16:00.204338Z","iopub.execute_input":"2023-03-04T20:16:00.204795Z","iopub.status.idle":"2023-03-04T20:16:00.213332Z","shell.execute_reply.started":"2023-03-04T20:16:00.204754Z","shell.execute_reply":"2023-03-04T20:16:00.211776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_face(sample.loc[sample.type=='face'][: 468], True)","metadata":{"execution":{"iopub.status.busy":"2023-03-04T20:16:44.353631Z","iopub.execute_input":"2023-03-04T20:16:44.354055Z","iopub.status.idle":"2023-03-04T20:16:44.808064Z","shell.execute_reply.started":"2023-03-04T20:16:44.354018Z","shell.execute_reply":"2023-03-04T20:16:44.806800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Face in 5 consecutive frames**","metadata":{}},{"cell_type":"code","source":"plot_face(sample.loc[sample.type=='face'][: 468], False)\nplot_face(sample.loc[sample.type=='face'][468: 468*2], False)\nplot_face(sample.loc[sample.type=='face'][468*2: 468*3], False)\nplot_face(sample.loc[sample.type=='face'][468*3: 468*4], False)\nplot_face(sample.loc[sample.type=='face'][468*4: 468*5], False)","metadata":{"execution":{"iopub.status.busy":"2023-03-04T20:19:57.503143Z","iopub.execute_input":"2023-03-04T20:19:57.503649Z","iopub.status.idle":"2023-03-04T20:19:58.767113Z","shell.execute_reply.started":"2023-03-04T20:19:57.503610Z","shell.execute_reply":"2023-03-04T20:19:58.765941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"First frame in this sequence:\")\nplot_face(sample.loc[sample.type=='face'][: 468], False)\n\nprint(\"Middle frame in this sequence:\")\nplot_face(sample.loc[sample.type=='face'][468*52: 468*53], False)\n\nprint(\"Last frame in this sequence:\")\nplot_face(sample.loc[sample.type=='face'][468*104: 468*106], False)","metadata":{"execution":{"iopub.status.busy":"2023-03-04T20:23:18.143712Z","iopub.execute_input":"2023-03-04T20:23:18.144191Z","iopub.status.idle":"2023-03-04T20:23:18.722105Z","shell.execute_reply.started":"2023-03-04T20:23:18.144148Z","shell.execute_reply":"2023-03-04T20:23:18.720864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Face rarely changes in each sequence.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Sample labels","metadata":{}},{"cell_type":"code","source":"def get_details_per_sign(sign):\n    train_sign_sample = train_df[train_df['sign'] == sign]\n    n_frames = 0\n    n_left_hand = 0\n    n_right_hand = 0\n    n_face = 0\n    n_both_hands = 0\n    for _,row in train_sign_sample.iterrows():\n        df = pd.read_parquet(os.path.join(\"/kaggle/input/asl-signs\", row.path))\n        n_frames += df['frame'].nunique()\n        n_left_hand += np.sum(df[(df['type'] == 'left_hand') & (df['landmark_index'] == 0)]['x'].isnull() == False)\n        n_right_hand += np.sum(df[(df['type'] == 'right_hand') & (df['landmark_index'] == 0)]['x'].isnull() == False)\n        n_face += np.sum(df[(df['type'] == 'face') & (df['landmark_index'] == 0)]['x'].isnull() == False)\n        \n        df_both_hands = df[(df['type'] == 'left_hand') & (df['landmark_index'] == 0)].merge(\\\n                            df[(df['type'] == 'right_hand') & (df['landmark_index'] == 0)], on='frame', suffixes=('_left', '_right'))\n        n_both_hands += df_both_hands[(df_both_hands['x_left'].isnull() == False) &\\\n                                             (df_both_hands['x_right'].isnull() == False)]['frame'].count()\n            \n    return n_frames/len(train_sign_sample), n_left_hand/n_frames, n_right_hand/n_frames, n_both_hands/n_frames, n_face/n_frames","metadata":{"execution":{"iopub.status.busy":"2023-03-04T03:23:45.127036Z","iopub.execute_input":"2023-03-04T03:23:45.127395Z","iopub.status.idle":"2023-03-04T03:23:45.141450Z","shell.execute_reply.started":"2023-03-04T03:23:45.127362Z","shell.execute_reply":"2023-03-04T03:23:45.139975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for sign in ['cloud', 'thankyou', 'donkey', 'because', 'yellow', 'icecream']:\n    total_frames, pct_left, pct_right, pct_both, pct_face = get_details_per_sign(sign)\n    print(\"=\"*20, f\"{sign}\", \"=\"*20)\n    print(f\"Average Number of Frames per Sequence = {total_frames}\")\n    print(f\"Percent of Frames in which a body part exists: Left Hand: {pct_left*100:.02f} %, Right Hand: {pct_right*100:.02f} %, Both Hands: {pct_both*100:.02f} %, Face: {pct_face*100:.02f} %\")\n    print()","metadata":{"execution":{"iopub.status.busy":"2023-03-04T03:23:45.143571Z","iopub.execute_input":"2023-03-04T03:23:45.144040Z","iopub.status.idle":"2023-03-04T03:25:01.717090Z","shell.execute_reply.started":"2023-03-04T03:23:45.143992Z","shell.execute_reply":"2023-03-04T03:25:01.716080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Average frames in a sequence: 30-40.\\\nLeft hand is more used than right hand.\\\nBoth hands are rarely used.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}