{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nplt.style.use('seaborn-colorblind')\nfrom tqdm.notebook import tqdm\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n\n ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-05T05:16:17.461719Z","iopub.execute_input":"2023-04-05T05:16:17.462148Z","iopub.status.idle":"2023-04-05T05:16:18.177210Z","shell.execute_reply.started":"2023-04-05T05:16:17.462113Z","shell.execute_reply":"2023-04-05T05:16:18.176101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install nb_black\n","metadata":{"execution":{"iopub.status.busy":"2023-04-05T05:16:18.181206Z","iopub.execute_input":"2023-04-05T05:16:18.181639Z","iopub.status.idle":"2023-04-05T05:16:34.121526Z","shell.execute_reply.started":"2023-04-05T05:16:18.181601Z","shell.execute_reply":"2023-04-05T05:16:34.120381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"EXPLORATORY DATA ANALYSIS","metadata":{}},{"cell_type":"code","source":"base_dir='../input/asl-signs/'\ntrain=pd.read_csv(f'{base_dir}/train.csv')","metadata":{"execution":{"iopub.status.busy":"2023-04-05T05:16:34.124744Z","iopub.execute_input":"2023-04-05T05:16:34.125117Z","iopub.status.idle":"2023-04-05T05:16:34.341135Z","shell.execute_reply.started":"2023-04-05T05:16:34.125077Z","shell.execute_reply":"2023-04-05T05:16:34.339861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-05T05:16:34.343948Z","iopub.execute_input":"2023-04-05T05:16:34.344333Z","iopub.status.idle":"2023-04-05T05:16:34.375618Z","shell.execute_reply.started":"2023-04-05T05:16:34.344271Z","shell.execute_reply":"2023-04-05T05:16:34.374389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* There are 250 unique signs ranging from 299 to 415 examplses of each","metadata":{}},{"cell_type":"code","source":"train['sign'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-04-05T05:16:34.377276Z","iopub.execute_input":"2023-04-05T05:16:34.377878Z","iopub.status.idle":"2023-04-05T05:16:34.400076Z","shell.execute_reply.started":"2023-04-05T05:16:34.377829Z","shell.execute_reply":"2023-04-05T05:16:34.398340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['sign'].value_counts().head(30).sort_values().plot(kind='barh',figsize=(15,8))","metadata":{"execution":{"iopub.status.busy":"2023-04-05T05:16:34.401791Z","iopub.execute_input":"2023-04-05T05:16:34.402209Z","iopub.status.idle":"2023-04-05T05:16:34.942687Z","shell.execute_reply.started":"2023-04-05T05:16:34.402166Z","shell.execute_reply":"2023-04-05T05:16:34.940040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['sign'].value_counts().tail(30).sort_values().plot(kind='barh',figsize=(15,8))","metadata":{"execution":{"iopub.status.busy":"2023-04-05T05:16:34.944114Z","iopub.execute_input":"2023-04-05T05:16:34.944501Z","iopub.status.idle":"2023-04-05T05:16:35.388111Z","shell.execute_reply.started":"2023-04-05T05:16:34.944466Z","shell.execute_reply":"2023-04-05T05:16:35.386745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.query('sign == \"listen\"')['path'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-04-05T05:16:35.389910Z","iopub.execute_input":"2023-04-05T05:16:35.390252Z","iopub.status.idle":"2023-04-05T05:16:35.410476Z","shell.execute_reply.started":"2023-04-05T05:16:35.390218Z","shell.execute_reply":"2023-04-05T05:16:35.409197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"PULLING AN EXAMPLE PARQUET FILE","metadata":{}},{"cell_type":"code","source":"ex_parquet=train.query('sign == \"listen\"')['path'].values[0]\nex_landmark=pd.read_parquet(f\"{base_dir}/{ex_parquet}\")\nex_landmark.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-05T05:16:35.411958Z","iopub.execute_input":"2023-04-05T05:16:35.412328Z","iopub.status.idle":"2023-04-05T05:16:35.560756Z","shell.execute_reply.started":"2023-04-05T05:16:35.412275Z","shell.execute_reply":"2023-04-05T05:16:35.559687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* For the listen sign , face , pose , left hand and right hand are included.\n* So there are 4 unique types.","metadata":{}},{"cell_type":"code","source":"ex_landmark['type'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-04-05T05:16:35.566917Z","iopub.execute_input":"2023-04-05T05:16:35.568667Z","iopub.status.idle":"2023-04-05T05:16:35.578096Z","shell.execute_reply.started":"2023-04-05T05:16:35.568613Z","shell.execute_reply":"2023-04-05T05:16:35.577120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* There are 6 unique frames each having 543 records","metadata":{}},{"cell_type":"code","source":"\nex_landmark['frame'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-04-05T05:16:35.579554Z","iopub.execute_input":"2023-04-05T05:16:35.580601Z","iopub.status.idle":"2023-04-05T05:16:35.589643Z","shell.execute_reply.started":"2023-04-05T05:16:35.580561Z","shell.execute_reply":"2023-04-05T05:16:35.588382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_frames = ex_landmark[\"frame\"].nunique()\nunique_types = ex_landmark[\"type\"].nunique()\ntypes_in_video = ex_landmark[\"type\"].unique()\nprint(\n    f\"The file has {unique_frames} unique frames and {unique_types} unique types: {types_in_video}\"\n)","metadata":{"execution":{"iopub.status.busy":"2023-04-05T05:16:35.592516Z","iopub.execute_input":"2023-04-05T05:16:35.594640Z","iopub.status.idle":"2023-04-05T05:16:35.603578Z","shell.execute_reply.started":"2023-04-05T05:16:35.594589Z","shell.execute_reply":"2023-04-05T05:16:35.602520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now, lets compare all the listen sign files.","metadata":{}},{"cell_type":"markdown","source":"* Number of frames is not consitent in files\n* Every file has the common 4 unique types {face, lefthand, pose, right hand}","metadata":{}},{"cell_type":"code","source":"listen_file=train.query('sign == \"listen\"')['path'].values\nfor idx,file in enumerate(listen_file):\n    files=pd.read_parquet(f\"{base_dir}/{file}\")\n    unique_frames = files[\"frame\"].nunique()\n    unique_types = files[\"type\"].nunique()\n    types_in_video = files[\"type\"].unique()\n    print(\n      f\"The file has {unique_frames} unique frames and {unique_types} unique types: {types_in_video}\"\n    )\n    if idx==30:\n        break\n    ","metadata":{"execution":{"iopub.status.busy":"2023-04-05T05:16:35.605023Z","iopub.execute_input":"2023-04-05T05:16:35.605659Z","iopub.status.idle":"2023-04-05T05:16:36.522113Z","shell.execute_reply.started":"2023-04-05T05:16:35.605611Z","shell.execute_reply":"2023-04-05T05:16:36.521077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Creating Meta Data for Training Dataset","metadata":{}},{"cell_type":"code","source":"N_parquets=1_000# so we dont have to load all 99k rows data\ntotal_data={}\nfor idx,d in tqdm(train.iterrows(),total=len(train)):\n    file_path = d['path']\n    \n    ex_landmark= pd.read_parquet(f\"{base_dir}/{file_path}\")\n    #getting the no.of landmarks  with x,y,z locations per type\n    \n    meta_data= ex_landmark.dropna(subset=['x','y','z'])['type'].value_counts().to_dict()\n    \n    meta_data['frames']=ex_landmark['frame'].nunique()\n    \n    xyz_mdata=(ex_landmark.agg(\n    {'x':['min','max','mean'],'y':['min','max','mean'],\n     'z':['min','max','mean']}).unstack().to_dict())\n   \n    for i in xyz_mdata.keys():\n        j = i[0] +'_'+i[1]\n        meta_data[j]=xyz_mdata[i]\n        \n    total_data[file_path] = meta_data\n    \n    if idx >= N_parquets:\n        break\n","metadata":{"execution":{"iopub.status.busy":"2023-04-05T05:43:41.486762Z","iopub.execute_input":"2023-04-05T05:43:41.487195Z","iopub.status.idle":"2023-04-05T05:44:05.267006Z","shell.execute_reply.started":"2023-04-05T05:43:41.487157Z","shell.execute_reply":"2023-04-05T05:44:05.265439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data=train.merge(pd.DataFrame(total_data).T.reset_index().rename(columns={'index':'path'}))\ntrain_data","metadata":{"execution":{"iopub.status.busy":"2023-04-05T05:44:19.131174Z","iopub.execute_input":"2023-04-05T05:44:19.131990Z","iopub.status.idle":"2023-04-05T05:44:19.242168Z","shell.execute_reply.started":"2023-04-05T05:44:19.131926Z","shell.execute_reply":"2023-04-05T05:44:19.240848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.columns","metadata":{"execution":{"iopub.status.busy":"2023-04-05T05:44:22.658420Z","iopub.execute_input":"2023-04-05T05:44:22.659650Z","iopub.status.idle":"2023-04-05T05:44:22.667166Z","shell.execute_reply.started":"2023-04-05T05:44:22.659603Z","shell.execute_reply":"2023-04-05T05:44:22.665736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_data","metadata":{"execution":{"iopub.status.busy":"2023-04-05T05:44:24.900358Z","iopub.execute_input":"2023-04-05T05:44:24.900773Z","iopub.status.idle":"2023-04-05T05:44:24.908752Z","shell.execute_reply.started":"2023-04-05T05:44:24.900736Z","shell.execute_reply":"2023-04-05T05:44:24.907352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data[['face','pose','left_hand','right_hand']].sum().sort_values().plot(kind='barh',title='sumof rows by landmarktype')","metadata":{"execution":{"iopub.status.busy":"2023-04-05T05:44:28.230517Z","iopub.execute_input":"2023-04-05T05:44:28.231663Z","iopub.status.idle":"2023-04-05T05:44:28.473867Z","shell.execute_reply.started":"2023-04-05T05:44:28.231613Z","shell.execute_reply":"2023-04-05T05:44:28.472403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(train_data.query('index<1000')\n .fillna(0)[['face','pose','left_hand','right_hand']] >0).mean().plot(kind='barh',title='percent/keypoints with data')","metadata":{"execution":{"iopub.status.busy":"2023-04-05T05:46:51.054250Z","iopub.execute_input":"2023-04-05T05:46:51.054702Z","iopub.status.idle":"2023-04-05T05:46:51.276858Z","shell.execute_reply.started":"2023-04-05T05:46:51.054663Z","shell.execute_reply":"2023-04-05T05:46:51.275744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data","metadata":{"execution":{"iopub.status.busy":"2023-04-05T05:41:24.968698Z","iopub.execute_input":"2023-04-05T05:41:24.969579Z","iopub.status.idle":"2023-04-05T05:41:25.014210Z","shell.execute_reply.started":"2023-04-05T05:41:24.969528Z","shell.execute_reply":"2023-04-05T05:41:25.012396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EXAMPLE OF HEAR ","metadata":{}},{"cell_type":"code","source":"train_data.dropna()","metadata":{"execution":{"iopub.status.busy":"2023-04-05T05:48:15.119397Z","iopub.execute_input":"2023-04-05T05:48:15.119777Z","iopub.status.idle":"2023-04-05T05:48:15.172542Z","shell.execute_reply.started":"2023-04-05T05:48:15.119742Z","shell.execute_reply":"2023-04-05T05:48:15.171304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xyz_mdata.keys()","metadata":{"execution":{"iopub.status.busy":"2023-04-05T05:17:08.853068Z","iopub.execute_input":"2023-04-05T05:17:08.853513Z","iopub.status.idle":"2023-04-05T05:17:08.860753Z","shell.execute_reply.started":"2023-04-05T05:17:08.853478Z","shell.execute_reply":"2023-04-05T05:17:08.859456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ex_parquet=train.query('sign == \"hear\"')['path'].values[0]\nex_landmark=pd.read_parquet(f\"{base_dir}/{ex_parquet}\")\nex_landmark.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-05T05:58:58.528684Z","iopub.execute_input":"2023-04-05T05:58:58.529126Z","iopub.status.idle":"2023-04-05T05:58:58.569991Z","shell.execute_reply.started":"2023-04-05T05:58:58.529091Z","shell.execute_reply":"2023-04-05T05:58:58.568674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ex_landmark['frame'].median()","metadata":{"execution":{"iopub.status.busy":"2023-04-05T06:00:09.814367Z","iopub.execute_input":"2023-04-05T06:00:09.815235Z","iopub.status.idle":"2023-04-05T06:00:09.824478Z","shell.execute_reply.started":"2023-04-05T06:00:09.815185Z","shell.execute_reply":"2023-04-05T06:00:09.822848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ex_landmark.query('frame==15')['type'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-04-05T06:00:15.948469Z","iopub.execute_input":"2023-04-05T06:00:15.948913Z","iopub.status.idle":"2023-04-05T06:00:15.962270Z","shell.execute_reply.started":"2023-04-05T06:00:15.948872Z","shell.execute_reply":"2023-04-05T06:00:15.960695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ex_landmark['no_xyz']=ex_landmark['type'].isna()","metadata":{"execution":{"iopub.status.busy":"2023-04-05T06:09:21.370481Z","iopub.execute_input":"2023-04-05T06:09:21.370918Z","iopub.status.idle":"2023-04-05T06:09:21.378047Z","shell.execute_reply.started":"2023-04-05T06:09:21.370873Z","shell.execute_reply":"2023-04-05T06:09:21.376720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ex_landmark.groupby('frame')['no_xyz'].sum()","metadata":{"execution":{"iopub.status.busy":"2023-04-05T06:09:24.977935Z","iopub.execute_input":"2023-04-05T06:09:24.978404Z","iopub.status.idle":"2023-04-05T06:09:24.989551Z","shell.execute_reply.started":"2023-04-05T06:09:24.978359Z","shell.execute_reply":"2023-04-05T06:09:24.988087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**3D plot of landmarks from look example**","metadata":{}},{"cell_type":"code","source":"ex_frame=ex_landmark.query('frame==10')","metadata":{"execution":{"iopub.status.busy":"2023-04-05T06:00:18.638703Z","iopub.execute_input":"2023-04-05T06:00:18.639138Z","iopub.status.idle":"2023-04-05T06:00:18.648233Z","shell.execute_reply.started":"2023-04-05T06:00:18.639100Z","shell.execute_reply":"2023-04-05T06:00:18.647005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ex_frame['type'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-04-05T06:00:21.410226Z","iopub.execute_input":"2023-04-05T06:00:21.410911Z","iopub.status.idle":"2023-04-05T06:00:21.419510Z","shell.execute_reply.started":"2023-04-05T06:00:21.410871Z","shell.execute_reply":"2023-04-05T06:00:21.418188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.express as px\npx.scatter_3d(ex_frame,'x','y','z',color='type')","metadata":{"execution":{"iopub.status.busy":"2023-04-05T06:04:27.158208Z","iopub.execute_input":"2023-04-05T06:04:27.158668Z","iopub.status.idle":"2023-04-05T06:04:27.235436Z","shell.execute_reply.started":"2023-04-05T06:04:27.158632Z","shell.execute_reply":"2023-04-05T06:04:27.234140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"USING MEDIAPIPE TO PLOT","metadata":{}},{"cell_type":"code","source":"!pip install mediapipe --quiet","metadata":{"execution":{"iopub.status.busy":"2023-04-05T06:13:32.955268Z","iopub.execute_input":"2023-04-05T06:13:32.956266Z","iopub.status.idle":"2023-04-05T06:13:47.172971Z","shell.execute_reply.started":"2023-04-05T06:13:32.956214Z","shell.execute_reply":"2023-04-05T06:13:47.171450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import mediapipe as mp\n\nmp_pose = mp.solutions.pose\n\n\nex_landmark[\"y_\"] = ex_landmark[\"y\"] * -1\n\nfig, ax = plt.subplots(figsize=(5, 5))\n\nfor pose in [\"pose\"]:\n    example_pose = ex_landmark.query(\"frame == 15 and type == @pose\")\n\n    ax.scatter(example_pose[\"x\"], example_pose[\"y_\"])\n\n    for connection in mp_pose.POSE_CONNECTIONS:\n        point_a = connection[0]\n        point_b = connection[1]\n        x1, y1 = example_pose.query(\"landmark_index == @point_a\")[[\"x\", \"y_\"]].values[0]\n        x2, y2 = example_pose.query(\"landmark_index == @point_b\")[[\"x\", \"y_\"]].values[0]\n        plt.plot([x1, x2], [y1, y2], color=\"Green\")\nax.set_title(\"HEAR - pose Data\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-05T06:24:24.490784Z","iopub.execute_input":"2023-04-05T06:24:24.491332Z","iopub.status.idle":"2023-04-05T06:24:24.955341Z","shell.execute_reply.started":"2023-04-05T06:24:24.491264Z","shell.execute_reply":"2023-04-05T06:24:24.954153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(5, 5))\n\nfor pose in [\"pose\"]:\n    example_pose = ex_landmark.query(\"frame == 8 and type == @pose\")\n\n    ax.scatter(example_pose[\"x\"], example_pose[\"y_\"])\n\n    for connection in mp_pose.POSE_CONNECTIONS:\n        point_a = connection[0]\n        point_b = connection[1]\n        x1, y1 = example_pose.query(\"landmark_index == @point_a\")[[\"x\", \"y_\"]].values[0]\n        x2, y2 = example_pose.query(\"landmark_index == @point_b\")[[\"x\", \"y_\"]].values[0]\n        plt.plot([x1, x2], [y1, y2], color=\"Green\")\nax.set_title(\"HEAR - pose Data\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-05T06:25:12.724765Z","iopub.execute_input":"2023-04-05T06:25:12.725214Z","iopub.status.idle":"2023-04-05T06:25:13.197987Z","shell.execute_reply.started":"2023-04-05T06:25:12.725171Z","shell.execute_reply":"2023-04-05T06:25:13.195493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**EVALUATION**","metadata":{}},{"cell_type":"code","source":"ROWS_PER_FRAME = 543  # number of landmarks per frame\n\ndef load_relevant_data_subset(pq_path):\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2023-04-05T06:27:33.353344Z","iopub.execute_input":"2023-04-05T06:27:33.354170Z","iopub.status.idle":"2023-04-05T06:27:33.361177Z","shell.execute_reply.started":"2023-04-05T06:27:33.354117Z","shell.execute_reply":"2023-04-05T06:27:33.360063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}