{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q mediapipe","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:18:33.006922Z","iopub.execute_input":"2023-07-02T15:18:33.007482Z","iopub.status.idle":"2023-07-02T15:18:50.995573Z","shell.execute_reply.started":"2023-07-02T15:18:33.007430Z","shell.execute_reply":"2023-07-02T15:18:50.994297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport mediapipe as mp\nimport tensorflow as tf\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:18:50.998283Z","iopub.execute_input":"2023-07-02T15:18:50.998730Z","iopub.status.idle":"2023-07-02T15:19:01.516390Z","shell.execute_reply.started":"2023-07-02T15:18:50.998685Z","shell.execute_reply":"2023-07-02T15:19:01.515405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"in_dir = \"/kaggle/input/asl-signs/\"\ntrain = pd.read_csv(f\"{in_dir}train.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:01.518502Z","iopub.execute_input":"2023-07-02T15:19:01.519210Z","iopub.status.idle":"2023-07-02T15:19:01.721052Z","shell.execute_reply.started":"2023-07-02T15:19:01.519176Z","shell.execute_reply":"2023-07-02T15:19:01.719991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:01.724179Z","iopub.execute_input":"2023-07-02T15:19:01.724612Z","iopub.status.idle":"2023-07-02T15:19:01.745510Z","shell.execute_reply.started":"2023-07-02T15:19:01.724560Z","shell.execute_reply":"2023-07-02T15:19:01.744534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"\"\"There are {train[\"sign\"].nunique()} unique signs.\"\"\")\nprint(f\"\"\"There are {train[\"participant_id\"].nunique()} different people who sign.\"\"\")\nprint(f\"\"\"There are {train[\"sequence_id\"].nunique()} total signs.\"\"\")","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:01.747695Z","iopub.execute_input":"2023-07-02T15:19:01.748479Z","iopub.status.idle":"2023-07-02T15:19:01.776027Z","shell.execute_reply.started":"2023-07-02T15:19:01.748442Z","shell.execute_reply":"2023-07-02T15:19:01.775066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"sign\"].value_counts().head().sort_values(ascending = False)","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:01.777713Z","iopub.execute_input":"2023-07-02T15:19:01.778079Z","iopub.status.idle":"2023-07-02T15:19:01.801420Z","shell.execute_reply.started":"2023-07-02T15:19:01.778045Z","shell.execute_reply":"2023-07-02T15:19:01.800199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(0,5):\n    exlm = pd.read_parquet(f\"\"\"{in_dir}/{train[\"path\"][i]}\"\"\")\n    print(exlm.head())","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:01.803298Z","iopub.execute_input":"2023-07-02T15:19:01.803947Z","iopub.status.idle":"2023-07-02T15:19:02.197667Z","shell.execute_reply.started":"2023-07-02T15:19:01.803914Z","shell.execute_reply":"2023-07-02T15:19:02.196482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"\"\"There are {exlm[\"type\"].nunique()} unique types, namely {exlm[\"type\"].unique()}.\"\"\")","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:02.199648Z","iopub.execute_input":"2023-07-02T15:19:02.200412Z","iopub.status.idle":"2023-07-02T15:19:02.209757Z","shell.execute_reply.started":"2023-07-02T15:19:02.200373Z","shell.execute_reply":"2023-07-02T15:19:02.208530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"\"\"There are {exlm[\"frame\"].nunique()} unique frames in this parquet.\"\"\")","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:02.211922Z","iopub.execute_input":"2023-07-02T15:19:02.212894Z","iopub.status.idle":"2023-07-02T15:19:02.221336Z","shell.execute_reply.started":"2023-07-02T15:19:02.212857Z","shell.execute_reply":"2023-07-02T15:19:02.219976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"leri = []\nfor signer in train[\"participant_id\"].unique():\n    sdf = train.query(f'participant_id == {signer}')[\"path\"].iloc[0]\n    lmleri = pd.read_parquet(f\"\"\"{in_dir}/{sdf}\"\"\")\n    leri.append([lmleri.query('type == \"left_hand\"')[\"frame\"].nunique(), lmleri.query('type == \"right_hand\"')[\"frame\"].nunique()])\nleri","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:02.223547Z","iopub.execute_input":"2023-07-02T15:19:02.224521Z","iopub.status.idle":"2023-07-02T15:19:03.037485Z","shell.execute_reply.started":"2023-07-02T15:19:02.224479Z","shell.execute_reply":"2023-07-02T15:19:03.036399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for signer in train[\"participant_id\"].unique():\n    sdf = train.query(f'participant_id == {signer}')[\"path\"].iloc[0]\n    lm = pd.read_parquet(f\"\"\"{in_dir}/{sdf}\"\"\")\n    lm[\"no_xyz\"] = lm[\"x\"].isna()\n    print(lm.query('no_xyz == True')[\"type\"].value_counts())","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:03.042715Z","iopub.execute_input":"2023-07-02T15:19:03.044116Z","iopub.status.idle":"2023-07-02T15:19:03.475417Z","shell.execute_reply.started":"2023-07-02T15:19:03.044077Z","shell.execute_reply":"2023-07-02T15:19:03.474269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.query('sign == \"police\"')[\"participant_id\"].nunique()","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:03.476766Z","iopub.execute_input":"2023-07-02T15:19:03.477538Z","iopub.status.idle":"2023-07-02T15:19:03.496527Z","shell.execute_reply.started":"2023-07-02T15:19:03.477504Z","shell.execute_reply":"2023-07-02T15:19:03.495504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.query('sign == \"police\"')[\"participant_id\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:03.500650Z","iopub.execute_input":"2023-07-02T15:19:03.500984Z","iopub.status.idle":"2023-07-02T15:19:03.521614Z","shell.execute_reply.started":"2023-07-02T15:19:03.500956Z","shell.execute_reply":"2023-07-02T15:19:03.517120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train.query('sign == \"police\"')[\"participant_id\"].index)","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:03.523093Z","iopub.execute_input":"2023-07-02T15:19:03.523475Z","iopub.status.idle":"2023-07-02T15:19:03.562365Z","shell.execute_reply.started":"2023-07-02T15:19:03.523435Z","shell.execute_reply":"2023-07-02T15:19:03.556016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train2 = train\ntrain2.sort_values(\"participant_id\", axis = 0, ascending = True, inplace = True)\ntrain2.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:03.563979Z","iopub.execute_input":"2023-07-02T15:19:03.564681Z","iopub.status.idle":"2023-07-02T15:19:03.609830Z","shell.execute_reply.started":"2023-07-02T15:19:03.564643Z","shell.execute_reply":"2023-07-02T15:19:03.608797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfleri = pd.DataFrame({\"participant_id\": [],\n                      \"left_missing\": [],\n                      \"right_missing\": []})\nfor i in range(0,388):\n    sdf = train2.query('sign == \"police\"')[\"path\"].iloc[i]\n    lm = pd.read_parquet(f\"\"\"{in_dir}/{sdf}\"\"\")\n    lm[\"no_xyz\"] = lm[\"x\"].isna()\n    dfleri.loc[i] = [train2.query('sign == \"police\"')[\"participant_id\"].iloc[i],\n                    len(lm.query('no_xyz == True').query('type == \"left_hand\"').index),\n                    len(lm.query('no_xyz == True').query('type == \"right_hand\"').index)]\n#    print(f\"\"\"{train2.query('sign == \"police\"')[\"participant_id\"]}: {lm.query('no_xyz == True')[\"type\"].value_counts()}\"\"\")\ndfleri.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:03.611340Z","iopub.execute_input":"2023-07-02T15:19:03.615074Z","iopub.status.idle":"2023-07-02T15:19:24.179654Z","shell.execute_reply.started":"2023-07-02T15:19:03.615036Z","shell.execute_reply":"2023-07-02T15:19:24.178486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dflerif = dfleri.groupby(\"participant_id\")[[\"left_missing\", \"right_missing\"]].sum()\ndflerif","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:24.181231Z","iopub.execute_input":"2023-07-02T15:19:24.181795Z","iopub.status.idle":"2023-07-02T15:19:24.203470Z","shell.execute_reply.started":"2023-07-02T15:19:24.181745Z","shell.execute_reply":"2023-07-02T15:19:24.202493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from mpl_toolkits import mplot3d\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:24.205255Z","iopub.execute_input":"2023-07-02T15:19:24.206050Z","iopub.status.idle":"2023-07-02T15:19:24.376935Z","shell.execute_reply.started":"2023-07-02T15:19:24.206012Z","shell.execute_reply":"2023-07-02T15:19:24.375380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure()\nax = fig.add_subplot(projection='3d')\nlme = pd.read_parquet(f\"\"\"{in_dir}/{train2[\"path\"].iloc[0]}\"\"\")\nfrm = lme[\"frame\"].iloc[547]\nlmex = lme.query(f'frame == {frm}')\nax.scatter([lmex[\"x\"]], [lmex[\"y\"]], [lmex[\"z\"]])","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:24.380715Z","iopub.execute_input":"2023-07-02T15:19:24.381262Z","iopub.status.idle":"2023-07-02T15:19:24.736477Z","shell.execute_reply.started":"2023-07-02T15:19:24.381233Z","shell.execute_reply":"2023-07-02T15:19:24.735565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nfrom mediapipe.framework.formats import landmark_pb2\nmp_drawing = mp.solutions.drawing_utils\nmp_drawing_styles = mp.solutions.drawing_styles\nmp_pose = mp.solutions.pose\nmp_holistic = mp.solutions.holistic","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:24.737893Z","iopub.execute_input":"2023-07-02T15:19:24.738855Z","iopub.status.idle":"2023-07-02T15:19:24.745317Z","shell.execute_reply.started":"2023-07-02T15:19:24.738809Z","shell.execute_reply":"2023-07-02T15:19:24.744138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Landmarksof:\n    def __init__(self, df, frame):\n        f = df[df.frame == frame]\n        face = landmark_pb2.NormalizedLandmarkList()\n        for i in f[f.type=='face'][['x','y','z']].itertuples(index=False):\n            face.landmark.add(x=i.x,y=i.y,z=i.z)\n        left_hand = landmark_pb2.NormalizedLandmarkList()\n        for i in f[f.type=='left_hand'][['x','y','z']].itertuples(index=False):\n            left_hand.landmark.add(x=i.x,y=i.y,z=i.z)\n        pose = landmark_pb2.NormalizedLandmarkList()\n        for i in f[f.type=='pose'][['x','y','z']].itertuples(index=False):\n            pose.landmark.add(x=i.x,y=i.y,z=i.z)\n        right_hand = landmark_pb2.NormalizedLandmarkList()\n        for i in f[f.type=='right_hand'][['x','y','z']].itertuples(index=False):\n            right_hand.landmark.add(x=i.x,y=i.y,z=i.z)\n        self.face_landmarks = face\n        self.left_hand_landmarks = left_hand\n        self.pose_landmarks = pose\n        self.right_hand_landmarks = right_hand","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:24.747041Z","iopub.execute_input":"2023-07-02T15:19:24.747810Z","iopub.status.idle":"2023-07-02T15:19:24.759951Z","shell.execute_reply.started":"2023-07-02T15:19:24.747773Z","shell.execute_reply":"2023-07-02T15:19:24.758507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def draw_landmarks(landmarks,image,show_pose=True,show_face_contour=True,show_face_tesselation=True,show_left_hand=True,show_right_hand=True):\n    annotated_image = image.copy()\n    results = landmarks\n    if show_face_tesselation:\n        mp_drawing.draw_landmarks(\n            annotated_image,\n            results.face_landmarks,\n            mp_holistic.FACEMESH_TESSELATION,\n            landmark_drawing_spec=None,\n            connection_drawing_spec=mp_drawing_styles\n            .get_default_face_mesh_tesselation_style())\n    if show_face_contour:\n        mp_drawing.draw_landmarks(\n            annotated_image,\n            results.face_landmarks,\n            mp_holistic.FACEMESH_CONTOURS,\n            landmark_drawing_spec=None,\n            connection_drawing_spec=mp_drawing_styles\n            .get_default_face_mesh_contours_style())\n    if show_pose:\n        mp_drawing.draw_landmarks(\n            annotated_image,\n            results.pose_landmarks,\n            mp_holistic.POSE_CONNECTIONS,\n            landmark_drawing_spec=mp_drawing_styles.\n            get_default_pose_landmarks_style())\n    if show_left_hand:\n        mp_drawing.draw_landmarks(\n            annotated_image,\n            results.left_hand_landmarks,\n            mp_holistic.HAND_CONNECTIONS,\n            landmark_drawing_spec=mp_drawing_styles\n            .get_default_hand_landmarks_style())\n    if show_right_hand:\n        mp_drawing.draw_landmarks(\n            annotated_image,\n            results.right_hand_landmarks,\n            mp_holistic.HAND_CONNECTIONS,\n            landmark_drawing_spec=mp_drawing_styles\n            .get_default_hand_landmarks_style())\n    return annotated_image","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:24.762577Z","iopub.execute_input":"2023-07-02T15:19:24.763014Z","iopub.status.idle":"2023-07-02T15:19:24.775472Z","shell.execute_reply.started":"2023-07-02T15:19:24.762978Z","shell.execute_reply":"2023-07-02T15:19:24.774354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lme[\"frame\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:24.777325Z","iopub.execute_input":"2023-07-02T15:19:24.777707Z","iopub.status.idle":"2023-07-02T15:19:24.793719Z","shell.execute_reply.started":"2023-07-02T15:19:24.777673Z","shell.execute_reply":"2023-07-02T15:19:24.792604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from fastai.vision.all import show_image\nannotated_image = np.zeros((1024,1024,3),dtype=np.uint8)\nlandmarks = Landmarksof(lme, 32)\nshow_image(draw_landmarks(landmarks,annotated_image))","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:24.795159Z","iopub.execute_input":"2023-07-02T15:19:24.795808Z","iopub.status.idle":"2023-07-02T15:19:30.479327Z","shell.execute_reply.started":"2023-07-02T15:19:24.795757Z","shell.execute_reply":"2023-07-02T15:19:30.478401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def Visual(lme, fname):\n    annotated_image = np.zeros((1024,1024,3),dtype=np.uint8)\n    img = []\n    for i in sorted(lme[\"frame\"].unique().tolist()):\n        landmarks = Landmarksof(lme, i)\n        img.append(draw_landmarks(landmarks,annotated_image))\n    height, width, _ = img[0].shape\n    out = cv2.VideoWriter(f'{fname}.avi',cv2.VideoWriter_fourcc(*'DIVX'), 15, (width, height))\n    [out.write(img[k]) for k in range(0, len(img))]\n    out.release()","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:30.480852Z","iopub.execute_input":"2023-07-02T15:19:30.481858Z","iopub.status.idle":"2023-07-02T15:19:30.490336Z","shell.execute_reply.started":"2023-07-02T15:19:30.481820Z","shell.execute_reply":"2023-07-02T15:19:30.489301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"lm2 = pd.read_parquet(f\"\"\"{in_dir}/{train2.query('sign == \"gift\"')[\"path\"].iloc[0]}\"\"\")\nVisual(lm2, \"output41\")","metadata":{"execution":{"iopub.status.busy":"2023-06-29T19:56:11.712950Z","iopub.execute_input":"2023-06-29T19:56:11.713814Z","iopub.status.idle":"2023-06-29T19:56:12.985369Z","shell.execute_reply.started":"2023-06-29T19:56:11.713777Z","shell.execute_reply":"2023-06-29T19:56:12.984304Z"}}},{"cell_type":"markdown","source":"lm3 = pd.read_parquet(f\"\"\"{in_dir}/{train2.query('sign == \"feet\"')[\"path\"].iloc[2]}\"\"\")\nVisual(lm3, \"output71\")","metadata":{"execution":{"iopub.status.busy":"2023-06-29T20:42:18.638273Z","iopub.execute_input":"2023-06-29T20:42:18.638794Z","iopub.status.idle":"2023-06-29T20:42:19.740274Z","shell.execute_reply.started":"2023-06-29T20:42:18.638760Z","shell.execute_reply":"2023-06-29T20:42:19.738973Z"}}},{"cell_type":"markdown","source":"train2.query('participant_id == 27610')[\"sign\"].iloc[7]","metadata":{"execution":{"iopub.status.busy":"2023-06-29T20:39:57.854867Z","iopub.execute_input":"2023-06-29T20:39:57.855341Z","iopub.status.idle":"2023-06-29T20:39:57.871761Z","shell.execute_reply.started":"2023-06-29T20:39:57.855309Z","shell.execute_reply":"2023-06-29T20:39:57.869955Z"}}},{"cell_type":"markdown","source":"27610 - not as reliable data: longer, unclear hand dominance, missing landmarks","metadata":{}},{"cell_type":"code","source":"train3 = train[train.participant_id != 27610]\ntrain3[\"participant_id\"].sort_values(ascending = True).unique()","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:30.492880Z","iopub.execute_input":"2023-07-02T15:19:30.493897Z","iopub.status.idle":"2023-07-02T15:19:30.519006Z","shell.execute_reply.started":"2023-07-02T15:19:30.493860Z","shell.execute_reply":"2023-07-02T15:19:30.518246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lme[[\"x\", \"y\", \"z\"]].iloc[0].tolist()","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:30.520794Z","iopub.execute_input":"2023-07-02T15:19:30.521541Z","iopub.status.idle":"2023-07-02T15:19:30.531428Z","shell.execute_reply.started":"2023-07-02T15:19:30.521503Z","shell.execute_reply":"2023-07-02T15:19:30.530149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"data = []\n\nfor y in range(0, 94477):\n    lmdf = pd.read_parquet(f\"\"\"{in_dir}/{train2[\"path\"].iloc[y]}\"\"\")\n    facet = []\n    left_handt = []\n    poset = []\n    right_handt = []\n    for j in sorted(lmdf[\"frame\"].unique()):\n        facel = [lmdf[lmdf.frame == j][lmdf.type == \"face\"][[\"x\", \"y\", \"z\"]].iloc[v].tolist() for v in range(0, len(lmdf[lmdf.frame == j][lmdf.type == \"face\"].index))]\n        facet += facel\n        left_handl = [lmdf[lmdf.frame == j][lmdf.type == \"left_hand\"][[\"x\", \"y\", \"z\"]].iloc[v].tolist() for v in range(0, len(lmdf[lmdf.frame == j][lmdf.type == \"left_hand\"].index))]\n        left_handt += left_handl\n        posel = [lmdf[lmdf.frame == j][lmdf.type == \"pose\"][[\"x\", \"y\", \"z\"]].iloc[v].tolist() for v in range(0, len(lmdf[lmdf.frame == j][lmdf.type == \"pose\"].index))]\n        poset += posel\n        right_handl = [lmdf[lmdf.frame == j][lmdf.type == \"right_hand\"][[\"x\", \"y\", \"z\"]].iloc[v].tolist() for v in range(0, len(lmdf[lmdf.frame == j][lmdf.type == \"right_hand\"].index))]\n        right_handt += right_handl\n    data += [facet, left_handt, poset, right_handt]","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:30.533603Z","iopub.execute_input":"2023-07-02T15:19:30.533952Z","iopub.status.idle":"2023-07-02T15:19:30.544029Z","shell.execute_reply.started":"2023-07-02T15:19:30.533925Z","shell.execute_reply":"2023-07-02T15:19:30.539178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"data = []\n\nfor signc in train3[\"sign\"]:\n    trainp = train3[train3.sign == signc]\n    for o in range(0, 42):\n        lmdf = pd.read_parquet(f\"\"\"{in_dir}/{trainp[\"path\"].iloc[o]}\"\"\")\n        facet = []\n        left_handt = []\n        poset = []\n        right_handt = []\n        for j in sorted(lmdf[\"frame\"].unique()):\n            facel = np.array([lmdf[lmdf.frame == j][lmdf.type == \"face\"][[\"x\", \"y\", \"z\"]].iloc[v].tolist() for v in range(0, len(lmdf[lmdf.frame == j][lmdf.type == \"face\"].index))])\n            facet += [facel.mean(), facel.std()]\n            left_handl = np.array([lmdf[lmdf.frame == j][lmdf.type == \"left_hand\"][[\"x\", \"y\", \"z\"]].iloc[v].tolist() for v in range(0, len(lmdf[lmdf.frame == j][lmdf.type == \"left_hand\"].index))])\n            left_handt += [left_handl.mean(), left_handl.std()]\n            posel = np.array([lmdf[lmdf.frame == j][lmdf.type == \"pose\"][[\"x\", \"y\", \"z\"]].iloc[v].tolist() for v in range(0, len(lmdf[lmdf.frame == j][lmdf.type == \"pose\"].index))])\n            poset += [posel.mean(), posel.std()]\n            right_handl = np.array([lmdf[lmdf.frame == j][lmdf.type == \"right_hand\"][[\"x\", \"y\", \"z\"]].iloc[v].tolist() for v in range(0, len(lmdf[lmdf.frame == j][lmdf.type == \"right_hand\"].index))])\n            right_handt += [right_handl.mean(), right_handl.std()]\n    data += [facet, left_handt, poset, right_handt]","metadata":{}},{"cell_type":"code","source":"counter = 0\nfor y in range(0, 94477):\n    counter += 1\nprint(counter)","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:30.547038Z","iopub.execute_input":"2023-07-02T15:19:30.547362Z","iopub.status.idle":"2023-07-02T15:19:30.569002Z","shell.execute_reply.started":"2023-07-02T15:19:30.547336Z","shell.execute_reply":"2023-07-02T15:19:30.567610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.array([lme[[\"x\", \"y\", \"z\"]].iloc[i].tolist() for i in range(0, 6)]).mean(axis = 0)","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:30.571056Z","iopub.execute_input":"2023-07-02T15:19:30.571458Z","iopub.status.idle":"2023-07-02T15:19:30.585875Z","shell.execute_reply.started":"2023-07-02T15:19:30.571423Z","shell.execute_reply":"2023-07-02T15:19:30.584887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frml = []\nfor i in lme[\"frame\"]:\n    frml.append(len(lme[lme.frame == i].index))\nprint(False in np.array(frml) == 543)","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:30.587692Z","iopub.execute_input":"2023-07-02T15:19:30.588515Z","iopub.status.idle":"2023-07-02T15:19:33.918420Z","shell.execute_reply.started":"2023-07-02T15:19:30.588462Z","shell.execute_reply":"2023-07-02T15:19:33.917136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"[len(lme[lme.frame == 21][lme.type == \"face\"].index),\n len(lme[lme.frame == 21][lme.type == \"left_hand\"].index),\n len(lme[lme.frame == 21][lme.type == \"pose\"].index),\n len(lme[lme.frame == 21][lme.type == \"right_hand\"].index)]","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:33.926561Z","iopub.execute_input":"2023-07-02T15:19:33.926925Z","iopub.status.idle":"2023-07-02T15:19:33.953521Z","shell.execute_reply.started":"2023-07-02T15:19:33.926895Z","shell.execute_reply":"2023-07-02T15:19:33.952316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"[https://developers.google.com/mediapipe/solutions/vision/holistic_landmarker](http://)\n\n[https://github.com/k-m-irfan/simplified_mediapipe_face_landmarks](http://)","metadata":{}},{"cell_type":"code","source":"lipindices = [78,191,80,81,82,13,312,311,310,415,308,324,318,402,317,14,87,178,88,95] + [61,185,40,39,37,0,267,269,270,409,291,375,321,405,314,17,84,181,91,146]","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:33.955341Z","iopub.execute_input":"2023-07-02T15:19:33.955732Z","iopub.status.idle":"2023-07-02T15:19:33.964391Z","shell.execute_reply.started":"2023-07-02T15:19:33.955699Z","shell.execute_reply":"2023-07-02T15:19:33.963311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(lipindices)","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:33.966108Z","iopub.execute_input":"2023-07-02T15:19:33.966673Z","iopub.status.idle":"2023-07-02T15:19:33.976014Z","shell.execute_reply.started":"2023-07-02T15:19:33.966637Z","shell.execute_reply":"2023-07-02T15:19:33.974676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rightarm = np.array([12, 14, 16, 18, 20, 22]) + 489\nleftarm = np.array([11, 13, 15, 17, 19, 21]) + 489","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:33.978314Z","iopub.execute_input":"2023-07-02T15:19:33.978810Z","iopub.status.idle":"2023-07-02T15:19:33.986539Z","shell.execute_reply.started":"2023-07-02T15:19:33.978775Z","shell.execute_reply":"2023-07-02T15:19:33.985388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lefth1 = []\nrighth1 = []\ndflerifn = dflerif.drop(27610)\nsigners = sorted(train3[\"participant_id\"].unique())\nfor signer in range(0, len(signers)):\n    if dflerifn[\"left_missing\"].iloc[signer] < dflerifn[\"right_missing\"].iloc[signer]:\n        lefth1.append(signers[signer])\n    else :\n        righth1.append(signers[signer])","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:33.988288Z","iopub.execute_input":"2023-07-02T15:19:33.988815Z","iopub.status.idle":"2023-07-02T15:19:34.000963Z","shell.execute_reply.started":"2023-07-02T15:19:33.988782Z","shell.execute_reply":"2023-07-02T15:19:33.999727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lefth1, righth1","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:34.002653Z","iopub.execute_input":"2023-07-02T15:19:34.003361Z","iopub.status.idle":"2023-07-02T15:19:34.012087Z","shell.execute_reply.started":"2023-07-02T15:19:34.003322Z","shell.execute_reply":"2023-07-02T15:19:34.010765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dflerifn","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:34.013819Z","iopub.execute_input":"2023-07-02T15:19:34.014282Z","iopub.status.idle":"2023-07-02T15:19:34.028430Z","shell.execute_reply.started":"2023-07-02T15:19:34.014247Z","shell.execute_reply":"2023-07-02T15:19:34.027261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train3.query('sign == \"listen\"').index)","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:34.032405Z","iopub.execute_input":"2023-07-02T15:19:34.032878Z","iopub.status.idle":"2023-07-02T15:19:34.050861Z","shell.execute_reply.started":"2023-07-02T15:19:34.032849Z","shell.execute_reply":"2023-07-02T15:19:34.049385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfleri2 = pd.DataFrame({\"participant_id\": [],\n                      \"left_missing\": [],\n                      \"right_missing\": []})\nfor i in range(0,396):\n    sdf = train3.query('sign == \"listen\"')[\"path\"].iloc[i]\n    lm = pd.read_parquet(f\"\"\"{in_dir}/{sdf}\"\"\")\n    lm[\"no_xyz\"] = lm[\"x\"].isna()\n    dfleri2.loc[i] = [train3.query('sign == \"listen\"')[\"participant_id\"].iloc[i],\n                    len(lm.query('no_xyz == True').query('type == \"left_hand\"').index),\n                    len(lm.query('no_xyz == True').query('type == \"right_hand\"').index)]\n#    print(f\"\"\"{train2.query('sign == \"police\"')[\"participant_id\"]}: {lm.query('no_xyz == True')[\"type\"].value_counts()}\"\"\")\ndflerif2 = dfleri2.groupby(\"participant_id\")[[\"left_missing\", \"right_missing\"]].sum()\ndflerif2","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:34.053035Z","iopub.execute_input":"2023-07-02T15:19:34.054006Z","iopub.status.idle":"2023-07-02T15:19:53.380744Z","shell.execute_reply.started":"2023-07-02T15:19:34.053971Z","shell.execute_reply":"2023-07-02T15:19:53.379540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lefth2 = []\nrighth2 = []\nfor signer in range(0, len(signers)):\n    if dflerif2[\"left_missing\"].iloc[signer] < dflerif2[\"right_missing\"].iloc[signer]:\n        lefth2.append(signers[signer])\n    else :\n        righth2.append(signers[signer])","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:53.382694Z","iopub.execute_input":"2023-07-02T15:19:53.383097Z","iopub.status.idle":"2023-07-02T15:19:53.390477Z","shell.execute_reply.started":"2023-07-02T15:19:53.383061Z","shell.execute_reply":"2023-07-02T15:19:53.389089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lefth1, lefth2","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:53.392698Z","iopub.execute_input":"2023-07-02T15:19:53.393904Z","iopub.status.idle":"2023-07-02T15:19:53.407769Z","shell.execute_reply.started":"2023-07-02T15:19:53.393861Z","shell.execute_reply":"2023-07-02T15:19:53.406731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"righth1, righth2","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:53.409548Z","iopub.execute_input":"2023-07-02T15:19:53.410038Z","iopub.status.idle":"2023-07-02T15:19:53.422018Z","shell.execute_reply.started":"2023-07-02T15:19:53.410000Z","shell.execute_reply":"2023-07-02T15:19:53.420892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfleri3 = pd.DataFrame({\"participant_id\": [],\n                      \"left_missing\": [],\n                      \"right_missing\": []})\nfor i in range(0,374):\n    sdf = train3.query('sign == \"gift\"')[\"path\"].iloc[i]\n    lm = pd.read_parquet(f\"\"\"{in_dir}/{sdf}\"\"\")\n    lm[\"no_xyz\"] = lm[\"x\"].isna()\n    dfleri3.loc[i] = [train3.query('sign == \"gift\"')[\"participant_id\"].iloc[i],\n                    len(lm.query('no_xyz == True').query('type == \"left_hand\"').index),\n                    len(lm.query('no_xyz == True').query('type == \"right_hand\"').index)]\ndflerif3 = dfleri3.groupby(\"participant_id\")[[\"left_missing\", \"right_missing\"]].sum()\ndflerif3","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:19:53.423638Z","iopub.execute_input":"2023-07-02T15:19:53.424175Z","iopub.status.idle":"2023-07-02T15:20:12.065985Z","shell.execute_reply.started":"2023-07-02T15:19:53.424141Z","shell.execute_reply":"2023-07-02T15:20:12.064982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lefth3 = []\nrighth3 = []\nfor signer in range(0, len(signers)):\n    if dflerif3[\"left_missing\"].iloc[signer] < dflerif3[\"right_missing\"].iloc[signer]:\n        lefth3.append(signers[signer])\n    else :\n        righth3.append(signers[signer])","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:20:12.067622Z","iopub.execute_input":"2023-07-02T15:20:12.068308Z","iopub.status.idle":"2023-07-02T15:20:12.076130Z","shell.execute_reply.started":"2023-07-02T15:20:12.068272Z","shell.execute_reply":"2023-07-02T15:20:12.075059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lefth1, lefth2, lefth3","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:20:12.077753Z","iopub.execute_input":"2023-07-02T15:20:12.078718Z","iopub.status.idle":"2023-07-02T15:20:12.090666Z","shell.execute_reply.started":"2023-07-02T15:20:12.078675Z","shell.execute_reply":"2023-07-02T15:20:12.089619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"righth1, righth2, righth3","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:20:12.092642Z","iopub.execute_input":"2023-07-02T15:20:12.093191Z","iopub.status.idle":"2023-07-02T15:20:12.102036Z","shell.execute_reply.started":"2023-07-02T15:20:12.093151Z","shell.execute_reply":"2023-07-02T15:20:12.100996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train3.query('sign == \"boy\"').index)","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:20:12.103876Z","iopub.execute_input":"2023-07-02T15:20:12.104631Z","iopub.status.idle":"2023-07-02T15:20:12.121982Z","shell.execute_reply.started":"2023-07-02T15:20:12.104595Z","shell.execute_reply":"2023-07-02T15:20:12.120835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfleri4 = pd.DataFrame({\"participant_id\": [],\n                      \"left_missing\": [],\n                      \"right_missing\": []})\nfor i in range(0,366):\n    sdf = train3.query('sign == \"boy\"')[\"path\"].iloc[i]\n    lm = pd.read_parquet(f\"\"\"{in_dir}/{sdf}\"\"\")\n    lm[\"no_xyz\"] = lm[\"x\"].isna()\n    dfleri4.loc[i] = [train3.query('sign == \"boy\"')[\"participant_id\"].iloc[i],\n                    len(lm.query('no_xyz == True').query('type == \"left_hand\"').index),\n                    len(lm.query('no_xyz == True').query('type == \"right_hand\"').index)]\ndflerif4 = dfleri4.groupby(\"participant_id\")[[\"left_missing\", \"right_missing\"]].sum()\ndflerif4","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:20:12.123730Z","iopub.execute_input":"2023-07-02T15:20:12.124352Z","iopub.status.idle":"2023-07-02T15:20:29.975944Z","shell.execute_reply.started":"2023-07-02T15:20:12.124318Z","shell.execute_reply":"2023-07-02T15:20:29.974837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lefth4 = []\nrighth4 = []\nfor signer in range(0, len(signers)):\n    if dflerif4[\"left_missing\"].iloc[signer] < dflerif4[\"right_missing\"].iloc[signer]:\n        lefth4.append(signers[signer])\n    else :\n        righth4.append(signers[signer])","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:20:29.977598Z","iopub.execute_input":"2023-07-02T15:20:29.978261Z","iopub.status.idle":"2023-07-02T15:20:29.986076Z","shell.execute_reply.started":"2023-07-02T15:20:29.978222Z","shell.execute_reply":"2023-07-02T15:20:29.984909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lefth1, lefth2, lefth3, lefth4","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:20:29.987900Z","iopub.execute_input":"2023-07-02T15:20:29.988306Z","iopub.status.idle":"2023-07-02T15:20:30.001006Z","shell.execute_reply.started":"2023-07-02T15:20:29.988270Z","shell.execute_reply":"2023-07-02T15:20:29.998852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"righth1, righth2, righth3, righth4","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:20:30.004263Z","iopub.execute_input":"2023-07-02T15:20:30.004632Z","iopub.status.idle":"2023-07-02T15:20:30.016464Z","shell.execute_reply.started":"2023-07-02T15:20:30.004584Z","shell.execute_reply":"2023-07-02T15:20:30.015251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"37055 consistently involves the other hand.","metadata":{}},{"cell_type":"code","source":"trainf = train3.drop(train3[train3.participant_id == 37055].index)","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:20:30.018147Z","iopub.execute_input":"2023-07-02T15:20:30.018766Z","iopub.status.idle":"2023-07-02T15:20:30.036270Z","shell.execute_reply.started":"2023-07-02T15:20:30.018731Z","shell.execute_reply":"2023-07-02T15:20:30.035311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lefth = lefth1.remove(37055)\nrighth = righth1","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:20:30.038130Z","iopub.execute_input":"2023-07-02T15:20:30.038873Z","iopub.status.idle":"2023-07-02T15:20:30.043511Z","shell.execute_reply.started":"2023-07-02T15:20:30.038838Z","shell.execute_reply":"2023-07-02T15:20:30.042437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"minl = []\nmaxl = []\nfor i in range(0,2000):\n    sdf = train3[\"path\"].iloc[i]\n    lm = pd.read_parquet(f\"\"\"{in_dir}/{sdf}\"\"\")\n    minl.append(lm[\"x\"].min())\n    maxl.append(lm[\"x\"].max())","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:20:30.045196Z","iopub.execute_input":"2023-07-02T15:20:30.046047Z","iopub.status.idle":"2023-07-02T15:21:07.739527Z","shell.execute_reply.started":"2023-07-02T15:20:30.046014Z","shell.execute_reply":"2023-07-02T15:21:07.738415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.array(minl).mean(), np.array(maxl).mean()","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:21:07.741374Z","iopub.execute_input":"2023-07-02T15:21:07.742200Z","iopub.status.idle":"2023-07-02T15:21:07.751333Z","shell.execute_reply.started":"2023-07-02T15:21:07.742140Z","shell.execute_reply":"2023-07-02T15:21:07.750138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from numpy import nan","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:21:07.753469Z","iopub.execute_input":"2023-07-02T15:21:07.753937Z","iopub.status.idle":"2023-07-02T15:21:07.763086Z","shell.execute_reply.started":"2023-07-02T15:21:07.753898Z","shell.execute_reply":"2023-07-02T15:21:07.761928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"str(float(\"NaN\")) == str(nan)\n","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:21:07.766561Z","iopub.execute_input":"2023-07-02T15:21:07.766863Z","iopub.status.idle":"2023-07-02T15:21:07.777277Z","shell.execute_reply.started":"2023-07-02T15:21:07.766836Z","shell.execute_reply":"2023-07-02T15:21:07.776133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bosl = [[lm[lm.type == \"left_hand\"][\"x\"].iloc[0], 4], [6, 5], [9, nan]]\nbosl = [0 if str(item) == \"nan\" else item for item in bosl]\nbosl","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:21:07.778586Z","iopub.execute_input":"2023-07-02T15:21:07.780000Z","iopub.status.idle":"2023-07-02T15:21:07.792448Z","shell.execute_reply.started":"2023-07-02T15:21:07.779961Z","shell.execute_reply":"2023-07-02T15:21:07.791238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.nanmean(lm[lm.type == \"left_hand\"][\"x\"])","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:21:07.796749Z","iopub.execute_input":"2023-07-02T15:21:07.797849Z","iopub.status.idle":"2023-07-02T15:21:07.808100Z","shell.execute_reply.started":"2023-07-02T15:21:07.797807Z","shell.execute_reply":"2023-07-02T15:21:07.806910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.math.reduce_sum(tf.where(tf.math.is_nan(bosl)))","metadata":{"execution":{"iopub.status.busy":"2023-07-02T15:44:29.120352Z","iopub.execute_input":"2023-07-02T15:44:29.120741Z","iopub.status.idle":"2023-07-02T15:44:29.131616Z","shell.execute_reply.started":"2023-07-02T15:44:29.120708Z","shell.execute_reply":"2023-07-02T15:44:29.130524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}}]}