{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":46105,"databundleVersionId":5087314,"sourceType":"competition"},{"sourceId":8271013,"sourceType":"datasetVersion","datasetId":4910685},{"sourceId":8287937,"sourceType":"datasetVersion","datasetId":4922852}],"dockerImageVersionId":30664,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Sign Language Recognition Challenge\n\nThis notebook was created during a live coding steam. Watch it on my youtube and twitch page:\n- [Watch on Youtube](http://www.youtube.com/@robmulla?sub_confirmation=1)\n- Live Coding on Twitch\n\nThe goal of this competition is to classify isolated American Sign Language (ASL) signs.\n\nThe landmarks were extracted from raw videos with the MediaPipe holistic model and are asked to predict the sign from this data.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.notebook import tqdm\n\n# plt.style.use(\"seaborn-colorblind\")\n# Set the seaborn style\nsns.set_style(\"white\")\n\n# Define a colorblind-friendly palette\ncolor_palette = sns.color_palette(\"colorblind\")\n\n# Set the color palette\nsns.set_palette(color_palette)","metadata":{"execution":{"iopub.status.busy":"2024-06-04T06:57:06.528622Z","iopub.execute_input":"2024-06-04T06:57:06.529279Z","iopub.status.idle":"2024-06-04T06:57:06.536883Z","shell.execute_reply.started":"2024-06-04T06:57:06.529233Z","shell.execute_reply":"2024-06-04T06:57:06.535326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Install nb_black for autoformating\n# !pip install nb_black --quiet\n# %load_ext lab_black\n!pip install black[jupyter]","metadata":{"execution":{"iopub.status.busy":"2024-06-04T06:57:06.539084Z","iopub.execute_input":"2024-06-04T06:57:06.539482Z","iopub.status.idle":"2024-06-04T06:57:22.200489Z","shell.execute_reply.started":"2024-06-04T06:57:06.53945Z","shell.execute_reply":"2024-06-04T06:57:22.199096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data EDA","metadata":{}},{"cell_type":"code","source":"!ls ../input/asl-signs/ -GFlash --color","metadata":{"execution":{"iopub.status.busy":"2024-06-04T06:57:22.202362Z","iopub.execute_input":"2024-06-04T06:57:22.20286Z","iopub.status.idle":"2024-06-04T06:57:23.238097Z","shell.execute_reply.started":"2024-06-04T06:57:22.202817Z","shell.execute_reply":"2024-06-04T06:57:23.236701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_DIR = '../input/asl-signs/'\ntrain = pd.read_csv(f'{BASE_DIR}/train.csv')","metadata":{"execution":{"iopub.status.busy":"2024-06-04T06:57:23.241103Z","iopub.execute_input":"2024-06-04T06:57:23.241489Z","iopub.status.idle":"2024-06-04T06:57:23.472069Z","shell.execute_reply.started":"2024-06-04T06:57:23.241454Z","shell.execute_reply":"2024-06-04T06:57:23.470934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train.csv has the path to each parquet file, the particpant id, sequence_id and sign.\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2024-06-04T06:57:23.473486Z","iopub.execute_input":"2024-06-04T06:57:23.47384Z","iopub.status.idle":"2024-06-04T06:57:23.494959Z","shell.execute_reply.started":"2024-06-04T06:57:23.473802Z","shell.execute_reply":"2024-06-04T06:57:23.493941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## What Signs are we trying to predict?\n- 250 Unique Signs\n- Ranging from 299-415 Examples of Each","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(8, 8))\ntrain[\"sign\"].value_counts().head(50).sort_values(ascending=True).plot(\n    kind=\"barh\", ax=ax, title=\"Top 50 Signs in Training Dataset\"\n)\nax.set_xlabel(\"Number of Training Examples\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-04T06:57:23.496392Z","iopub.execute_input":"2024-06-04T06:57:23.496684Z","iopub.status.idle":"2024-06-04T06:57:24.404525Z","shell.execute_reply.started":"2024-06-04T06:57:23.496659Z","shell.execute_reply":"2024-06-04T06:57:24.403121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(8, 8))\ntrain[\"sign\"].value_counts().tail(50).sort_values(ascending=True).plot(\n    kind=\"barh\", ax=ax, title=\"Bottom 50 Signs in Training Dataset\"\n)\nax.set_xlabel(\"Number of Training Examples\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-04T06:57:24.405926Z","iopub.execute_input":"2024-06-04T06:57:24.406426Z","iopub.status.idle":"2024-06-04T06:57:25.131999Z","shell.execute_reply.started":"2024-06-04T06:57:24.406388Z","shell.execute_reply":"2024-06-04T06:57:25.130574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Parquent Landmark Data\n- Each Parquet file is in the path:\n    - train_landmark_files/[participant_id]/[sequence_id].parquet\n- The parquet's associated sign can be found in train.csv","metadata":{}},{"cell_type":"markdown","source":"## Pull an example parquet file data...\n\nWe pull an example landmark file for the sign \"listen\"","metadata":{}},{"cell_type":"code","source":"example_fn = train.query('sign == \"listen\"')[\"path\"].values[0]\n\nexample_landmark = pd.read_parquet(f\"{BASE_DIR}/{example_fn}\")\nexample_landmark.head()","metadata":{"execution":{"iopub.status.busy":"2024-06-04T06:57:25.133554Z","iopub.execute_input":"2024-06-04T06:57:25.134019Z","iopub.status.idle":"2024-06-04T06:57:25.33494Z","shell.execute_reply.started":"2024-06-04T06:57:25.133976Z","shell.execute_reply":"2024-06-04T06:57:25.333853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_frames = example_landmark[\"frame\"].nunique()\nunique_types = example_landmark[\"type\"].nunique()\ntypes_in_video = example_landmark[\"type\"].unique()\nprint(\n    f\"The file has {unique_frames} unique frames and {unique_types} unique types: {types_in_video}\"\n)","metadata":{"execution":{"iopub.status.busy":"2024-06-04T06:57:25.336588Z","iopub.execute_input":"2024-06-04T06:57:25.336946Z","iopub.status.idle":"2024-06-04T06:57:25.346526Z","shell.execute_reply.started":"2024-06-04T06:57:25.336916Z","shell.execute_reply":"2024-06-04T06:57:25.345434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Lets Compare for a bunch of parquet files what type of data we have.\n- We notice the number of frames is not consistent\n- Almost every file has 4 types of landmarks: face, left_hands, pose and right_hands.","metadata":{}},{"cell_type":"code","source":"listen_files = train.query('sign == \"listen\"')[\"path\"].values\nfor i, f in enumerate(listen_files):\n    example_landmark = pd.read_parquet(f\"{BASE_DIR}/{f}\")\n    unique_frames = example_landmark[\"frame\"].nunique()\n    unique_types = example_landmark[\"type\"].nunique()\n    types_in_video = example_landmark[\"type\"].unique()\n    print(\n        f\"The file has {unique_frames} unique frames and {unique_types} unique types: {types_in_video}\"\n    )\n    if i == 20:\n        break","metadata":{"execution":{"iopub.status.busy":"2024-06-04T06:57:25.350297Z","iopub.execute_input":"2024-06-04T06:57:25.350668Z","iopub.status.idle":"2024-06-04T06:57:26.070735Z","shell.execute_reply.started":"2024-06-04T06:57:25.350638Z","shell.execute_reply":"2024-06-04T06:57:26.069689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create Metadata for Training Dataset","metadata":{}},{"cell_type":"code","source":"N_PARQUETS_TO_READ = 100_000 # So we don't have to load all 95k\n\ncombined_meta = {}\nfor i, d in tqdm(train.iterrows(), total=len(train)):\n    file_path = d[\"path\"]\n    example_landmark = pd.read_parquet(f\"{BASE_DIR}/{file_path}\")\n    # Get the number of landmarks with x,y,z data per type\n    meta = (\n        example_landmark.dropna(subset=[\"x\", \"y\", \"z\"])[\"type\"].value_counts().to_dict()\n    )\n    meta[\"frames\"] = example_landmark[\"frame\"].nunique()\n    xyz_meta = (\n        example_landmark.agg(\n            {\n                \"x\": [\"min\", \"max\", \"mean\"], \n                \"y\": [\"min\", \"max\", \"mean\"], \n                \"z\": [\"min\", \"max\", \"mean\"], \n        }\n    )\n    .unstack()\n    .to_dict()\n)\n\n    for key in xyz_meta.keys():\n        new_key = key[0] + \"-\" + key[1]\n        meta[new_key] = xyz_meta[key]\n    combined_meta[file_path] = meta\n    if i >= N_PARQUETS_TO_READ:\n        break","metadata":{"execution":{"iopub.status.busy":"2024-06-04T06:57:26.072228Z","iopub.execute_input":"2024-06-04T06:57:26.072565Z","iopub.status.idle":"2024-06-04T08:08:23.329268Z","shell.execute_reply.started":"2024-06-04T06:57:26.072538Z","shell.execute_reply":"2024-06-04T08:08:23.326037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_with_meta = train.merge(\n    pd.DataFrame(combined_meta).T.reset_index().rename(columns={\"index\": \"path\"}),\n    how=\"left\",\n)\ntrain_with_meta.to_parquet(\"train_with_meta.parquet\")","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:08:23.333909Z","iopub.execute_input":"2024-06-04T08:08:23.334439Z","iopub.status.idle":"2024-06-04T08:08:27.099096Z","shell.execute_reply.started":"2024-06-04T08:08:23.334386Z","shell.execute_reply":"2024-06-04T08:08:27.097522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## What are the most frequent types of landmarks provided?\n- Face has a lot more datapoints because mediapipe provides 468 3D datapoints per frame.","metadata":{}},{"cell_type":"code","source":"train_with_meta[[\"face\", \"pose\", \"left_hand\", \"right_hand\"]].sum().sort_values().plot(\n    kind=\"barh\", title=\"Sum of Rows by Landmark Type\"\n)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:08:27.101351Z","iopub.execute_input":"2024-06-04T08:08:27.101904Z","iopub.status.idle":"2024-06-04T08:08:27.460708Z","shell.execute_reply.started":"2024-06-04T08:08:27.101861Z","shell.execute_reply":"2024-06-04T08:08:27.45958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Every parquet file has at least some datapoints for all four types of landmarks:\n","metadata":{}},{"cell_type":"markdown","source":"- Face, pose, left hand and right hand.","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"# checking to see if the number of landmarks for this type is zero\n(\n    train_with_meta.query(\"index < 1000\").fillna(0)[\n        [\"face\", \"pose\", \"left_hand\", \"right_hand\"]\n    ] \n    > 0\n).mean().plot(kind=\"barh\", title=\"Rate of Frame/Keypoints with Data\")","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:08:27.462339Z","iopub.execute_input":"2024-06-04T08:08:27.462806Z","iopub.status.idle":"2024-06-04T08:08:27.827843Z","shell.execute_reply.started":"2024-06-04T08:08:27.462746Z","shell.execute_reply":"2024-06-04T08:08:27.826452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Check out one Example","metadata":{}},{"cell_type":"code","source":"example_fn = train_with_meta.dropna().query('sign == \"shhh\"')[\"path\"].values[0]\nexample_landmark = pd.read_parquet(f\"{BASE_DIR}/{example_fn}\")","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:08:27.829555Z","iopub.execute_input":"2024-06-04T08:08:27.830436Z","iopub.status.idle":"2024-06-04T08:08:27.881072Z","shell.execute_reply.started":"2024-06-04T08:08:27.830388Z","shell.execute_reply":"2024-06-04T08:08:27.879864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark.query(\"frame == 25\")[\"type\"].value_counts() # Middle of the video","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:08:27.882539Z","iopub.execute_input":"2024-06-04T08:08:27.882918Z","iopub.status.idle":"2024-06-04T08:08:27.897537Z","shell.execute_reply.started":"2024-06-04T08:08:27.882888Z","shell.execute_reply":"2024-06-04T08:08:27.896213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark[\"no_xyz\"] = example_landmark[\"x\"].isna()","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:08:27.898916Z","iopub.execute_input":"2024-06-04T08:08:27.899239Z","iopub.status.idle":"2024-06-04T08:08:27.905399Z","shell.execute_reply.started":"2024-06-04T08:08:27.899213Z","shell.execute_reply":"2024-06-04T08:08:27.904289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark.groupby(\"frame\")[\"no_xyz\"].sum().plot(\n    title=\"missing xyz per frame\", kind=\"bar\"\n)","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:08:27.906968Z","iopub.execute_input":"2024-06-04T08:08:27.9074Z","iopub.status.idle":"2024-06-04T08:08:28.4775Z","shell.execute_reply.started":"2024-06-04T08:08:27.907359Z","shell.execute_reply":"2024-06-04T08:08:28.47618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3D plot of Landmarks from \"shhh\" example\nPick frame 17 because we have no missing xyz data","metadata":{}},{"cell_type":"code","source":"import plotly.express as px\n\nexample_frame = example_landmark.query(\"frame == 17\")\npx.scatter_3d(example_frame, x=\"x\", y=\"y\", z=\"z\", color=\"type\")","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:08:28.478911Z","iopub.execute_input":"2024-06-04T08:08:28.479265Z","iopub.status.idle":"2024-06-04T08:08:31.425443Z","shell.execute_reply.started":"2024-06-04T08:08:28.479234Z","shell.execute_reply":"2024-06-04T08:08:31.42416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark[\"y_\"] = example_landmark[\"y\"] * -1\nexample_frame = example_landmark.query(\"frame == 17 and type== 'face'\")\npx.scatter(example_frame, x=\"x\", y=\"y_\", color=\"type\")","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:08:31.427049Z","iopub.execute_input":"2024-06-04T08:08:31.427515Z","iopub.status.idle":"2024-06-04T08:08:31.563449Z","shell.execute_reply.started":"2024-06-04T08:08:31.427473Z","shell.execute_reply":"2024-06-04T08:08:31.561877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Try to draw the example with mediapipe's hand connections?","metadata":{}},{"cell_type":"code","source":"!pip install mediapipe --quiet","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:08:31.565313Z","iopub.execute_input":"2024-06-04T08:08:31.565817Z","iopub.status.idle":"2024-06-04T08:09:07.864791Z","shell.execute_reply.started":"2024-06-04T08:08:31.56576Z","shell.execute_reply":"2024-06-04T08:09:07.863086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import mediapipe as mp\nmp_hands = mp.solutions.hands\n\n\nexample_landmark[\"y_\"] = example_landmark[\"y\"] * -1\n\nfig, ax = plt.subplots(figsize=(5, 5))\n\nfor hand in [\"left_hand\", \"right_hand\"]:\nexample_hand = example_landmark.query(\"frame == 17 and type == @hand\")\n\n\nax.scatter(example_hand[\"x\"], example_hand[\"y_\"])\n\nfor connection in mp_hands.HAND_CONNECTIONS:\n    point_a = connection[0]\n    point_b = connection[1]\n    x1, y1 = example_hand.query(\"landmark_index == @point_a\")[[\"x\", \"y_\"]].values[0]\n    x2, y2 = example_hand.query(\"landmark_index == @point_b\")[[\"x\", \"y_\"]].values[0]\n    plt.plot([x1,x2],[y1,y2], color=\"purple\")\nax.set_title(\"Shhh - Hands Data\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:09:07.866861Z","iopub.execute_input":"2024-06-04T08:09:07.867266Z","iopub.status.idle":"2024-06-04T08:09:07.87916Z","shell.execute_reply.started":"2024-06-04T08:09:07.86723Z","shell.execute_reply":"2024-06-04T08:09:07.876877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Try to use mediapipe to plot\n- Pull some example images\n- Run mediapipe holistic to see how it produces the results\n- plot them on the image","metadata":{}},{"cell_type":"code","source":"!wget https://i.ytimg.com/vi/mi9f9zOaqM8/hqdefault.jpg --quiet\n!wget https://previews.123rf.com/images/mimagephotography/mimagephotography1411/mimagephotography141100022/33214722-full-length-portrait-of-a-fashionable-young-man-standing-on-isolated-white-background.jpg --quiet\n","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:22:05.132796Z","iopub.execute_input":"2024-06-04T08:22:05.133328Z","iopub.status.idle":"2024-06-04T08:22:08.058335Z","shell.execute_reply.started":"2024-06-04T08:22:05.133287Z","shell.execute_reply":"2024-06-04T08:22:08.056748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nimport mediapipe as mp\nmp_drawing = mp.solutions.drawing_utils\nmp_drawing_styles = mp.solutions.drawing_styles\nmp_holistic = mp.solutions.holistic\n\n# For static images:\nIMAGE_FILES = [\n    \"hqdefault.jpg\",\n    \"33214722-full-length-portrait-of-a-fashionable-young-man-standing-on-isolated-white-background.jpg\"\n]\nBG_COLOR = (192, 192, 192) # gray\nwith mp_holistic.Holistic(\n    static_image_mode=True,\n    model_complexity=2,\n    enable_segmentation=True,\n    refine_face_landmarks=True,\n) as holistic:\n  for idx, file in enumerate(IMAGE_FILES):\n    image = cv2.imread(file)\n    image_height, image_width, _ = image.shape\n    # Convert the BGR image to RGB before processing.\n    results = holistic.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))\n\n    if results.pose_landmarks:\n      print(\n          f\"Nose coordinates: (\"\n          f\"{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].x * image_width}, \"\n          f\"{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].y * image_height})\"\n      )\n\n    annotated_image = image.copy()\n    # Draw segmentation on the image.\n    # To improve segmentation around boundaries, consider applying a joint\n    \n    # bilateral filter to \"results.segmentation_mask\" with \"image\".\n    condition = np.stack((results.segmentation_mask,) * 3, axis=-1) > 0.1\n    bg_image = np.zeros(image.shape, dtype=np.uint8)\n    bg_image[:] = BG_COLOR\n    annotated_image = np.where(condition, annotated_image, bg_image)\n    # Draw pose, left and right hands, and face landmarks on the image.\n    mp_drawing.draw_landmarks(\n        annotated_image,\n        results.face_landmarks,\n        mp_holistic.FACEMESH_TESSELATION,\n        landmark_drawing_spec=None,\n        connection_drawing_spec=mp_drawing_styles.get_default_face_mesh_tesselation_style(),\n    )\n    mp_drawing.draw_landmarks(\n        annotated_image,\n        results.pose_landmarks,\n        mp_holistic.POSE_CONNECTIONS,\n        landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style(),\n    )\n    cv2.imwrite(\"/tmp/annotated_image\" + str(idx) + \".png\", annotated_image)\n    # Plot pose world landmarks.\n#     mp_drawing.plot_landmarks(\n#         results.pose_world_landmarks, mp_holistic.POSE_CONNECTIONS\n#     )","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:22:13.767609Z","iopub.execute_input":"2024-06-04T08:22:13.768153Z","iopub.status.idle":"2024-06-04T08:22:32.87474Z","shell.execute_reply.started":"2024-06-04T08:22:13.768112Z","shell.execute_reply":"2024-06-04T08:22:32.873396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(plt.imread(\"/tmp/annotated_image\" + str(0) + \".png\"))\nplt.show()\n\nplt.imshow(plt.imread(\"/tmp/annotated_image\" + str(1) + \".png\"))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:22:37.760891Z","iopub.execute_input":"2024-06-04T08:22:37.76172Z","iopub.status.idle":"2024-06-04T08:22:38.622897Z","shell.execute_reply.started":"2024-06-04T08:22:37.761681Z","shell.execute_reply":"2024-06-04T08:22:38.620991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Try to use the same format for plotting of parquet data","metadata":{}},{"cell_type":"code","source":"background_image = np.zeros([720, 720, 3])\n\nmp_drawing.draw_landmarks(\n    background_image,\n    results.face_landmarks,\n    mp_holistic.FACEMESH_TESSELATION,\n    landmark_drawing_spec=None,\n    connection_drawing_spec=mp_drawing_styles\n    .get_default_face_mesh_tesselation_style(),\n)\nmp_drawing.draw_landmarks(\n    background_image,\n    results.pose_landmarks,\n    mp_holistic.POSE_CONNECTIONS,\n    landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style(),\n)\nplt.imshow(background_image)","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:22:42.561899Z","iopub.execute_input":"2024-06-04T08:22:42.562317Z","iopub.status.idle":"2024-06-04T08:22:43.116029Z","shell.execute_reply.started":"2024-06-04T08:22:42.562283Z","shell.execute_reply":"2024-06-04T08:22:43.114641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type(results.face_landmarks)\n\nfrom mediapipe.framework.formats import landmark_pb2\n\n# face_landmarks = landmark_pb2.NormalizedLandmarkList(example_frame.query('type == \"face\"')[[\"x\", \"y\", \"z\"]].values)","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:22:46.208894Z","iopub.execute_input":"2024-06-04T08:22:46.209396Z","iopub.status.idle":"2024-06-04T08:22:46.21715Z","shell.execute_reply.started":"2024-06-04T08:22:46.209354Z","shell.execute_reply":"2024-06-04T08:22:46.215319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## TODO\n- Figure out how to transform the parquet file data into mediapipe  `NormalizedLandmarkList`","metadata":{}},{"cell_type":"markdown","source":"\n\n# Evalution\n\nThe evaluation metric for this contest is simple classification accuracy.\nBelow code was taken from the evaluation page.","metadata":{}},{"cell_type":"code","source":"def load_relevant_data_subset(pq_path):\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:22:49.075303Z","iopub.execute_input":"2024-06-04T08:22:49.075739Z","iopub.status.idle":"2024-06-04T08:22:49.083108Z","shell.execute_reply.started":"2024-06-04T08:22:49.075699Z","shell.execute_reply":"2024-06-04T08:22:49.081592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import tflite_runtime.interpreter as tflite\n\n# def run_model(model.path)\n#     interpreter = tflite.Interpreter(model_path)\n\n#     found_signatures = list(interpreter.get_signature_list().keys())\n\n#     if REQUIRED_SIGNATURE not in found_signatures:\n#         raise KernelEvalException('Required input signature not found.')\n\n#     prediction_fn = interpreter.get_signature_runner(\"serving_default\")\n#     output = prediction_fn(inputs=frames)\n#     sign = np.argmax(output[\"outputs\"])","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:22:51.507789Z","iopub.execute_input":"2024-06-04T08:22:51.5086Z","iopub.status.idle":"2024-06-04T08:22:51.513813Z","shell.execute_reply.started":"2024-06-04T08:22:51.508556Z","shell.execute_reply":"2024-06-04T08:22:51.512417Z"},"trusted":true},"execution_count":null,"outputs":[]}]}