{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.notebook import tqdm\n\nplt.style.use(\"seaborn-colorblind\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-06T13:03:15.620867Z","iopub.execute_input":"2023-09-06T13:03:15.622302Z","iopub.status.idle":"2023-09-06T13:03:15.636035Z","shell.execute_reply.started":"2023-09-06T13:03:15.622253Z","shell.execute_reply":"2023-09-06T13:03:15.634781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Install nb_black for autoformatting\n!pip install nb_black --quiet\n%load_ext lab_black","metadata":{"execution":{"iopub.status.busy":"2023-09-06T13:03:35.732861Z","iopub.execute_input":"2023-09-06T13:03:35.733367Z","iopub.status.idle":"2023-09-06T13:03:47.608218Z","shell.execute_reply.started":"2023-09-06T13:03:35.733326Z","shell.execute_reply":"2023-09-06T13:03:47.606676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls ../input/asl-signs/ -GFlash --color","metadata":{"execution":{"iopub.status.busy":"2023-09-06T13:03:52.574898Z","iopub.execute_input":"2023-09-06T13:03:52.575455Z","iopub.status.idle":"2023-09-06T13:03:53.635428Z","shell.execute_reply.started":"2023-09-06T13:03:52.575409Z","shell.execute_reply":"2023-09-06T13:03:53.633909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_DIR = \"../input/asl-signs/\"\ntrain = pd.read_csv(f\"{BASE_DIR}/train.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-09-06T13:04:06.368626Z","iopub.execute_input":"2023-09-06T13:04:06.369169Z","iopub.status.idle":"2023-09-06T13:04:06.645858Z","shell.execute_reply.started":"2023-09-06T13:04:06.369122Z","shell.execute_reply":"2023-09-06T13:04:06.644515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2023-09-06T13:04:21.338676Z","iopub.execute_input":"2023-09-06T13:04:21.339451Z","iopub.status.idle":"2023-09-06T13:04:21.361621Z","shell.execute_reply.started":"2023-09-06T13:04:21.33941Z","shell.execute_reply":"2023-09-06T13:04:21.360406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(8, 8))\ntrain[\"sign\"].value_counts().head(50).sort_values(ascending=True).plot(\n    kind=\"barh\", ax=ax, title=\"Top 50 Signs in Training Dataset\"\n)\nax.set_xlabel(\"Number of Training Examples\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-06T13:04:55.035328Z","iopub.execute_input":"2023-09-06T13:04:55.035801Z","iopub.status.idle":"2023-09-06T13:04:55.737932Z","shell.execute_reply.started":"2023-09-06T13:04:55.035765Z","shell.execute_reply":"2023-09-06T13:04:55.736794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(8, 8))\ntrain[\"sign\"].value_counts().tail(50).sort_values(ascending=True).plot(\n    kind=\"barh\", ax=ax, title=\"Bottom 50 Signs in Training Dataset\"\n)\nax.set_xlabel(\"Number of Training Examples\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-06T13:05:11.381644Z","iopub.execute_input":"2023-09-06T13:05:11.382109Z","iopub.status.idle":"2023-09-06T13:05:12.70624Z","shell.execute_reply.started":"2023-09-06T13:05:11.382072Z","shell.execute_reply":"2023-09-06T13:05:12.704869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Parquet Landmark Data ","metadata":{}},{"cell_type":"code","source":"example_fn = train.query('sign == \"listen\"')[\"path\"].values[0]\n\nexample_landmark = pd.read_parquet(f\"{BASE_DIR}/{example_fn}\")\nexample_landmark.head()","metadata":{"execution":{"iopub.status.busy":"2023-09-06T13:05:25.371114Z","iopub.execute_input":"2023-09-06T13:05:25.37158Z","iopub.status.idle":"2023-09-06T13:05:25.419539Z","shell.execute_reply.started":"2023-09-06T13:05:25.371543Z","shell.execute_reply":"2023-09-06T13:05:25.418123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_frames = example_landmark[\"frame\"].nunique()\nunique_types = example_landmark[\"type\"].nunique()\ntypes_in_video = example_landmark[\"type\"].unique()\nprint(\n    f\"The file has {unique_frames} unique frames and {unique_types} unique types: {types_in_video}\"\n)","metadata":{"execution":{"iopub.status.busy":"2023-09-06T13:05:37.913919Z","iopub.execute_input":"2023-09-06T13:05:37.915049Z","iopub.status.idle":"2023-09-06T13:05:37.933461Z","shell.execute_reply.started":"2023-09-06T13:05:37.914996Z","shell.execute_reply":"2023-09-06T13:05:37.932371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Compare parquet files ","metadata":{}},{"cell_type":"code","source":"listen_files = train.query('sign == \"listen\"')[\"path\"].values\nfor i, f in enumerate(listen_files):\n    example_landmark = pd.read_parquet(f\"{BASE_DIR}/{f}\")\n    unique_frames = example_landmark[\"frame\"].nunique()\n    unique_types = example_landmark[\"type\"].nunique()\n    types_in_video = example_landmark[\"type\"].unique()\n    print(\n        f\"The file has {unique_frames} unique frames and {unique_types} unique types: {types_in_video}\"\n    )\n    if i == 20:\n        break","metadata":{"execution":{"iopub.status.busy":"2023-09-06T13:05:52.586359Z","iopub.execute_input":"2023-09-06T13:05:52.586838Z","iopub.status.idle":"2023-09-06T13:05:53.311077Z","shell.execute_reply.started":"2023-09-06T13:05:52.586798Z","shell.execute_reply":"2023-09-06T13:05:53.309635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" Metadata for Training Datase","metadata":{}},{"cell_type":"code","source":"N_PARQUETS_TO_READ = 100_000  # So we don't have to load all 95k\n\ncombined_meta = {}\nfor i, d in tqdm(train.iterrows(), total=len(train)):\n    file_path = d[\"path\"]\n    example_landmark = pd.read_parquet(f\"{BASE_DIR}/{file_path}\")\n    # Get the number of landmarks with x,y,z data per type\n    meta = (\n        example_landmark.dropna(subset=[\"x\", \"y\", \"z\"])[\"type\"].value_counts().to_dict()\n    )\n    meta[\"frames\"] = example_landmark[\"frame\"].nunique()\n    xyz_meta = (\n        example_landmark.agg(\n            {\n                \"x\": [\"min\", \"max\", \"mean\"],\n                \"y\": [\"min\", \"max\", \"mean\"],\n                \"z\": [\"min\", \"max\", \"mean\"],\n            }\n        )\n        .unstack()\n        .to_dict()\n    )\n\n    for key in xyz_meta.keys():\n        new_key = key[0] + \"_\" + key[1]\n        meta[new_key] = xyz_meta[key]\n    combined_meta[file_path] = meta\n    if i >= N_PARQUETS_TO_READ:\n        break","metadata":{"execution":{"iopub.status.busy":"2023-09-06T11:59:35.63775Z","iopub.execute_input":"2023-09-06T11:59:35.638416Z","iopub.status.idle":"2023-09-06T12:00:06.0939Z","shell.execute_reply.started":"2023-09-06T11:59:35.638365Z","shell.execute_reply":"2023-09-06T12:00:06.09106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"training dataset","metadata":{}},{"cell_type":"code","source":"train_with_meta = train.merge(\n    pd.DataFrame(combined_meta).T.reset_index().rename(columns={\"index\": \"path\"}),\n    how=\"left\",\n)\ntrain_with_meta.to_parquet(\"train_with_meta.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-09-06T13:06:16.071403Z","iopub.execute_input":"2023-09-06T13:06:16.071888Z","iopub.status.idle":"2023-09-06T13:06:16.285597Z","shell.execute_reply.started":"2023-09-06T13:06:16.071849Z","shell.execute_reply":"2023-09-06T13:06:16.284336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_with_meta[[\"face\", \"pose\", \"left_hand\", \"right_hand\"]].sum().sort_values().plot(\n    kind=\"barh\", title=\"Sum of Rows by Landmark Type\"\n)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-06T13:06:30.34872Z","iopub.execute_input":"2023-09-06T13:06:30.349239Z","iopub.status.idle":"2023-09-06T13:06:30.589555Z","shell.execute_reply.started":"2023-09-06T13:06:30.349194Z","shell.execute_reply":"2023-09-06T13:06:30.588413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# checking to see if the number of landmarks for this type is zero\n(\n    train_with_meta.query(\"index < 1000\").fillna(0)[\n        [\"face\", \"pose\", \"left_hand\", \"right_hand\"]\n    ]\n    > 0\n).mean().plot(kind=\"barh\", title=\"Rate of Frame/Keypoints with Data\")","metadata":{"execution":{"iopub.status.busy":"2023-09-06T13:06:45.992244Z","iopub.execute_input":"2023-09-06T13:06:45.99273Z","iopub.status.idle":"2023-09-06T13:06:46.269449Z","shell.execute_reply.started":"2023-09-06T13:06:45.99269Z","shell.execute_reply":"2023-09-06T13:06:46.268142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_fn = train_with_meta.dropna().query('sign == \"shhh\"')[\"path\"].values[0]\nexample_landmark = pd.read_parquet(f\"{BASE_DIR}/{example_fn}\")","metadata":{"execution":{"iopub.status.busy":"2023-09-06T13:07:39.730388Z","iopub.execute_input":"2023-09-06T13:07:39.730878Z","iopub.status.idle":"2023-09-06T13:07:39.784593Z","shell.execute_reply.started":"2023-09-06T13:07:39.730836Z","shell.execute_reply":"2023-09-06T13:07:39.783308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark.query(\"frame == 25\")[\"type\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-09-06T13:07:57.060512Z","iopub.execute_input":"2023-09-06T13:07:57.061032Z","iopub.status.idle":"2023-09-06T13:07:57.081725Z","shell.execute_reply.started":"2023-09-06T13:07:57.060967Z","shell.execute_reply":"2023-09-06T13:07:57.080544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark.query(\"frame == 25\")[\"type\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-09-06T12:00:42.831686Z","iopub.execute_input":"2023-09-06T12:00:42.832192Z","iopub.status.idle":"2023-09-06T12:00:42.853578Z","shell.execute_reply.started":"2023-09-06T12:00:42.832153Z","shell.execute_reply":"2023-09-06T12:00:42.852063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark[\"no_xyz\"] = example_landmark[\"x\"].isna()","metadata":{"execution":{"iopub.status.busy":"2023-09-06T13:08:10.253421Z","iopub.execute_input":"2023-09-06T13:08:10.254471Z","iopub.status.idle":"2023-09-06T13:08:10.267437Z","shell.execute_reply.started":"2023-09-06T13:08:10.254427Z","shell.execute_reply":"2023-09-06T13:08:10.266063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark.groupby(\"frame\")[\"no_xyz\"].sum().plot(\n    title=\"missing xyz per frame\", kind=\"bar\"\n)","metadata":{"execution":{"iopub.status.busy":"2023-09-06T13:08:15.08867Z","iopub.execute_input":"2023-09-06T13:08:15.090081Z","iopub.status.idle":"2023-09-06T13:08:15.567647Z","shell.execute_reply.started":"2023-09-06T13:08:15.090017Z","shell.execute_reply":"2023-09-06T13:08:15.56627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.express as px\n\nexample_frame = example_landmark.query(\"frame == 17\")\npx.scatter_3d(example_frame, x=\"x\", y=\"y\", z=\"z\", color=\"type\")","metadata":{"execution":{"iopub.status.busy":"2023-09-06T13:08:29.991876Z","iopub.execute_input":"2023-09-06T13:08:29.9934Z","iopub.status.idle":"2023-09-06T13:08:30.087791Z","shell.execute_reply.started":"2023-09-06T13:08:29.993344Z","shell.execute_reply":"2023-09-06T13:08:30.086816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark[\"y_\"] = example_landmark[\"y\"] * -1\nexample_frame = example_landmark.query(\"frame == 17 and type== 'face'\")\npx.scatter(example_frame, x=\"x\", y=\"y_\", color=\"type\")","metadata":{"execution":{"iopub.status.busy":"2023-09-06T13:09:33.713252Z","iopub.execute_input":"2023-09-06T13:09:33.713761Z","iopub.status.idle":"2023-09-06T13:09:33.803371Z","shell.execute_reply.started":"2023-09-06T13:09:33.713715Z","shell.execute_reply":"2023-09-06T13:09:33.802088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install mediapipe --quiet","metadata":{"execution":{"iopub.status.busy":"2023-09-06T13:09:45.628827Z","iopub.execute_input":"2023-09-06T13:09:45.629331Z","iopub.status.idle":"2023-09-06T13:09:57.300997Z","shell.execute_reply.started":"2023-09-06T13:09:45.629286Z","shell.execute_reply":"2023-09-06T13:09:57.299447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip list | grep tensorflow","metadata":{"execution":{"iopub.status.busy":"2023-09-06T13:14:16.330511Z","iopub.execute_input":"2023-09-06T13:14:16.331437Z","iopub.status.idle":"2023-09-06T13:14:19.792282Z","shell.execute_reply.started":"2023-09-06T13:14:16.33138Z","shell.execute_reply":"2023-09-06T13:14:19.790643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip uninstall tensorflow tensorflow-addons tensorflow-cloud tensorflow-datasets tensorflow-decision-forests tensorflow-estimator tensorflow-hub tensorflow-io tensorflow-io-gcs-filesystem tensorflow-metadata tensorflow-probability tensorflow-serving-api tensorflow-text tensorflow-transform tensorflowjs --y\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install mediapipe","metadata":{"execution":{"iopub.status.busy":"2023-09-06T12:01:51.551875Z","iopub.execute_input":"2023-09-06T12:01:51.552378Z","iopub.status.idle":"2023-09-06T12:02:03.271932Z","shell.execute_reply.started":"2023-09-06T12:01:51.552339Z","shell.execute_reply":"2023-09-06T12:02:03.270401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import mediapipe as mp\nfrom mediapipe.tasks import python\nfrom mediapipe.tasks.python import audio\nmp_hands = mp.solutions.hands\n\n\nexample_landmark[\"y_\"] = example_landmark[\"y\"] * -1\n\nfig, ax = plt.subplots(figsize=(5, 5))\n\naudio_classifier = mp.tasks.audio.audio_classifier\naudio_classifierOptions = mp.tasks.audio_classifierOptions\naudio_classifierResult = mp.tasks.audio.audio_classifierResult\nAudioRunningMode = mp.tasks.audio.RunningMode\nBaseOptions = mp.tasks.BaseOptions\n\ndef print_result(result: audio_classifierResult, timestamp_ms: int):\n    print('audio_classifierResult result: {}’.format(result))\n\noptions = audio_classifierOptions(\n    base_options=BaseOptions(model_asset_path='/path/to/model.tflite'),\n    running_mode=AudioRunningMode.AUDIO_STREAM,\n    max_results=5,\n    result_callback=print_result)\n\nwith audio_classifier.create_from_options(options) as classifier:\n\n\nfor hand in [\"left_hand\", \"right_hand\"]:\n    example_hand = example_landmark.query(\"frame == 17 and type == @hand\")\n\n    ax.scatter(example_hand[\"x\"], example_hand[\"y_\"])\n\n    for connection in mp_hands.HAND_CONNECTIONS:\n        point_a = connection[0]\n        point_b = connection[1]\n        x1, y1 = example_hand.query(\"landmark_index == @point_a\")[[\"x\", \"y_\"]].values[0]\n        x2, y2 = example_hand.query(\"landmark_index == @point_b\")[[\"x\", \"y_\"]].values[0]\n        plt.plot([x1, x2], [y1, y2], color=\"purple\")\nax.set_title(\"Shhh - Hands Data\")\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-09-06T12:42:28.367504Z","iopub.execute_input":"2023-09-06T12:42:28.369098Z","iopub.status.idle":"2023-09-06T12:42:28.409791Z","shell.execute_reply.started":"2023-09-06T12:42:28.369031Z","shell.execute_reply":"2023-09-06T12:42:28.407805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import mediapipe as mp\nimport matplotlib.pyplot as plt\nimport pandas as pd  # Make sure you have pandas imported\nfrom mediapipe.framework.formats import landmark_pb2\nfrom mediapipe.util import landmark_pb2\nfrom mediapipe.solutions.hands import HandLandmark\n\n# Create a MediaPipe Hands instance\nmp_hands = mp.solutions.hands\n\n# Sample data for example_landmark (Replace this with your actual data)\n# Make sure example_landmark is a pandas DataFrame with columns 'frame', 'type', 'x', 'y', and 'landmark_index'\nexample_landmark = pd.DataFrame({\n    'frame': [17, 17, 17, 17, 17],\n    'type': ['left_hand', 'left_hand', 'right_hand', 'right_hand', 'right_hand'],\n    'x': [0.1, 0.2, 0.3, 0.4, 0.5],\n    'y': [0.6, 0.7, 0.8, 0.9, 1.0],\n    'landmark_index': [0, 1, 0, 1, 2]\n})\n\n# Correct the y-coordinates\nexample_landmark[\"y\"] = example_landmark[\"y\"] * -1\n\n# Create a figure for plotting\nfig, ax = plt.subplots(figsize=(5, 5))\n\n# Initialize the MediaPipe Hands module\nwith mp_hands.Hands() as hands:\n\n    for hand_type in [\"left_hand\", \"right_hand\"]:\n        example_hand = example_landmark.query(\"frame == 17 and type == @hand_type\")\n\n        ax.scatter(example_hand[\"x\"], example_hand[\"y\"])\n\n        for connection in mp_hands.HAND_CONNECTIONS:\n            point_a = connection[0]\n            point_b = connection[1]\n            x1, y1 = example_hand.query(\"landmark_index == @point_a\")[[\"x\", \"y\"]].values[0]\n            x2, y2 = example_hand.query(\"landmark_index == @point_b\")[[\"x\", \"y\"]].values[0]\n            plt.plot([x1, x2], [y1, y2], color=\"purple\")\n\n    ax.set_title(\"Shhh - Hands Data\")\n    plt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-09-06T12:56:02.507525Z","iopub.execute_input":"2023-09-06T12:56:02.508076Z","iopub.status.idle":"2023-09-06T12:56:02.709317Z","shell.execute_reply.started":"2023-09-06T12:56:02.508038Z","shell.execute_reply":"2023-09-06T12:56:02.70756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nimport mediapipe as mp\n\nmp_drawing = mp.solutions.drawing_utils\nmp_drawing_styles = mp.solutions.drawing_styles\nmp_holistic = mp.solutions.holistic\n\n# For static images:\nIMAGE_FILES = [\n    \"hqdefault.jpg\",\n    \"33214722-full-length-portrait-of-a-fashionable-young-man-standing-on-isolated-white-background.jpg\",\n]\nBG_COLOR = (192, 192, 192)  # gray\nwith mp_holistic.Holistic(\n    static_image_mode=True,\n    model_complexity=2,\n    enable_segmentation=True,\n    refine_face_landmarks=True,\n) as holistic:\n    for idx, file in enumerate(IMAGE_FILES):\n        image = cv2.imread(file)\n        image_height, image_width, _ = image.shape\n        # Convert the BGR image to RGB before processing.\n        results = holistic.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))\n\n        if results.pose_landmarks:\n            print(\n                f\"Nose coordinates: (\"\n                f\"{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].x * image_width}, \"\n                f\"{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].y * image_height})\"\n            )\n\n        annotated_image = image.copy()\n        # Draw segmentation on the image.\n        # To improve segmentation around boundaries, consider applying a joint\n        # bilateral filter to \"results.segmentation_mask\" with \"image\".\n        condition = np.stack((results.segmentation_mask,) * 3, axis=-1) > 0.1\n        bg_image = np.zeros(image.shape, dtype=np.uint8)\n        bg_image[:] = BG_COLOR\n        annotated_image = np.where(condition, annotated_image, bg_image)\n        # Draw pose, left and right hands, and face landmarks on the image.\n        mp_drawing.draw_landmarks(\n            annotated_image,\n            results.face_landmarks,\n            mp_holistic.FACEMESH_TESSELATION,\n            landmark_drawing_spec=None,\n            connection_drawing_spec=mp_drawing_styles.get_default_face_mesh_tesselation_style(),\n        )\n        mp_drawing.draw_landmarks(\n            annotated_image,\n            results.pose_landmarks,\n            mp_holistic.POSE_CONNECTIONS,\n            landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style(),\n        )\n        cv2.imwrite(\"/tmp/annotated_image\" + str(idx) + \".png\", annotated_image)\n        # Plot pose world landmarks.\n#         mp_drawing.plot_landmarks(\n#             results.pose_world_landmarks, mp_holistic.POSE_CONNECTIONS\n#         )","metadata":{"execution":{"iopub.status.busy":"2023-09-06T12:00:06.138428Z","iopub.status.idle":"2023-09-06T12:00:06.139187Z","shell.execute_reply.started":"2023-09-06T12:00:06.138813Z","shell.execute_reply":"2023-09-06T12:00:06.138853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(plt.imread(\"/tmp/annotated_image\" + str(0) + \".png\"))\nplt.show()\n\nplt.imshow(plt.imread(\"/tmp/annotated_image\" + str(1) + \".png\"))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-06T12:00:06.141364Z","iopub.status.idle":"2023-09-06T12:00:06.142142Z","shell.execute_reply.started":"2023-09-06T12:00:06.141764Z","shell.execute_reply":"2023-09-06T12:00:06.141803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"background_image = np.zeros([720, 720, 3])\n\nmp_drawing.draw_landmarks(\n    background_image,\n    results.face_landmarks,\n    mp_holistic.FACEMESH_TESSELATION,\n    landmark_drawing_spec=None,\n    connection_drawing_spec=mp_drawing_styles.get_default_face_mesh_tesselation_style(),\n)\nmp_drawing.draw_landmarks(\n    background_image,\n    results.pose_landmarks,\n    mp_holistic.POSE_CONNECTIONS,\n    landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style(),\n)\nplt.imshow(background_image)","metadata":{"execution":{"iopub.status.busy":"2023-09-06T12:00:06.1444Z","iopub.status.idle":"2023-09-06T12:00:06.145149Z","shell.execute_reply.started":"2023-09-06T12:00:06.144778Z","shell.execute_reply":"2023-09-06T12:00:06.144812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type(results.face_landmarks)\n\nfrom mediapipe.framework.formats import landmark_pb2\n\n# face_landmarks = landmark_pb2.NormalizedLandmarkList(example_frame.query('type == \"face\"')[[\"x\", \"y\", \"z\"]].values)","metadata":{"execution":{"iopub.status.busy":"2023-09-06T12:00:06.147402Z","iopub.status.idle":"2023-09-06T12:00:06.148107Z","shell.execute_reply.started":"2023-09-06T12:00:06.147768Z","shell.execute_reply":"2023-09-06T12:00:06.147799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_relevant_data_subset(pq_path):\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2023-09-06T12:00:06.15028Z","iopub.status.idle":"2023-09-06T12:00:06.150935Z","shell.execute_reply.started":"2023-09-06T12:00:06.150621Z","shell.execute_reply":"2023-09-06T12:00:06.150653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import tflite_runtime.interpreter as tflite\n\n# def run_model(model_path):\n#     interpreter = tflite.Interpreter(model_path)\n\n#     found_signatures = list(interpreter.get_signature_list().keys())\n\n#     if REQUIRED_SIGNATURE not in found_signatures:\n#         raise KernelEvalException('Required input signature not found.')\n\n#     prediction_fn = interpreter.get_signature_runner(\"serving_default\")\n#     output = prediction_fn(inputs=frames)\n#     sign = np.argmax(output[\"outputs\"])","metadata":{"execution":{"iopub.status.busy":"2023-09-06T12:00:06.153071Z","iopub.status.idle":"2023-09-06T12:00:06.153553Z","shell.execute_reply.started":"2023-09-06T12:00:06.153349Z","shell.execute_reply":"2023-09-06T12:00:06.153371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}