{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":46105,"databundleVersionId":5087314,"sourceType":"competition"},{"sourceId":120208086,"sourceType":"kernelVersion"}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.774057Z","iopub.status.idle":"2024-12-16T12:26:41.774470Z","shell.execute_reply.started":"2024-12-16T12:26:41.774275Z","shell.execute_reply":"2024-12-16T12:26:41.774293Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd \nimport numpy as np \nimport matplotlib.pyplot as plt\nimport seaborn as sns ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.775956Z","iopub.status.idle":"2024-12-16T12:26:41.776485Z","shell.execute_reply.started":"2024-12-16T12:26:41.776212Z","shell.execute_reply":"2024-12-16T12:26:41.776241Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ROWS_PER_FRAME = 543  # number of landmarks per frame\n\ndef load_relevant_data_subset(pq_path):\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.777585Z","iopub.status.idle":"2024-12-16T12:26:41.778161Z","shell.execute_reply.started":"2024-12-16T12:26:41.777887Z","shell.execute_reply":"2024-12-16T12:26:41.777916Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install tflite-runtime\n\nimport tflite_runtime.interpreter as tflite\ndef run_model(model_path):\n    \n    \n    interpreter = tflite.Interpreter(model_path)\n\n    found_signatures = list(interpreter.get_signature_list().keys())\n\n    if REQUIRED_SIGNATURE not in found_signatures:\n        raise KernelEvalException('Required input signature not found.')\n\n    prediction_fn = interpreter.get_signature_runner(\"serving_default\")\n    output = prediction_fn(inputs=frames)\n    sign = np.argmax(output[\"outputs\"])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.780549Z","iopub.status.idle":"2024-12-16T12:26:41.781125Z","shell.execute_reply.started":"2024-12-16T12:26:41.780837Z","shell.execute_reply":"2024-12-16T12:26:41.780866Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tqdm.notebook import tqdm\n\nplt.style.use(\"seaborn-colorblind\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.782728Z","iopub.status.idle":"2024-12-16T12:26:41.783277Z","shell.execute_reply.started":"2024-12-16T12:26:41.783005Z","shell.execute_reply":"2024-12-16T12:26:41.783032Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!ls ../input/asl-signs/ -GFlash --color","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.785278Z","iopub.status.idle":"2024-12-16T12:26:41.785677Z","shell.execute_reply.started":"2024-12-16T12:26:41.785496Z","shell.execute_reply":"2024-12-16T12:26:41.785515Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"BASE_DIR='../input/asl-signs/'\ntrain = pd.read_csv(f'{BASE_DIR}/train.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.787730Z","iopub.status.idle":"2024-12-16T12:26:41.788116Z","shell.execute_reply.started":"2024-12-16T12:26:41.787948Z","shell.execute_reply":"2024-12-16T12:26:41.787966Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.790120Z","iopub.status.idle":"2024-12-16T12:26:41.790459Z","shell.execute_reply.started":"2024-12-16T12:26:41.790294Z","shell.execute_reply":"2024-12-16T12:26:41.790311Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.792070Z","iopub.status.idle":"2024-12-16T12:26:41.792461Z","shell.execute_reply.started":"2024-12-16T12:26:41.792280Z","shell.execute_reply":"2024-12-16T12:26:41.792299Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['sign'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.793621Z","iopub.status.idle":"2024-12-16T12:26:41.794005Z","shell.execute_reply.started":"2024-12-16T12:26:41.793826Z","shell.execute_reply":"2024-12-16T12:26:41.793844Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pip install black\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.795491Z","iopub.status.idle":"2024-12-16T12:26:41.795910Z","shell.execute_reply.started":"2024-12-16T12:26:41.795687Z","shell.execute_reply":"2024-12-16T12:26:41.795705Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install blackcellmagic\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.797704Z","iopub.status.idle":"2024-12-16T12:26:41.798092Z","shell.execute_reply.started":"2024-12-16T12:26:41.797918Z","shell.execute_reply":"2024-12-16T12:26:41.797936Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%load_ext blackcellmagic\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.799120Z","iopub.status.idle":"2024-12-16T12:26:41.799449Z","shell.execute_reply.started":"2024-12-16T12:26:41.799288Z","shell.execute_reply":"2024-12-16T12:26:41.799304Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['sign'].value_counts().head(20).sort_values(ascending=True).plot(kind='barh',figsize=(10,5),title='top 20')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.800484Z","iopub.status.idle":"2024-12-16T12:26:41.800915Z","shell.execute_reply.started":"2024-12-16T12:26:41.800686Z","shell.execute_reply":"2024-12-16T12:26:41.800705Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['sign'].value_counts().tail(20).sort_values(ascending=True).plot(kind='barh',figsize=(10,5),title='bottom 20')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.802486Z","iopub.status.idle":"2024-12-16T12:26:41.802949Z","shell.execute_reply.started":"2024-12-16T12:26:41.802690Z","shell.execute_reply":"2024-12-16T12:26:41.802724Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(8, 8))\ntrain[\"sign\"].value_counts().head(50).sort_values(ascending=True).plot(\n    kind=\"barh\", ax=ax, title=\"Top 50 Signs in Training Dataset\"\n)\nax.set_xlabel(\"Number of Training Examples\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.804594Z","iopub.status.idle":"2024-12-16T12:26:41.804984Z","shell.execute_reply.started":"2024-12-16T12:26:41.804802Z","shell.execute_reply":"2024-12-16T12:26:41.804825Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.query('sign == \"listen\"')['path'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.806430Z","iopub.status.idle":"2024-12-16T12:26:41.806817Z","shell.execute_reply.started":"2024-12-16T12:26:41.806608Z","shell.execute_reply":"2024-12-16T12:26:41.806624Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['path'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.808105Z","iopub.status.idle":"2024-12-16T12:26:41.808470Z","shell.execute_reply.started":"2024-12-16T12:26:41.808294Z","shell.execute_reply":"2024-12-16T12:26:41.808314Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.809679Z","iopub.status.idle":"2024-12-16T12:26:41.810102Z","shell.execute_reply.started":"2024-12-16T12:26:41.809919Z","shell.execute_reply":"2024-12-16T12:26:41.809938Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"example_fn = train.query('sign == \"listen\"')[\"path\"].values[0]\n\nexample_landmark = pd.read_parquet(f\"{BASE_DIR}/{example_fn}\")\nexample_landmark.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.811869Z","iopub.status.idle":"2024-12-16T12:26:41.812248Z","shell.execute_reply.started":"2024-12-16T12:26:41.812063Z","shell.execute_reply":"2024-12-16T12:26:41.812080Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"unique_types=example_landmark['type'].nunique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.813720Z","iopub.status.idle":"2024-12-16T12:26:41.815335Z","shell.execute_reply.started":"2024-12-16T12:26:41.813937Z","shell.execute_reply":"2024-12-16T12:26:41.813955Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"unique_frame = example_landmark['frame'].value_counts()\nprint(f\"the file has {unique_frame}-unique frame and {unique_types} unique type \")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.816516Z","iopub.status.idle":"2024-12-16T12:26:41.816935Z","shell.execute_reply.started":"2024-12-16T12:26:41.816713Z","shell.execute_reply":"2024-12-16T12:26:41.816732Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"example_landmark['type'].unique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.818430Z","iopub.status.idle":"2024-12-16T12:26:41.818810Z","shell.execute_reply.started":"2024-12-16T12:26:41.818605Z","shell.execute_reply":"2024-12-16T12:26:41.818621Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"example_fn = train.query('sign == \"listen\"')[\"path\"].values[0]\n\nexample_landmark = pd.read_parquet(f\"{BASE_DIR}/{example_fn}\")\nexample_landmark.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.820966Z","iopub.status.idle":"2024-12-16T12:26:41.821344Z","shell.execute_reply.started":"2024-12-16T12:26:41.821159Z","shell.execute_reply":"2024-12-16T12:26:41.821178Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"unique_frames = example_landmark[\"frame\"].nunique()\nunique_types = example_landmark[\"type\"].nunique()\ntypes_in_video = example_landmark[\"type\"].unique()\nprint(\n    f\"The file has {unique_frames} unique frames and {unique_types} unique types: {types_in_video}\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.822932Z","iopub.status.idle":"2024-12-16T12:26:41.823319Z","shell.execute_reply.started":"2024-12-16T12:26:41.823130Z","shell.execute_reply":"2024-12-16T12:26:41.823149Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"listen_files = train.query('sign == \"listen\"')[\"path\"].values\nfor i, f in enumerate(listen_files):\n    example_landmark = pd.read_parquet(f\"{BASE_DIR}/{f}\")\n    unique_frames = example_landmark[\"frame\"].nunique()\n    unique_types = example_landmark[\"type\"].nunique()\n    types_in_video = example_landmark[\"type\"].unique()\n    print(\n        f\"The file has {unique_frames} unique frames and {unique_types} unique types: {types_in_video}\"\n    )\n    if i == 20:\n        break","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.824491Z","iopub.status.idle":"2024-12-16T12:26:41.824881Z","shell.execute_reply.started":"2024-12-16T12:26:41.824672Z","shell.execute_reply":"2024-12-16T12:26:41.824688Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.827169Z","iopub.status.idle":"2024-12-16T12:26:41.827607Z","shell.execute_reply.started":"2024-12-16T12:26:41.827412Z","shell.execute_reply":"2024-12-16T12:26:41.827432Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"N_PARQUETS_TO_READ = 100_000  # So we don't have to load all 95k\n\ncombined_meta = {}\nfor i, d in tqdm(train.iterrows(), total=len(train)):\n    file_path = d[\"path\"]\n    example_landmark = pd.read_parquet(f\"{BASE_DIR}/{file_path}\")\n    # Get the number of landmarks with x,y,z data per type\n    meta = (\n        example_landmark.dropna(subset=[\"x\", \"y\", \"z\"])[\"type\"].value_counts().to_dict()\n    )\n    meta[\"frames\"] = example_landmark[\"frame\"].nunique()\n    xyz_meta = (\n        example_landmark.agg(\n            {\n                \"x\": [\"min\", \"max\", \"mean\"],\n                \"y\": [\"min\", \"max\", \"mean\"],\n                \"z\": [\"min\", \"max\", \"mean\"],\n            }\n        )\n        .unstack()\n        .to_dict()\n    )\n\n    for key in xyz_meta.keys():\n        new_key = key[0] + \"_\" + key[1]\n        meta[new_key] = xyz_meta[key]\n    combined_meta[file_path] = meta\n    if i >= N_PARQUETS_TO_READ:\n        break","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.828620Z","iopub.status.idle":"2024-12-16T12:26:41.829014Z","shell.execute_reply.started":"2024-12-16T12:26:41.828826Z","shell.execute_reply":"2024-12-16T12:26:41.828845Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_with_meta = train.merge(\n    pd.DataFrame(combined_meta).T.reset_index().rename(columns={\"index\": \"path\"}),\n    how=\"left\",\n)\ntrain_with_meta.to_parquet(\"train_with_meta.parquet\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.830854Z","iopub.status.idle":"2024-12-16T12:26:41.831391Z","shell.execute_reply.started":"2024-12-16T12:26:41.831112Z","shell.execute_reply":"2024-12-16T12:26:41.831140Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_with_meta[[\"face\", \"pose\", \"left_hand\", \"right_hand\"]].sum().sort_values().plot(\n    kind=\"barh\", title=\"Sum of Rows by Landmark Type\"\n)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.833181Z","iopub.status.idle":"2024-12-16T12:26:41.833533Z","shell.execute_reply.started":"2024-12-16T12:26:41.833362Z","shell.execute_reply":"2024-12-16T12:26:41.833380Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# checking to see if the number of landmarks for this type is zero\n(\n    train_with_meta.query(\"index < 1000\").fillna(0)[\n        [\"face\", \"pose\", \"left_hand\", \"right_hand\"]\n    ]\n    > 0\n).mean().plot(kind=\"barh\", title=\"Rate of Frame/Keypoints with Data\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.834968Z","iopub.status.idle":"2024-12-16T12:26:41.835359Z","shell.execute_reply.started":"2024-12-16T12:26:41.835162Z","shell.execute_reply":"2024-12-16T12:26:41.835180Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_with_meta[[\"face\", \"pose\", \"left_hand\", \"right_hand\"]].sum().sort_values().plot(\n    kind=\"barh\", title=\"Sum of Rows by Landmark Type\"\n)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.836527Z","iopub.status.idle":"2024-12-16T12:26:41.836909Z","shell.execute_reply.started":"2024-12-16T12:26:41.836697Z","shell.execute_reply":"2024-12-16T12:26:41.836713Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"(\n    train_with_meta.query(\"index < 1000\").fillna(0)[\n        [\"face\", \"pose\", \"left_hand\", \"right_hand\"]\n    ]\n    > 0\n).mean().plot(kind=\"barh\", title=\"Rate of Frame/Keypoints with Data\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.838526Z","iopub.status.idle":"2024-12-16T12:26:41.838938Z","shell.execute_reply.started":"2024-12-16T12:26:41.838726Z","shell.execute_reply":"2024-12-16T12:26:41.838745Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"example_fn = train_with_meta.dropna().query('sign == \"shhh\"')[\"path\"].values[0]\nexample_landmark = pd.read_parquet(f\"{BASE_DIR}/{example_fn}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.839927Z","iopub.status.idle":"2024-12-16T12:26:41.840288Z","shell.execute_reply.started":"2024-12-16T12:26:41.840116Z","shell.execute_reply":"2024-12-16T12:26:41.840134Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"example_landmark.query(\"frame == 25\")[\"type\"].value_counts()  # Middle of the video","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.841863Z","iopub.status.idle":"2024-12-16T12:26:41.842231Z","shell.execute_reply.started":"2024-12-16T12:26:41.842058Z","shell.execute_reply":"2024-12-16T12:26:41.842076Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"example_landmark[\"no_xyz\"] = example_landmark[\"x\"].isna()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.843920Z","iopub.status.idle":"2024-12-16T12:26:41.844252Z","shell.execute_reply.started":"2024-12-16T12:26:41.844091Z","shell.execute_reply":"2024-12-16T12:26:41.844107Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"example_landmark.groupby(\"frame\")[\"no_xyz\"].sum().plot(\n    title=\"missing xyz per frame\", kind=\"bar\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.845500Z","iopub.status.idle":"2024-12-16T12:26:41.845891Z","shell.execute_reply.started":"2024-12-16T12:26:41.845678Z","shell.execute_reply":"2024-12-16T12:26:41.845695Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import plotly.express as px\n\nexample_frame = example_landmark.query(\"frame == 17\")\npx.scatter_3d(example_frame, x=\"x\", y=\"y\", z=\"z\", color=\"type\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.847354Z","iopub.status.idle":"2024-12-16T12:26:41.847727Z","shell.execute_reply.started":"2024-12-16T12:26:41.847541Z","shell.execute_reply":"2024-12-16T12:26:41.847562Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"example_landmark[\"y_\"] = example_landmark[\"y\"] * -1\nexample_frame = example_landmark.query(\"frame == 17 and type== 'face'\")\npx.scatter(example_frame, x=\"x\", y=\"y_\", color=\"type\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.849348Z","iopub.status.idle":"2024-12-16T12:26:41.849722Z","shell.execute_reply.started":"2024-12-16T12:26:41.849539Z","shell.execute_reply":"2024-12-16T12:26:41.849564Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Try to draw the example with mediapipe's hand connections?\n","metadata":{}},{"cell_type":"code","source":"!pip install mediapipe --quiet","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.771699Z","iopub.status.idle":"2024-12-16T12:26:41.772252Z","shell.execute_reply.started":"2024-12-16T12:26:41.771977Z","shell.execute_reply":"2024-12-16T12:26:41.772004Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import mediapipe as mp\n\nmp_hands = mp.solutions.hands\n\n\nexample_landmark[\"y_\"] = example_landmark[\"y\"] * -1\n\nfig, ax = plt.subplots(figsize=(5, 5))\n\nfor hand in [\"left_hand\", \"right_hand\"]:\n    example_hand = example_landmark.query(\"frame == 17 and type == @hand\")\n\n    ax.scatter(example_hand[\"x\"], example_hand[\"y_\"])\n\n    for connection in mp_hands.HAND_CONNECTIONS:\n        point_a = connection[0]\n        point_b = connection[1]\n        x1, y1 = example_hand.query(\"landmark_index == @point_a\")[[\"x\", \"y_\"]].values[0]\n        x2, y2 = example_hand.query(\"landmark_index == @point_b\")[[\"x\", \"y_\"]].values[0]\n        plt.plot([x1, x2], [y1, y2], color=\"purple\")\nax.set_title(\"Shhh - Hands Data\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.851796Z","iopub.status.idle":"2024-12-16T12:26:41.852139Z","shell.execute_reply.started":"2024-12-16T12:26:41.851975Z","shell.execute_reply":"2024-12-16T12:26:41.851992Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Try to use mediapipe to plot","metadata":{}},{"cell_type":"code","source":"!wget  https://i.ytimg.com/vi/mi9f9zOaqM8/hqdefault.jpg --quiet\n!wget https://previews.123rf.com/images/mimagephotography/mimagephotography1411/mimagephotography141100022/33214722-full-length-portrait-of-a-fashionable-young-man-standing-on-isolated-white-background.jpg --quiet","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.853385Z","iopub.status.idle":"2024-12-16T12:26:41.853824Z","shell.execute_reply.started":"2024-12-16T12:26:41.853560Z","shell.execute_reply":"2024-12-16T12:26:41.853576Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip uninstall cloudpickle dill numpy protobuf pyarrow -y\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.855680Z","iopub.status.idle":"2024-12-16T12:26:41.856067Z","shell.execute_reply.started":"2024-12-16T12:26:41.855895Z","shell.execute_reply":"2024-12-16T12:26:41.855913Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install cloudpickle==2.2.1 dill==0.3.1.1 numpy==1.24.0 protobuf==3.20.0 pyarrow==9.0.0\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.857354Z","iopub.status.idle":"2024-12-16T12:26:41.857729Z","shell.execute_reply.started":"2024-12-16T12:26:41.857547Z","shell.execute_reply":"2024-12-16T12:26:41.857565Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install cloudpickle==2.2.1 dill==0.3.1.1 numpy==1.24.0 protobuf==3.20.0 pyarrow==9.0.0\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.976258Z","iopub.status.idle":"2024-12-16T12:26:41.976809Z","shell.execute_reply.started":"2024-12-16T12:26:41.976547Z","shell.execute_reply":"2024-12-16T12:26:41.976567Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install apache-beam==2.46.0 --force-reinstall\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.977992Z","iopub.status.idle":"2024-12-16T12:26:41.978421Z","shell.execute_reply.started":"2024-12-16T12:26:41.978228Z","shell.execute_reply":"2024-12-16T12:26:41.978249Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install pipdeptree\n!pipdeptree\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.980657Z","iopub.status.idle":"2024-12-16T12:26:41.981096Z","shell.execute_reply.started":"2024-12-16T12:26:41.980907Z","shell.execute_reply":"2024-12-16T12:26:41.980927Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import cv2\nimport mediapipe as mp\n\nmp_drawing = mp.solutions.drawing_utils\nmp_drawing_styles = mp.solutions.drawing_styles\nmp_holistic = mp.solutions.holistic\n\n# For static images:\nIMAGE_FILES = [\n    \"hqdefault.jpg\",\n    \"33214722-full-length-portrait-of-a-fashionable-young-man-standing-on-isolated-white-background.jpg\",\n]\nBG_COLOR = (192, 192, 192)  # gray\nwith mp_holistic.Holistic(\n    static_image_mode=True,\n    model_complexity=2,\n    enable_segmentation=True,\n    refine_face_landmarks=True,\n) as holistic:\n    for idx, file in enumerate(IMAGE_FILES):\n        image = cv2.imread(file)\n        image_height, image_width, _ = image.shape\n        # Convert the BGR image to RGB before processing.\n        results = holistic.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))\n\n        if results.pose_landmarks:\n            print(\n                f\"Nose coordinates: (\"\n                f\"{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].x * image_width}, \"\n                f\"{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].y * image_height})\"\n            )\n\n        annotated_image = image.copy()\n        # Draw segmentation on the image.\n        # To improve segmentation around boundaries, consider applying a joint\n        # bilateral filter to \"results.segmentation_mask\" with \"image\".\n        condition = np.stack((results.segmentation_mask,) * 3, axis=-1) > 0.1\n        bg_image = np.zeros(image.shape, dtype=np.uint8)\n        bg_image[:] = BG_COLOR\n        annotated_image = np.where(condition, annotated_image, bg_image)\n        # Draw pose, left and right hands, and face landmarks on the image.\n        mp_drawing.draw_landmarks(\n            annotated_image,\n            results.face_landmarks,\n            mp_holistic.FACEMESH_TESSELATION,\n            landmark_drawing_spec=None,\n            connection_drawing_spec=mp_drawing_styles.get_default_face_mesh_tesselation_style(),\n        )\n        mp_drawing.draw_landmarks(\n            annotated_image,\n            results.pose_landmarks,\n            mp_holistic.POSE_CONNECTIONS,\n            landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style(),\n        )\n        cv2.imwrite(\"/tmp/annotated_image\" + str(idx) + \".png\", annotated_image)\n        # Plot pose world landmarks.\n#         mp_drawing.plot_landmarks(\n#             results.pose_world_landmarks, mp_holistic.POSE_CONNECTIONS\n#         )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.982851Z","iopub.status.idle":"2024-12-16T12:26:41.983215Z","shell.execute_reply.started":"2024-12-16T12:26:41.983043Z","shell.execute_reply":"2024-12-16T12:26:41.983060Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.imshow(plt.imread(\"/tmp/annotated_image\" + str(0) + \".png\"))\nplt.show()\n\nplt.imshow(plt.imread(\"/tmp/annotated_image\" + str(1) + \".png\"))\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.984847Z","iopub.status.idle":"2024-12-16T12:26:41.985197Z","shell.execute_reply.started":"2024-12-16T12:26:41.985031Z","shell.execute_reply":"2024-12-16T12:26:41.985048Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Try to use the same format for plotting of parquet data¶\r\n","metadata":{}},{"cell_type":"code","source":"background_image = np.zeros([720, 720, 3])\n\nmp_drawing.draw_landmarks(\n    background_image,\n    results.face_landmarks,\n    mp_holistic.FACEMESH_TESSELATION,\n    landmark_drawing_spec=None,\n    connection_drawing_spec=mp_drawing_styles.get_default_face_mesh_tesselation_style(),\n)\nmp_drawing.draw_landmarks(\n    background_image,\n    results.pose_landmarks,\n    mp_holistic.POSE_CONNECTIONS,\n    landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style(),\n)\nplt.imshow(background_image)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.769655Z","iopub.status.idle":"2024-12-16T12:26:41.770201Z","shell.execute_reply.started":"2024-12-16T12:26:41.769931Z","shell.execute_reply":"2024-12-16T12:26:41.769957Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"type(results.face_landmarks)\n\nfrom mediapipe.framework.formats import landmark_pb2\n\n# face_landmarks = landmark_pb2.NormalizedLandmarkList(example_frame.query('type == \"face\"')[[\"x\", \"y\", \"z\"]].values)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T12:26:41.767828Z","iopub.status.idle":"2024-12-16T12:26:41.768199Z","shell.execute_reply.started":"2024-12-16T12:26:41.768022Z","shell.execute_reply":"2024-12-16T12:26:41.768040Z"}},"outputs":[],"execution_count":null}]}