{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":46105,"databundleVersionId":5087314,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install --upgrade mediapipe tensorflow tensorflow-transform tensorflow-serving-api","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.notebook import tqdm\n\nplt.style.use(\"seaborn-v0_8-colorblind\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install \"black[jupyter]\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T22:06:32.822875Z","iopub.execute_input":"2025-10-04T22:06:32.823136Z","iopub.status.idle":"2025-10-04T22:06:38.733051Z","shell.execute_reply.started":"2025-10-04T22:06:32.823108Z","shell.execute_reply":"2025-10-04T22:06:38.731699Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!ls ../input/asl-signs/ -GFlash --color","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T22:06:38.735780Z","iopub.execute_input":"2025-10-04T22:06:38.736108Z","iopub.status.idle":"2025-10-04T22:06:38.916045Z","shell.execute_reply.started":"2025-10-04T22:06:38.736075Z","shell.execute_reply":"2025-10-04T22:06:38.914806Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Data EDA","metadata":{}},{"cell_type":"code","source":"BASE_DIR = '../input/asl-signs/'\ntrain = pd.read_csv(f'{BASE_DIR}/train.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T22:06:38.917351Z","iopub.execute_input":"2025-10-04T22:06:38.917673Z","iopub.status.idle":"2025-10-04T22:06:45.711742Z","shell.execute_reply.started":"2025-10-04T22:06:38.917642Z","shell.execute_reply":"2025-10-04T22:06:45.710176Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Train.csv has the path to each parquet file, the particpant id, sequence_id and sign.\ntrain.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T22:06:45.713129Z","iopub.execute_input":"2025-10-04T22:06:45.713498Z","iopub.status.idle":"2025-10-04T22:06:46.530603Z","shell.execute_reply.started":"2025-10-04T22:06:45.713468Z","shell.execute_reply":"2025-10-04T22:06:46.529498Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T22:06:46.532002Z","iopub.execute_input":"2025-10-04T22:06:46.532379Z","iopub.status.idle":"2025-10-04T22:06:47.762825Z","shell.execute_reply.started":"2025-10-04T22:06:46.532342Z","shell.execute_reply":"2025-10-04T22:06:47.761642Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(8, 8))\ntrain[\"sign\"].value_counts().head(50).sort_values(ascending=True).plot(\n    kind=\"barh\", ax=ax, title=\"Top 50 Signs in Training Dataset\"\n)\nax.set_xlabel(\"Number of Training Examples\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T22:06:47.764877Z","iopub.execute_input":"2025-10-04T22:06:47.765171Z","iopub.status.idle":"2025-10-04T22:06:49.406929Z","shell.execute_reply.started":"2025-10-04T22:06:47.765149Z","shell.execute_reply":"2025-10-04T22:06:49.405961Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(8, 8))\ntrain[\"sign\"].value_counts().tail(50).sort_values(ascending=True).plot(\n    kind=\"barh\", ax=ax, title=\"Bottom 50 Signs in Training Dataset\"\n)\nax.set_xlabel(\"Number of Training Examples\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T22:06:49.408087Z","iopub.execute_input":"2025-10-04T22:06:49.408530Z","iopub.status.idle":"2025-10-04T22:06:49.891389Z","shell.execute_reply.started":"2025-10-04T22:06:49.408500Z","shell.execute_reply":"2025-10-04T22:06:49.890384Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"example_fn = train.query('sign == \"listen\"')[\"path\"].values[0]\n\nexample_landmark = pd.read_parquet(f\"{BASE_DIR}/{example_fn}\")\nexample_landmark.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T22:06:49.894780Z","iopub.execute_input":"2025-10-04T22:06:49.895086Z","iopub.status.idle":"2025-10-04T22:06:49.929538Z","shell.execute_reply.started":"2025-10-04T22:06:49.895064Z","shell.execute_reply":"2025-10-04T22:06:49.928671Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"unique_frames = example_landmark[\"frame\"].nunique()\nunique_types = example_landmark[\"type\"].nunique()\ntypes_in_video = example_landmark[\"type\"].unique()\nprint(\n    f\"The file has {unique_frames} unique frames and {unique_types} unique types: {types_in_video}\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T22:06:49.930754Z","iopub.execute_input":"2025-10-04T22:06:49.931036Z","iopub.status.idle":"2025-10-04T22:06:49.938357Z","shell.execute_reply.started":"2025-10-04T22:06:49.931008Z","shell.execute_reply":"2025-10-04T22:06:49.937318Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"listen_files = train.query('sign == \"listen\"')[\"path\"].values\nfor i, f in enumerate(listen_files):\n    example_landmark = pd.read_parquet(f\"{BASE_DIR}/{f}\")\n    unique_frames = example_landmark[\"frame\"].nunique()\n    unique_types = example_landmark[\"type\"].nunique()\n    types_in_video = example_landmark[\"type\"].unique()\n    print(\n        f\"The file has {unique_frames} unique frames and {unique_types} unique types: {types_in_video}\"\n    )\n    if i == 20:\n        break","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T22:06:49.939355Z","iopub.execute_input":"2025-10-04T22:06:49.939768Z","iopub.status.idle":"2025-10-04T22:06:50.328625Z","shell.execute_reply.started":"2025-10-04T22:06:49.939734Z","shell.execute_reply":"2025-10-04T22:06:50.327601Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"N_PARQUETS_TO_READ = 100_000  # So we don't have to load all 95k\n\ncombined_meta = {}\nfor i, d in tqdm(train.iterrows(), total=len(train)):\n    file_path = d[\"path\"]\n    example_landmark = pd.read_parquet(f\"{BASE_DIR}/{file_path}\")\n    # Get the number of landmarks with x,y,z data per type\n    meta = (\n        example_landmark.dropna(subset=[\"x\", \"y\", \"z\"])[\"type\"].value_counts().to_dict()\n    )\n    meta[\"frames\"] = example_landmark[\"frame\"].nunique()\n    xyz_meta = (\n        example_landmark.agg(\n            {\n                \"x\": [\"min\", \"max\", \"mean\"],\n                \"y\": [\"min\", \"max\", \"mean\"],\n                \"z\": [\"min\", \"max\", \"mean\"],\n            }\n        )\n        .unstack()\n        .to_dict()\n    )\n\n    for key in xyz_meta.keys():\n        new_key = key[0] + \"_\" + key[1]\n        meta[new_key] = xyz_meta[key]\n    combined_meta[file_path] = meta\n    if i >= N_PARQUETS_TO_READ:\n        break","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T22:06:50.329847Z","iopub.execute_input":"2025-10-04T22:06:50.330076Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_with_meta = train.merge(\n    pd.DataFrame(combined_meta).T.reset_index().rename(columns={\"index\": \"path\"}),\n    how=\"left\",\n)\ntrain_with_meta.to_parquet(\"train_with_meta.parquet\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_with_meta[[\"face\", \"pose\", \"left_hand\", \"right_hand\"]].sum().sort_values().plot(\n    kind=\"barh\", title=\"Sum of Rows by Landmark Type\"\n)\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# checking to see if the number of landmarks for this type is zero\n(\n    train_with_meta.query(\"index < 1000\").fillna(0)[\n        [\"face\", \"pose\", \"left_hand\", \"right_hand\"]\n    ]\n    > 0\n).mean().plot(kind=\"barh\", title=\"Rate of Frame/Keypoints with Data\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"example_fn = train_with_meta.dropna().query('sign == \"shhh\"')[\"path\"].values[0]\nexample_landmark = pd.read_parquet(f\"{BASE_DIR}/{example_fn}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"example_landmark.query(\"frame == 25\")[\"type\"].value_counts()  # Middle of the video","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"example_landmark[\"no_xyz\"] = example_landmark[\"x\"].isna()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"example_landmark.groupby(\"frame\")[\"no_xyz\"].sum().plot(\n    title=\"missing xyz per frame\", kind=\"bar\"\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"3D plot of Landmarks from \"shhh\" example","metadata":{}},{"cell_type":"code","source":"import plotly.express as px\n\nexample_frame = example_landmark.query(\"frame == 17\")\npx.scatter_3d(example_frame, x=\"x\", y=\"y\", z=\"z\", color=\"type\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"example_landmark[\"y_\"] = example_landmark[\"y\"] * -1\nexample_frame = example_landmark.query(\"frame == 17 and type== 'face'\")\npx.scatter(example_frame, x=\"x\", y=\"y_\", color=\"type\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#Try to draw the example with mediapipe's hand connections?","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import mediapipe as mp\n\nmp_hands = mp.solutions.hands\n\n\nexample_landmark[\"y_\"] = example_landmark[\"y\"] * -1\n\nfig, ax = plt.subplots(figsize=(5, 5))\n\nfor hand in [\"left_hand\", \"right_hand\"]:\n    example_hand = example_landmark.query(\"frame == 17 and type == @hand\")\n\n    ax.scatter(example_hand[\"x\"], example_hand[\"y_\"])\n\n    for connection in mp_hands.HAND_CONNECTIONS:\n        point_a = connection[0]\n        point_b = connection[1]\n        x1, y1 = example_hand.query(\"landmark_index == @point_a\")[[\"x\", \"y_\"]].values[0]\n        x2, y2 = example_hand.query(\"landmark_index == @point_b\")[[\"x\", \"y_\"]].values[0]\n        plt.plot([x1, x2], [y1, y2], color=\"purple\")\nax.set_title(\"Shhh - Hands Data\")\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}