{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.notebook import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-27T04:21:00.400823Z","iopub.execute_input":"2023-02-27T04:21:00.401881Z","iopub.status.idle":"2023-02-27T04:21:01.314692Z","shell.execute_reply.started":"2023-02-27T04:21:00.401787Z","shell.execute_reply":"2023-02-27T04:21:01.313561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/asl-signs/train.csv')\ntrain","metadata":{"execution":{"iopub.status.busy":"2023-02-27T04:21:01.316553Z","iopub.execute_input":"2023-02-27T04:21:01.316943Z","iopub.status.idle":"2023-02-27T04:21:01.534231Z","shell.execute_reply.started":"2023-02-27T04:21:01.316910Z","shell.execute_reply":"2023-02-27T04:21:01.533166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.subplots(figsize=(8, 8))\ntrain[\"sign\"].value_counts().head(50).sort_values(ascending=True).plot(\n    kind=\"barh\", title=\"Top 50 Signs in Training Dataset\", \n)","metadata":{"execution":{"iopub.status.busy":"2023-02-27T04:21:01.537701Z","iopub.execute_input":"2023-02-27T04:21:01.538029Z","iopub.status.idle":"2023-02-27T04:21:02.416781Z","shell.execute_reply.started":"2023-02-27T04:21:01.537999Z","shell.execute_reply":"2023-02-27T04:21:02.415808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark = pd.read_parquet('/kaggle/input/asl-signs/train_landmark_files/37055/100035691.parquet')\nexample_landmark.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-27T04:21:02.419567Z","iopub.execute_input":"2023-02-27T04:21:02.419958Z","iopub.status.idle":"2023-02-27T04:21:02.566569Z","shell.execute_reply.started":"2023-02-27T04:21:02.419919Z","shell.execute_reply":"2023-02-27T04:21:02.565401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfrom tqdm.notebook import tqdm\nN_PARQUETS_TO_READ = 100_000\ncombined_meta = {}\nfor i, features in tqdm(train[:50].iterrows(),total = len(train[:50])):\n    name_file = features[\"path\"]\n    file_path = os.path.join('/kaggle/input/asl-signs/',name_file)\n    example_landmark = pd.read_parquet(file_path)\n    meta = example_landmark.dropna(subset=[\"x\", \"y\", \"z\"])[\"type\"].value_counts().to_dict()\n    meta['frames'] = example_landmark['frame'].nunique()\n    xyz_meta = (\n        example_landmark.agg(\n            {\n                \"x\": [\"min\", \"max\", \"mean\"],\n                \"y\": [\"min\", \"max\", \"mean\"],\n                \"z\": [\"min\", \"max\", \"mean\"],\n            }\n        )\n        .unstack()\n        .to_dict()\n    )\n    for key in xyz_meta.keys():\n        new_key = key[0] + \"_\" + key[1]\n        meta[new_key] = xyz_meta[key]\n    combined_meta[file_path] = meta\n    if i >= N_PARQUETS_TO_READ:\n        break","metadata":{"execution":{"iopub.status.busy":"2023-02-27T04:22:37.675663Z","iopub.execute_input":"2023-02-27T04:22:37.676715Z","iopub.status.idle":"2023-02-27T04:22:38.779122Z","shell.execute_reply.started":"2023-02-27T04:22:37.676674Z","shell.execute_reply":"2023-02-27T04:22:38.778007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"combined_meta_df = pd.DataFrame(combined_meta).T.reset_index().rename(columns={\"index\": \"path\"})\ncombined_meta_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-27T04:22:48.756748Z","iopub.execute_input":"2023-02-27T04:22:48.757290Z","iopub.status.idle":"2023-02-27T04:22:48.793788Z","shell.execute_reply.started":"2023-02-27T04:22:48.757253Z","shell.execute_reply":"2023-02-27T04:22:48.792716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_with_meta = pd.merge(train[:50], combined_meta_df,left_index=True,right_index=True)\ntrain_with_meta.to_parquet(\"train_with_meta.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-02-27T04:38:26.666271Z","iopub.execute_input":"2023-02-27T04:38:26.666679Z","iopub.status.idle":"2023-02-27T04:38:26.680693Z","shell.execute_reply.started":"2023-02-27T04:38:26.666643Z","shell.execute_reply":"2023-02-27T04:38:26.679418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_with_meta.drop(columns=['path_x'],axis = 1,inplace=True)\ntrain_with_meta.rename(mapper={'path_y':'path'},axis = 1, inplace=True)\ntrain_with_meta.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-27T04:38:27.383333Z","iopub.execute_input":"2023-02-27T04:38:27.383746Z","iopub.status.idle":"2023-02-27T04:38:27.416276Z","shell.execute_reply.started":"2023-02-27T04:38:27.383711Z","shell.execute_reply":"2023-02-27T04:38:27.415116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_with_meta.columns","metadata":{"execution":{"iopub.status.busy":"2023-02-27T04:38:38.394523Z","iopub.execute_input":"2023-02-27T04:38:38.394975Z","iopub.status.idle":"2023-02-27T04:38:38.403051Z","shell.execute_reply.started":"2023-02-27T04:38:38.394936Z","shell.execute_reply":"2023-02-27T04:38:38.401925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_with_meta[[\"face\", \"pose\", \"left_hand\", \"right_hand\"]].sum().sort_values().plot(\n    kind=\"barh\", title=\"Sum of Rows by Landmark Type\"\n)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-27T04:38:39.318438Z","iopub.execute_input":"2023-02-27T04:38:39.318843Z","iopub.status.idle":"2023-02-27T04:38:39.584702Z","shell.execute_reply.started":"2023-02-27T04:38:39.318791Z","shell.execute_reply":"2023-02-27T04:38:39.583412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_fn = train_with_meta[train_with_meta['sign']=='blow']['path'][0]\nexample_landmark = pd.read_parquet(example_fn)\nexample_landmark","metadata":{"execution":{"iopub.status.busy":"2023-02-27T04:54:05.668525Z","iopub.execute_input":"2023-02-27T04:54:05.668937Z","iopub.status.idle":"2023-02-27T04:54:05.702263Z","shell.execute_reply.started":"2023-02-27T04:54:05.668901Z","shell.execute_reply":"2023-02-27T04:54:05.701070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark.query(\"frame == 25\")[\"type\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-02-27T04:54:06.350293Z","iopub.execute_input":"2023-02-27T04:54:06.351431Z","iopub.status.idle":"2023-02-27T04:54:06.368752Z","shell.execute_reply.started":"2023-02-27T04:54:06.351389Z","shell.execute_reply":"2023-02-27T04:54:06.367651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark[\"no_xyz\"] = example_landmark[\"x\"].isna()\nexample_landmark","metadata":{"execution":{"iopub.status.busy":"2023-02-27T04:54:06.575039Z","iopub.execute_input":"2023-02-27T04:54:06.575940Z","iopub.status.idle":"2023-02-27T04:54:06.596788Z","shell.execute_reply.started":"2023-02-27T04:54:06.575896Z","shell.execute_reply":"2023-02-27T04:54:06.595776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark.groupby(\"frame\")[\"no_xyz\"].sum().plot(\n    title=\"missing xyz per frame\", kind=\"bar\"\n)","metadata":{"execution":{"iopub.status.busy":"2023-02-27T04:54:06.758956Z","iopub.execute_input":"2023-02-27T04:54:06.759642Z","iopub.status.idle":"2023-02-27T04:54:07.110498Z","shell.execute_reply.started":"2023-02-27T04:54:06.759600Z","shell.execute_reply":"2023-02-27T04:54:07.109530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.express as px\n\nexample_frame = example_landmark.query(\"frame == 20\")\npx.scatter_3d(example_frame, x=\"x\", y=\"y\", z=\"z\", color=\"type\")","metadata":{"execution":{"iopub.status.busy":"2023-02-27T04:54:07.112484Z","iopub.execute_input":"2023-02-27T04:54:07.113360Z","iopub.status.idle":"2023-02-27T04:54:07.189884Z","shell.execute_reply.started":"2023-02-27T04:54:07.113317Z","shell.execute_reply":"2023-02-27T04:54:07.188615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Using mediapipe google","metadata":{}},{"cell_type":"code","source":"!pip install mediapipe --quiet\n","metadata":{"execution":{"iopub.status.busy":"2023-02-27T04:54:07.349237Z","iopub.execute_input":"2023-02-27T04:54:07.349826Z","iopub.status.idle":"2023-02-27T04:54:17.767593Z","shell.execute_reply.started":"2023-02-27T04:54:07.349786Z","shell.execute_reply":"2023-02-27T04:54:17.766245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark","metadata":{"execution":{"iopub.status.busy":"2023-02-27T04:54:17.770951Z","iopub.execute_input":"2023-02-27T04:54:17.772091Z","iopub.status.idle":"2023-02-27T04:54:17.796726Z","shell.execute_reply.started":"2023-02-27T04:54:17.772043Z","shell.execute_reply":"2023-02-27T04:54:17.795550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark","metadata":{"execution":{"iopub.status.busy":"2023-02-27T04:54:17.798296Z","iopub.execute_input":"2023-02-27T04:54:17.799109Z","iopub.status.idle":"2023-02-27T04:54:17.821785Z","shell.execute_reply.started":"2023-02-27T04:54:17.799060Z","shell.execute_reply":"2023-02-27T04:54:17.820850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t = example_landmark[example_landmark['frame'] == 20]\nt[t['type']=='right_hand']","metadata":{"execution":{"iopub.status.busy":"2023-02-27T04:55:49.140617Z","iopub.execute_input":"2023-02-27T04:55:49.141046Z","iopub.status.idle":"2023-02-27T04:55:49.165087Z","shell.execute_reply.started":"2023-02-27T04:55:49.141009Z","shell.execute_reply":"2023-02-27T04:55:49.163873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import mediapipe as mp\n\nmp_hands = mp.solutions.hands\n\n\nexample_landmark[\"y_\"] = example_landmark[\"y\"] * -1\n\nfig, ax = plt.subplots(figsize=(5, 5))\n\nfor hand in [\"left_hand\", \"right_hand\"]:\n    example_hand = example_landmark[example_landmark['frame'] == 20]\n    example_hand = example_hand[example_hand['type'] == hand]\n    ax.scatter(example_hand[\"x\"], example_hand[\"y_\"])\n\n    for connection in mp_hands.HAND_CONNECTIONS:\n        point_a = connection[0]\n        point_b = connection[1]\n        x1,y1= example_hand[example_hand['landmark_index'] == point_a][[\"x\", \"y_\"]].values[0]\n        x2, y2 = example_hand[example_hand['landmark_index'] == point_b][[\"x\", \"y_\"]].values[0]\n        plt.plot([x1, x2], [y1, y2], color=\"purple\")\nax.set_title(\"Shhh - Hands Data\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-27T04:58:44.066752Z","iopub.execute_input":"2023-02-27T04:58:44.067189Z","iopub.status.idle":"2023-02-27T04:58:44.460019Z","shell.execute_reply.started":"2023-02-27T04:58:44.067154Z","shell.execute_reply":"2023-02-27T04:58:44.458837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}