{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Importing Necessary Libraries**","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.notebook import tqdm\nimport plotly.express as px\nimport os\nfrom tensorflow.keras import layers, optimizers\nplt.style.use(\"seaborn-colorblind\")","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:09:09.834653Z","iopub.execute_input":"2023-04-24T04:09:09.836061Z","iopub.status.idle":"2023-04-24T04:09:25.273860Z","shell.execute_reply.started":"2023-04-24T04:09:09.835998Z","shell.execute_reply":"2023-04-24T04:09:25.272191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"plt.style.use('seaborn-colorblind') --> By setting the style to \"seaborn-colorblind\", all subsequent plots created using Matplotlib will have a color palette that is optimized for colorblind viewers.","metadata":{}},{"cell_type":"code","source":"# nb_black is used for autoformatting\n!pip install nb_black --quiet\n%load_ext lab_black","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:09:59.381519Z","iopub.execute_input":"2023-04-24T04:09:59.382849Z","iopub.status.idle":"2023-04-24T04:10:16.145642Z","shell.execute_reply.started":"2023-04-24T04:09:59.382789Z","shell.execute_reply":"2023-04-24T04:10:16.143669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(\"/kaggle/input/asl-signs/train.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:10:16.149409Z","iopub.execute_input":"2023-04-24T04:10:16.150058Z","iopub.status.idle":"2023-04-24T04:10:16.415015Z","shell.execute_reply.started":"2023-04-24T04:10:16.149985Z","shell.execute_reply":"2023-04-24T04:10:16.413043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:10:16.417011Z","iopub.execute_input":"2023-04-24T04:10:16.417631Z","iopub.status.idle":"2023-04-24T04:10:16.454766Z","shell.execute_reply.started":"2023-04-24T04:10:16.417576Z","shell.execute_reply":"2023-04-24T04:10:16.453047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **EDA**","metadata":{}},{"cell_type":"code","source":"# number of unique signs\ntrain_df[\"sign\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:10:16.458459Z","iopub.execute_input":"2023-04-24T04:10:16.459059Z","iopub.status.idle":"2023-04-24T04:10:16.484982Z","shell.execute_reply.started":"2023-04-24T04:10:16.459000Z","shell.execute_reply":"2023-04-24T04:10:16.483788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"sign\"].value_counts().head(30).sort_values().plot(\n    kind=\"barh\", figsize=(8, 6), title=\"Top 30 signs of train data\"\n)\nplt.xlabel(\"NO. of training samples\")\nplt.ylabel(\"Signs\")","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:10:16.487008Z","iopub.execute_input":"2023-04-24T04:10:16.487635Z","iopub.status.idle":"2023-04-24T04:10:17.001050Z","shell.execute_reply.started":"2023-04-24T04:10:16.487573Z","shell.execute_reply":"2023-04-24T04:10:16.999264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"sign\"].value_counts().tail(30).sort_values().plot(\n    kind=\"barh\", figsize=(8, 6), title=\"bottom 30 signs of train data\"\n)\nplt.xlabel(\"NO. of training samples\")\nplt.ylabel(\"Signs\")","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:10:17.004357Z","iopub.execute_input":"2023-04-24T04:10:17.005457Z","iopub.status.idle":"2023-04-24T04:10:17.505154Z","shell.execute_reply.started":"2023-04-24T04:10:17.005375Z","shell.execute_reply":"2023-04-24T04:10:17.503429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.query(\"sign == 'listen'\")","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:10:17.507032Z","iopub.execute_input":"2023-04-24T04:10:17.507509Z","iopub.status.idle":"2023-04-24T04:10:17.539066Z","shell.execute_reply.started":"2023-04-24T04:10:17.507463Z","shell.execute_reply":"2023-04-24T04:10:17.537511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Analysing Single Parquet file\n  **for sign = \"listen\"**","metadata":{}},{"cell_type":"code","source":"p1 = train_df.query(\"sign == 'listen'\")[\"path\"].iloc[0]","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:10:17.540853Z","iopub.execute_input":"2023-04-24T04:10:17.541323Z","iopub.status.idle":"2023-04-24T04:10:17.559218Z","shell.execute_reply.started":"2023-04-24T04:10:17.541260Z","shell.execute_reply":"2023-04-24T04:10:17.557239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p1","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:10:17.561755Z","iopub.execute_input":"2023-04-24T04:10:17.562258Z","iopub.status.idle":"2023-04-24T04:10:17.572268Z","shell.execute_reply.started":"2023-04-24T04:10:17.562207Z","shell.execute_reply":"2023-04-24T04:10:17.570611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"root_dir = \"/kaggle/input/asl-signs/\"","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:10:17.579904Z","iopub.execute_input":"2023-04-24T04:10:17.580940Z","iopub.status.idle":"2023-04-24T04:10:17.589747Z","shell.execute_reply.started":"2023-04-24T04:10:17.580878Z","shell.execute_reply":"2023-04-24T04:10:17.588095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p1_file = pd.read_parquet(root_dir + p1)","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:10:17.591520Z","iopub.execute_input":"2023-04-24T04:10:17.592012Z","iopub.status.idle":"2023-04-24T04:10:17.750209Z","shell.execute_reply.started":"2023-04-24T04:10:17.591965Z","shell.execute_reply":"2023-04-24T04:10:17.748835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p1_file","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:10:17.751930Z","iopub.execute_input":"2023-04-24T04:10:17.752793Z","iopub.status.idle":"2023-04-24T04:10:17.779760Z","shell.execute_reply.started":"2023-04-24T04:10:17.752743Z","shell.execute_reply":"2023-04-24T04:10:17.778209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frames = p1_file[\"frame\"]\ntypes = p1_file[\"type\"]\n\nprint(\"frame:\\n\", frames.value_counts())\nprint(f\"this file has {frames.nunique()} unique frames \\n\")\nprint(\"type:\\n\", types.value_counts())\nprint(f\"this file has {types.nunique()} unique types\")","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:10:17.781795Z","iopub.execute_input":"2023-04-24T04:10:17.782672Z","iopub.status.idle":"2023-04-24T04:10:17.802479Z","shell.execute_reply.started":"2023-04-24T04:10:17.782613Z","shell.execute_reply":"2023-04-24T04:10:17.801259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Comparing parquet files to check what type of data they have","metadata":{}},{"cell_type":"code","source":"listen_files = train_df.query(\"sign == 'listen'\")[\"path\"].values","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:10:17.803870Z","iopub.execute_input":"2023-04-24T04:10:17.805139Z","iopub.status.idle":"2023-04-24T04:10:17.820660Z","shell.execute_reply.started":"2023-04-24T04:10:17.805091Z","shell.execute_reply":"2023-04-24T04:10:17.818637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, j in enumerate(listen_files):\n    parquet_file = pd.read_parquet(root_dir + j)\n    p_frames = parquet_file[\"frame\"]\n    p_types = parquet_file[\"type\"]\n    print(\n        f\"this file has {p_frames.nunique()} unique frames and {p_types.nunique()} unique types \\n\"\n    )\n    if i == 20:\n        break","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:10:17.822863Z","iopub.execute_input":"2023-04-24T04:10:17.823399Z","iopub.status.idle":"2023-04-24T04:10:18.534131Z","shell.execute_reply.started":"2023-04-24T04:10:17.823316Z","shell.execute_reply":"2023-04-24T04:10:18.532784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"almost all files has same types but each file hase different number of unique frames","metadata":{}},{"cell_type":"markdown","source":"# Create MetaData for Training dataset","metadata":{}},{"cell_type":"markdown","source":"\"Metadata\" is data that provides information about other data, but not the content of the data","metadata":{}},{"cell_type":"code","source":"p1_file[\"type\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:10:18.535605Z","iopub.execute_input":"2023-04-24T04:10:18.536951Z","iopub.status.idle":"2023-04-24T04:10:18.549389Z","shell.execute_reply.started":"2023-04-24T04:10:18.536891Z","shell.execute_reply":"2023-04-24T04:10:18.547789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p1_file.dropna(subset=[\"x\", \"y\", \"z\"])[\"type\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:10:18.551377Z","iopub.execute_input":"2023-04-24T04:10:18.552924Z","iopub.status.idle":"2023-04-24T04:10:18.573337Z","shell.execute_reply.started":"2023-04-24T04:10:18.552860Z","shell.execute_reply":"2023-04-24T04:10:18.570768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata = {}\nfor i, d in tqdm(train_df.iterrows(), total=len(train_df)):\n    file_path = d[\"path\"]\n    parquet_file = pd.read_parquet(root_dir + file_path)\n    # get the number of landmarks with x,y,z data per type.\n    # i.e x,y,z values can be null so we are taking the data with x,y,z having non null values\n    meta = parquet_file.dropna(subset=[\"x\", \"y\", \"z\"])[\"type\"].value_counts().to_dict()\n    meta[\"frames\"] = parquet_file[\"frame\"].nunique()\n    xyz = (\n        (\n            parquet_file[[\"x\", \"y\", \"z\"]].agg(\n                {\n                    \"x\": [\"min\", \"max\", \"mean\"],\n                    \"y\": [\"min\", \"max\", \"mean\"],\n                    \"z\": [\"min\", \"max\", \"mean\"],\n                }\n            )\n        )\n        .unstack()\n        .to_dict()\n    )\n\n    for k in xyz.keys():\n        new_key = k[0] + \"_\" + k[1]\n        meta[new_key] = xyz[k]\n\n    metadata[file_path] = meta\n    if i == 1000:\n        break","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:10:18.575369Z","iopub.execute_input":"2023-04-24T04:10:18.575924Z","iopub.status.idle":"2023-04-24T04:11:02.697543Z","shell.execute_reply.started":"2023-04-24T04:10:18.575872Z","shell.execute_reply":"2023-04-24T04:11:02.696204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata_df = pd.DataFrame(metadata).T.reset_index().rename(columns={\"index\": \"path\"})","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:11:02.699882Z","iopub.execute_input":"2023-04-24T04:11:02.701613Z","iopub.status.idle":"2023-04-24T04:11:02.762111Z","shell.execute_reply.started":"2023-04-24T04:11:02.701548Z","shell.execute_reply":"2023-04-24T04:11:02.760254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata_df","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:11:02.764633Z","iopub.execute_input":"2023-04-24T04:11:02.765254Z","iopub.status.idle":"2023-04-24T04:11:02.805235Z","shell.execute_reply.started":"2023-04-24T04:11:02.765184Z","shell.execute_reply":"2023-04-24T04:11:02.803735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_with_meta = train_df.merge(metadata_df, how=\"left\")","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:11:02.807137Z","iopub.execute_input":"2023-04-24T04:11:02.807627Z","iopub.status.idle":"2023-04-24T04:11:02.912210Z","shell.execute_reply.started":"2023-04-24T04:11:02.807577Z","shell.execute_reply":"2023-04-24T04:11:02.910623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_with_meta","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:11:02.914041Z","iopub.execute_input":"2023-04-24T04:11:02.914713Z","iopub.status.idle":"2023-04-24T04:11:02.951488Z","shell.execute_reply.started":"2023-04-24T04:11:02.914670Z","shell.execute_reply":"2023-04-24T04:11:02.950042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_with_meta.to_parquet(\"train_with_meta.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:11:02.953689Z","iopub.execute_input":"2023-04-24T04:11:02.954112Z","iopub.status.idle":"2023-04-24T04:11:03.076974Z","shell.execute_reply.started":"2023-04-24T04:11:02.954073Z","shell.execute_reply":"2023-04-24T04:11:03.075534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_with_meta.fillna(0, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:11:03.079175Z","iopub.execute_input":"2023-04-24T04:11:03.080071Z","iopub.status.idle":"2023-04-24T04:11:03.113106Z","shell.execute_reply.started":"2023-04-24T04:11:03.079997Z","shell.execute_reply":"2023-04-24T04:11:03.111730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Finding most frequent types of landmarks provided","metadata":{}},{"cell_type":"code","source":"train_with_meta[[\"face\", \"pose\", \"left_hand\", \"right_hand\"]].sum().sort_values().plot(\n    kind=\"barh\", title=\"Sum of Rows by Landmark Type\"\n)","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:11:03.115246Z","iopub.execute_input":"2023-04-24T04:11:03.115792Z","iopub.status.idle":"2023-04-24T04:11:03.382784Z","shell.execute_reply.started":"2023-04-24T04:11:03.115733Z","shell.execute_reply":"2023-04-24T04:11:03.381064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Face has a lot more datapoints because mediapipe provides 468 3D datapoints per frame.","metadata":{}},{"cell_type":"code","source":"(\n    train_with_meta.query(\"index <= 1001\").fillna(0)[\n        [\"face\", \"pose\", \"right_hand\", \"left_hand\"]\n    ]\n    > 0\n).mean().plot(kind=\"barh\", title=\"Rate of Frame/Keypoints with Data\")","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:11:03.384919Z","iopub.execute_input":"2023-04-24T04:11:03.385968Z","iopub.status.idle":"2023-04-24T04:11:03.639651Z","shell.execute_reply.started":"2023-04-24T04:11:03.385918Z","shell.execute_reply":"2023-04-24T04:11:03.637676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Every parquet file has at least some datapoints for all four types of landmarks\n* face\n* pose\n* right_hand\n* left_hand","metadata":{}},{"cell_type":"code","source":"(\n    train_with_meta.query(\"index < 1000\").fillna(0)[\n        [\"face\", \"pose\", \"left_hand\", \"right_hand\"]\n    ]\n    > 0\n).mean().plot(kind=\"barh\", title=\"Rate of Frame/Keypoints with Data\")","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:11:03.642550Z","iopub.execute_input":"2023-04-24T04:11:03.643128Z","iopub.status.idle":"2023-04-24T04:11:03.885617Z","shell.execute_reply.started":"2023-04-24T04:11:03.643074Z","shell.execute_reply":"2023-04-24T04:11:03.884045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plotting datapoints for single example","metadata":{}},{"cell_type":"markdown","source":"**Taking the datapoints from middle frame of the example**","metadata":{}},{"cell_type":"code","source":"example_path = train_with_meta.dropna().query(\"sign == 'shhh'\")[\"path\"].values[0]\nexample_file = pd.read_parquet(root_dir + example_path)","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:11:03.894203Z","iopub.execute_input":"2023-04-24T04:11:03.894671Z","iopub.status.idle":"2023-04-24T04:11:03.962491Z","shell.execute_reply.started":"2023-04-24T04:11:03.894630Z","shell.execute_reply":"2023-04-24T04:11:03.961039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_file[\"frame\"].mean()  # finding the middle frame no. for this example","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:11:03.964502Z","iopub.execute_input":"2023-04-24T04:11:03.965811Z","iopub.status.idle":"2023-04-24T04:11:03.977411Z","shell.execute_reply.started":"2023-04-24T04:11:03.965249Z","shell.execute_reply":"2023-04-24T04:11:03.975774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Taking 25 as middle frame","metadata":{}},{"cell_type":"code","source":"# middle frame\nexample_file.query(\"frame== 25\")[\"type\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:11:03.978943Z","iopub.execute_input":"2023-04-24T04:11:03.979389Z","iopub.status.idle":"2023-04-24T04:11:03.999225Z","shell.execute_reply.started":"2023-04-24T04:11:03.979350Z","shell.execute_reply":"2023-04-24T04:11:03.997557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frame_file = example_file.query(\"frame== 25\")\nfig = px.scatter_3d(frame_file, x=\"x\", y=\"y\", z=\"z\", color=\"type\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:11:04.000873Z","iopub.execute_input":"2023-04-24T04:11:04.001751Z","iopub.status.idle":"2023-04-24T04:11:06.048277Z","shell.execute_reply.started":"2023-04-24T04:11:04.001696Z","shell.execute_reply":"2023-04-24T04:11:06.046854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frame_file = example_file.query(\"frame== 36\")\nfig = px.scatter_3d(frame_file, x=\"x\", y=\"y\", z=\"z\", color=\"type\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:11:06.050015Z","iopub.execute_input":"2023-04-24T04:11:06.050547Z","iopub.status.idle":"2023-04-24T04:11:06.142016Z","shell.execute_reply.started":"2023-04-24T04:11:06.050503Z","shell.execute_reply":"2023-04-24T04:11:06.140560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_file[\"y_\"] = example_file[\"y\"] * -1\nexample_file = example_file.query(\"frame == 17 and type == 'face'\")\npx.scatter(example_file, x=\"x\", y=\"y_\", color=\"type\")","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:11:06.144110Z","iopub.execute_input":"2023-04-24T04:11:06.145230Z","iopub.status.idle":"2023-04-24T04:11:06.252907Z","shell.execute_reply.started":"2023-04-24T04:11:06.145167Z","shell.execute_reply":"2023-04-24T04:11:06.251435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Using mediaPipe for ploting","metadata":{}},{"cell_type":"markdown","source":"**taking image from google**","metadata":{}},{"cell_type":"code","source":"!wget https://media-cldnry.s-nbcnews.com/image/upload/streams/2012/December/121214/1C5179134-121213-sittingTest-909p.jpg --quiet\n!wget https://previews.123rf.com/images/josemagon/josemagon1507/josemagon150701039/42451732-young-man-hands-to-front-on-a-white-background.jpg --quiet\n!pip install mediapipe --quiet","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:11:06.254916Z","iopub.execute_input":"2023-04-24T04:11:06.255863Z","iopub.status.idle":"2023-04-24T04:11:23.820403Z","shell.execute_reply.started":"2023-04-24T04:11:06.255800Z","shell.execute_reply":"2023-04-24T04:11:23.818320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nimport mediapipe as mp","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:11:23.824067Z","iopub.execute_input":"2023-04-24T04:11:23.824780Z","iopub.status.idle":"2023-04-24T04:11:24.179954Z","shell.execute_reply.started":"2023-04-24T04:11:23.824704Z","shell.execute_reply":"2023-04-24T04:11:24.178321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(2, 1, figsize=(6, 8))\n\naxes[0].imshow(\n    cv2.cvtColor(cv2.imread(\"1C5179134-121213-sittingTest-909p.jpg\"), cv2.COLOR_BGR2RGB)\n)\naxes[1].imshow(\n    cv2.cvtColor(\n        cv2.imread(\"42451732-young-man-hands-to-front-on-a-white-background.jpg\"),\n        cv2.COLOR_BGR2RGB,\n    )\n)","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:11:24.181875Z","iopub.execute_input":"2023-04-24T04:11:24.182370Z","iopub.status.idle":"2023-04-24T04:11:25.482414Z","shell.execute_reply.started":"2023-04-24T04:11:24.182323Z","shell.execute_reply":"2023-04-24T04:11:25.480607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**MediaPipe Holistic Model for Static Images**","metadata":{}},{"cell_type":"code","source":"mp_drawing = mp.solutions.drawing_utils\nmp_drawing_styles = mp.solutions.drawing_styles\nmp_holistic = mp.solutions.holistic\nmp_hands = mp.solutions.hands","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:11:25.484090Z","iopub.execute_input":"2023-04-24T04:11:25.484537Z","iopub.status.idle":"2023-04-24T04:11:25.494054Z","shell.execute_reply.started":"2023-04-24T04:11:25.484496Z","shell.execute_reply":"2023-04-24T04:11:25.492756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For static images:\nIMAGE_FILES = [\n    \"1C5179134-121213-sittingTest-909p.jpg\",\n    \"42451732-young-man-hands-to-front-on-a-white-background.jpg\",\n]\nBG_COLOR = (192, 192, 192)  # gray\nwith mp_holistic.Holistic(\n    static_image_mode=True,\n    model_complexity=2,\n    enable_segmentation=True,\n    refine_face_landmarks=True,\n) as holistic:\n    for idx, file in enumerate(IMAGE_FILES):\n        image = cv2.imread(file)\n        image_height, image_width, _ = image.shape\n        # Convert the BGR image to RGB before processing.\n        results = holistic.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))\n\n        if results.pose_landmarks:\n            print(\n                f\"Nose coordinates: (\"\n                f\"{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].x * image_width}, \"\n                f\"{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].y * image_height})\"\n            )\n\n        annotated_image = image.copy()\n        # Draw segmentation on the image.\n        # To improve segmentation around boundaries, consider applying a joint\n        # bilateral filter to \"results.segmentation_mask\" with \"image\".\n        condition = np.stack((results.segmentation_mask,) * 3, axis=-1) > 0.1\n        bg_image = np.zeros(image.shape, dtype=np.uint8)\n        bg_image[:] = BG_COLOR\n        annotated_image = np.where(condition, annotated_image, bg_image)\n        # Draw pose, left and right hands, and face landmarks on the image.\n        mp_drawing.draw_landmarks(\n            annotated_image,\n            results.face_landmarks,\n            mp_holistic.FACEMESH_TESSELATION,\n            landmark_drawing_spec=None,\n            connection_drawing_spec=mp_drawing_styles.get_default_face_mesh_tesselation_style(),\n        )\n        mp_drawing.draw_landmarks(\n            annotated_image,\n            results.pose_landmarks,\n            mp_holistic.POSE_CONNECTIONS,\n            landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style(),\n        )\n        mp_drawing.draw_landmarks(\n            annotated_image,\n            results.left_hand_landmarks,\n            mp_hands.HAND_CONNECTIONS,\n            mp_drawing_styles.get_default_hand_landmarks_style(),\n            mp_drawing_styles.get_default_hand_connections_style(),\n        )\n        mp_drawing.draw_landmarks(\n            annotated_image,\n            results.right_hand_landmarks,\n            mp_hands.HAND_CONNECTIONS,\n            mp_drawing_styles.get_default_hand_landmarks_style(),\n            mp_drawing_styles.get_default_hand_connections_style(),\n        )\n        cv2.imwrite(\"/tmp/annotated_image\" + str(idx) + \".png\", annotated_image)\n        # Plot pose world landmarks.\n        mp_drawing.plot_landmarks(\n            results.pose_world_landmarks, mp_holistic.POSE_CONNECTIONS\n        )","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:11:25.496239Z","iopub.execute_input":"2023-04-24T04:11:25.497168Z","iopub.status.idle":"2023-04-24T04:11:28.808167Z","shell.execute_reply.started":"2023-04-24T04:11:25.497099Z","shell.execute_reply":"2023-04-24T04:11:28.806827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(plt.imread(\"/tmp/annotated_image\" + str(0) + \".png\"))\nplt.show()\nplt.imshow(plt.imread(\"/tmp/annotated_image\" + str(1) + \".png\"))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:11:28.812558Z","iopub.execute_input":"2023-04-24T04:11:28.812938Z","iopub.status.idle":"2023-04-24T04:11:30.000337Z","shell.execute_reply.started":"2023-04-24T04:11:28.812904Z","shell.execute_reply":"2023-04-24T04:11:29.998488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Applying same method on single parquet file for plotting","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nfrom mediapipe.framework.formats import landmark_pb2","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:11:30.002110Z","iopub.execute_input":"2023-04-24T04:11:30.002897Z","iopub.status.idle":"2023-04-24T04:11:30.009708Z","shell.execute_reply.started":"2023-04-24T04:11:30.002842Z","shell.execute_reply":"2023-04-24T04:11:30.008319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def landmarklists(sig_n, framenum):\n    random_example_file_path = train_with_meta.query(\"sign==@sig_n\")[\"path\"].iloc[0]\n    random_example_file = pd.read_parquet(root_dir + random_example_file_path)\n    f1 = random_example_file.dropna().query(\"frame==@framenum\")\n\n    # Creating the NormalizedLandmarkList for each type\n    landmark_list_face = landmark_pb2.NormalizedLandmarkList()\n    landmark_list_pose = landmark_pb2.NormalizedLandmarkList()\n    landmark_list_left_hand = landmark_pb2.NormalizedLandmarkList()\n    landmark_list_right_hand = landmark_pb2.NormalizedLandmarkList()\n\n    for i, row in f1.iterrows():\n        if row[\"type\"] == \"face\":\n            landmark = landmark_list_face.landmark.add()\n            landmark.x = row[\"x\"]\n            landmark.y = row[\"y\"]\n            landmark.z = row[\"z\"]\n        elif row[\"type\"] == \"pose\":\n            landmark = landmark_list_pose.landmark.add()\n            landmark.x = row[\"x\"]\n            landmark.y = row[\"y\"]\n            landmark.z = row[\"z\"]\n        elif row[\"type\"] == \"left_hand\":\n            landmark = landmark_list_left_hand.landmark.add()\n            landmark.x = row[\"x\"]\n            landmark.y = row[\"y\"]\n            landmark.z = row[\"z\"]\n        elif row[\"type\"] == \"right_hand\":\n            landmark = landmark_list_right_hand.landmark.add()\n            landmark.x = row[\"x\"]\n            landmark.y = row[\"y\"]\n            landmark.z = row[\"z\"]\n\n    results = {}\n    results[\"pose_landmarks\"] = landmark_list_pose\n    results[\"left_hand_landmarks\"] = landmark_list_left_hand\n    results[\"right_hand_landmarks\"] = landmark_list_right_hand\n    results[\"face_landmarks\"] = landmark_list_face\n    return results","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:11:30.011867Z","iopub.execute_input":"2023-04-24T04:11:30.012496Z","iopub.status.idle":"2023-04-24T04:11:30.040245Z","shell.execute_reply.started":"2023-04-24T04:11:30.012449Z","shell.execute_reply":"2023-04-24T04:11:30.038781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_landmarks(results):\n    # creating black background\n    bg_img = np.zeros([1300, 1146, 3])\n\n    # drawing landmarks if landmark_list is not empty\n    if len(results[\"face_landmarks\"].landmark) != 0:\n        mp_drawing.draw_landmarks(\n            bg_img,\n            results[\"face_landmarks\"],\n            mp_holistic.FACEMESH_TESSELATION,\n            landmark_drawing_spec=None,\n            connection_drawing_spec=mp_drawing_styles.get_default_face_mesh_tesselation_style(),\n            #             mp_drawing.DrawingSpec(color=(0, 0, 255), thickness=2, circle_radius=2),\n            #             mp_drawing.DrawingSpec(color=(0, 255, 0), thickness=2, circle_radius=2),\n        )\n    if len(results[\"pose_landmarks\"].landmark) != 0:\n        mp_drawing.draw_landmarks(\n            bg_img,\n            results[\"pose_landmarks\"],\n            mp_holistic.POSE_CONNECTIONS,\n            landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style(),\n            #             mp_drawing.DrawingSpec(color=(255, 0, 0), thickness=5, circle_radius=5),\n            #             mp_drawing.DrawingSpec(color=(0, 50, 100), thickness=5, circle_radius=2),\n        )\n    if len(results[\"right_hand_landmarks\"].landmark) != 0:\n        mp_drawing.draw_landmarks(\n            bg_img,\n            results[\"right_hand_landmarks\"],\n            mp_hands.HAND_CONNECTIONS,\n            mp_drawing_styles.get_default_hand_landmarks_style(),\n            mp_drawing_styles.get_default_hand_connections_style(),\n        )\n    if len(results[\"left_hand_landmarks\"].landmark) != 0:\n        mp_drawing.draw_landmarks(\n            bg_img,\n            results[\"left_hand_landmarks\"],\n            mp_hands.HAND_CONNECTIONS,\n            mp_drawing_styles.get_default_hand_landmarks_style(),\n            mp_drawing_styles.get_default_hand_connections_style(),\n        )\n    # plotting the landmarks\n    plt.imshow(bg_img)","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:11:30.042232Z","iopub.execute_input":"2023-04-24T04:11:30.043363Z","iopub.status.idle":"2023-04-24T04:11:30.069871Z","shell.execute_reply.started":"2023-04-24T04:11:30.043311Z","shell.execute_reply":"2023-04-24T04:11:30.068256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = landmarklists(\"listen\", 40)\nplot_landmarks(results)","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:11:30.071767Z","iopub.execute_input":"2023-04-24T04:11:30.072314Z","iopub.status.idle":"2023-04-24T04:11:31.133251Z","shell.execute_reply.started":"2023-04-24T04:11:30.072262Z","shell.execute_reply":"2023-04-24T04:11:31.131707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = landmarklists(\"shhh\", 17)\nplot_landmarks(results)","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:11:31.134936Z","iopub.execute_input":"2023-04-24T04:11:31.135410Z","iopub.status.idle":"2023-04-24T04:11:32.173999Z","shell.execute_reply.started":"2023-04-24T04:11:31.135356Z","shell.execute_reply":"2023-04-24T04:11:32.172768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = landmarklists(\"look\", 34)\nplot_landmarks(results)","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:11:32.175677Z","iopub.execute_input":"2023-04-24T04:11:32.176389Z","iopub.status.idle":"2023-04-24T04:11:33.204667Z","shell.execute_reply.started":"2023-04-24T04:11:32.176339Z","shell.execute_reply":"2023-04-24T04:11:33.203021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = landmarklists(\"donkey\", 39)\nplot_landmarks(results)","metadata":{"execution":{"iopub.status.busy":"2023-04-24T04:11:33.206374Z","iopub.execute_input":"2023-04-24T04:11:33.206798Z","iopub.status.idle":"2023-04-24T04:11:34.231579Z","shell.execute_reply.started":"2023-04-24T04:11:33.206752Z","shell.execute_reply":"2023-04-24T04:11:34.230236Z"},"trusted":true},"execution_count":null,"outputs":[]}]}