{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":46105,"databundleVersionId":5087314},{"sourceType":"datasetVersion","sourceId":5082760,"datasetId":2950885,"databundleVersionId":5153639}],"dockerImageVersionId":30461,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Installing requirements","metadata":{}},{"cell_type":"code","source":"!pip install mediapipe --quiet","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:16:17.935125Z","iopub.execute_input":"2024-02-27T17:16:17.936539Z","iopub.status.idle":"2024-02-27T17:16:30.628629Z","shell.execute_reply.started":"2024-02-27T17:16:17.936491Z","shell.execute_reply":"2024-02-27T17:16:30.627294Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# nb_black is used for autoformatting\n!pip install nb_black --quiet\n%load_ext lab_black","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:16:30.631191Z","iopub.execute_input":"2024-02-27T17:16:30.631526Z","iopub.status.idle":"2024-02-27T17:16:42.992029Z","shell.execute_reply.started":"2024-02-27T17:16:30.631494Z","shell.execute_reply":"2024-02-27T17:16:42.991001Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Importaing required libraries","metadata":{}},{"cell_type":"code","source":"# import libraries\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nfrom tensorflow.keras import layers, optimizers\nimport seaborn as sns\nfrom tqdm.notebook import tqdm\nimport plotly.express as px\nimport os\nimport cv2\nimport mediapipe as mp\nfrom mediapipe.framework.formats import landmark_pb2\nimport time\nimport json\n\nplt.style.use(\"seaborn-colorblind\")","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:16:42.993564Z","iopub.execute_input":"2024-02-27T17:16:42.993891Z","iopub.status.idle":"2024-02-27T17:16:53.254683Z","shell.execute_reply.started":"2024-02-27T17:16:42.993859Z","shell.execute_reply":"2024-02-27T17:16:53.253711Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Mediapipe Example (How to Use)","metadata":{}},{"cell_type":"code","source":"mp_drawing = mp.solutions.drawing_utils  # drawing utilities\nmp_drawing_styles = mp.solutions.drawing_styles\nmp_holistic = mp.solutions.holistic  # holistic model\nmp_hands = mp.solutions.hands","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:16:53.257304Z","iopub.execute_input":"2024-02-27T17:16:53.257613Z","iopub.status.idle":"2024-02-27T17:16:53.26473Z","shell.execute_reply.started":"2024-02-27T17:16:53.257582Z","shell.execute_reply":"2024-02-27T17:16:53.263653Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def mediapipe_detection(image, model):\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)  # COLOR CONVERSION BGR 2 RGB\n    image.flags.writeable = False  # Image is no longer writeable\n    results = model.process(image)  # Make prediction\n    image.flags.writeable = True  # Image is now writeable\n    image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)  # COLOR COVERSION RGB 2 BGR\n    return image, results","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:16:53.266099Z","iopub.execute_input":"2024-02-27T17:16:53.266386Z","iopub.status.idle":"2024-02-27T17:16:53.280384Z","shell.execute_reply.started":"2024-02-27T17:16:53.266357Z","shell.execute_reply":"2024-02-27T17:16:53.279481Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!wget https://media-cldnry.s-nbcnews.com/image/upload/streams/2012/December/121214/1C5179134-121213-sittingTest-909p.jpg --quiet\n!wget https://previews.123rf.com/images/josemagon/josemagon1507/josemagon150701039/42451732-young-man-hands-to-front-on-a-white-background.jpg --quiet","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:16:53.281603Z","iopub.execute_input":"2024-02-27T17:16:53.2826Z","iopub.status.idle":"2024-02-27T17:16:55.305215Z","shell.execute_reply.started":"2024-02-27T17:16:53.282569Z","shell.execute_reply":"2024-02-27T17:16:55.304019Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axes = plt.subplots(2, 1, figsize=(6, 8))\n\nimg1 = cv2.cvtColor(\n    cv2.imread(\n        \"/kaggle/working/42451732-young-man-hands-to-front-on-a-white-background.jpg\"\n    ),\n    cv2.COLOR_BGR2RGB,\n)\n\nimg2 = cv2.cvtColor(\n    cv2.imread(\"/kaggle/working/1C5179134-121213-sittingTest-909p.jpg\"),\n    cv2.COLOR_BGR2RGB,\n)\n\naxes[0].imshow(img1)\naxes[1].imshow(img2)","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:16:55.306871Z","iopub.execute_input":"2024-02-27T17:16:55.307201Z","iopub.status.idle":"2024-02-27T17:16:56.421849Z","shell.execute_reply.started":"2024-02-27T17:16:55.307169Z","shell.execute_reply":"2024-02-27T17:16:56.42083Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def draw_landmarks(image, results):\n    mp_drawing.draw_landmarks(\n        image, results.face_landmarks, mp_holistic.FACEMESH_TESSELATION\n    )  # draw face connections\n    mp_drawing.draw_landmarks(\n        image, results.pose_landmarks, mp_holistic.POSE_CONNECTIONS\n    )  # draw pose connections\n    mp_drawing.draw_landmarks(\n        image, results.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS\n    )  # draw left hand connections\n    mp_drawing.draw_landmarks(\n        image, results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS\n    )  # draw right hand connections","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:16:56.423347Z","iopub.execute_input":"2024-02-27T17:16:56.423736Z","iopub.status.idle":"2024-02-27T17:16:56.43705Z","shell.execute_reply.started":"2024-02-27T17:16:56.423696Z","shell.execute_reply":"2024-02-27T17:16:56.436027Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# This function is addition to draw_landmarks function, we are just changing color\n# and shape, size of landmarks\n\n\ndef draw_styled_landmarks(image, results):\n    # Draw face connections\n    mp_drawing.draw_landmarks(\n        image,\n        results.face_landmarks,\n        mp_holistic.FACEMESH_TESSELATION,\n        mp_drawing.DrawingSpec(color=(80, 110, 10), thickness=1, circle_radius=1),\n        mp_drawing.DrawingSpec(color=(80, 256, 121), thickness=1, circle_radius=1),\n    )\n    # Draw pose connections\n    mp_drawing.draw_landmarks(\n        image,\n        results.pose_landmarks,\n        mp_holistic.POSE_CONNECTIONS,\n        mp_drawing.DrawingSpec(color=(80, 22, 10), thickness=2, circle_radius=4),\n        mp_drawing.DrawingSpec(color=(80, 44, 121), thickness=2, circle_radius=2),\n    )\n    # Draw left hand connections\n    mp_drawing.draw_landmarks(\n        image,\n        results.left_hand_landmarks,\n        mp_holistic.HAND_CONNECTIONS,\n        mp_drawing.DrawingSpec(color=(121, 22, 76), thickness=2, circle_radius=4),\n        mp_drawing.DrawingSpec(color=(121, 44, 250), thickness=2, circle_radius=2),\n    )\n    # Draw right hand connections\n    mp_drawing.draw_landmarks(\n        image,\n        results.right_hand_landmarks,\n        mp_holistic.HAND_CONNECTIONS,\n        mp_drawing.DrawingSpec(color=(245, 117, 66), thickness=2, circle_radius=4),\n        mp_drawing.DrawingSpec(color=(245, 66, 230), thickness=2, circle_radius=2),\n    )","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:16:56.438278Z","iopub.execute_input":"2024-02-27T17:16:56.43857Z","iopub.status.idle":"2024-02-27T17:16:56.46275Z","shell.execute_reply.started":"2024-02-27T17:16:56.43854Z","shell.execute_reply":"2024-02-27T17:16:56.461832Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"image_height, image_width, _ = img1.shape\nimage_height1, image_width1, _ = img2.shape\n\nwith mp_holistic.Holistic(\n    static_image_mode=True,\n    model_complexity=2,\n    enable_segmentation=True,\n    refine_face_landmarks=True,\n) as holistic:\n    image, results = mediapipe_detection(img1, holistic)\n    annotated_image = image.copy()\n    draw_styled_landmarks(annotated_image, results)\n\n    image1, results1 = mediapipe_detection(img2, holistic)\n    annotated_image1 = image1.copy()\n    draw_styled_landmarks(annotated_image1, results1)","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:16:56.468954Z","iopub.execute_input":"2024-02-27T17:16:56.469491Z","iopub.status.idle":"2024-02-27T17:16:57.264517Z","shell.execute_reply.started":"2024-02-27T17:16:56.46946Z","shell.execute_reply":"2024-02-27T17:16:57.263609Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.imshow(annotated_image)","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:16:57.265894Z","iopub.execute_input":"2024-02-27T17:16:57.266287Z","iopub.status.idle":"2024-02-27T17:16:57.763494Z","shell.execute_reply.started":"2024-02-27T17:16:57.266245Z","shell.execute_reply":"2024-02-27T17:16:57.762435Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.imshow(annotated_image1)","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:16:57.764786Z","iopub.execute_input":"2024-02-27T17:16:57.765072Z","iopub.status.idle":"2024-02-27T17:16:58.351304Z","shell.execute_reply.started":"2024-02-27T17:16:57.765042Z","shell.execute_reply":"2024-02-27T17:16:58.350288Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# how landmarks are\nresults.face_landmarks.landmark","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2024-02-27T17:16:58.352814Z","iopub.execute_input":"2024-02-27T17:16:58.353218Z","iopub.status.idle":"2024-02-27T17:16:58.371829Z","shell.execute_reply.started":"2024-02-27T17:16:58.353173Z","shell.execute_reply":"2024-02-27T17:16:58.37083Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ----------------------------------------------------------------","metadata":{}},{"cell_type":"markdown","source":"# EDA (exploretory Data Analysis)","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(\"/kaggle/input/asl-signs/train.csv\")\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:16:58.373127Z","iopub.execute_input":"2024-02-27T17:16:58.373497Z","iopub.status.idle":"2024-02-27T17:16:58.57698Z","shell.execute_reply.started":"2024-02-27T17:16:58.373455Z","shell.execute_reply":"2024-02-27T17:16:58.575562Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# number of unique signs\nprint(\"maximum number of samples: \", max(train_df[\"sign\"].value_counts()))\nprint(\"minimum number of samples: \", min(train_df[\"sign\"].value_counts()))","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:16:58.578356Z","iopub.execute_input":"2024-02-27T17:16:58.578685Z","iopub.status.idle":"2024-02-27T17:16:58.62001Z","shell.execute_reply.started":"2024-02-27T17:16:58.578652Z","shell.execute_reply":"2024-02-27T17:16:58.618714Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"dataset contains maximum number of a samples of sign are 415 and minimum number of samples of a sign are 299","metadata":{}},{"cell_type":"code","source":"train_df[\"sign\"].nunique()","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:16:58.62166Z","iopub.execute_input":"2024-02-27T17:16:58.622499Z","iopub.status.idle":"2024-02-27T17:16:58.641706Z","shell.execute_reply.started":"2024-02-27T17:16:58.622448Z","shell.execute_reply":"2024-02-27T17:16:58.64048Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"there are 250 unique signs in this dataset","metadata":{}},{"cell_type":"code","source":"train_df[\"sign\"].value_counts().head(30).sort_values().plot(\n    kind=\"barh\", figsize=(8, 6), title=\"Top 30 signs of train data\"\n)\nplt.xlabel(\"NO. of training samples\")\nplt.ylabel(\"Signs\")","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:16:58.643244Z","iopub.execute_input":"2024-02-27T17:16:58.643986Z","iopub.status.idle":"2024-02-27T17:16:59.113395Z","shell.execute_reply.started":"2024-02-27T17:16:58.643944Z","shell.execute_reply":"2024-02-27T17:16:59.112289Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df[\"sign\"].value_counts().tail(30).sort_values().plot(\n    kind=\"barh\", figsize=(8, 6), title=\"bottom 30 signs of train data\"\n)\nplt.xlabel(\"NO. of training samples\")\nplt.ylabel(\"Signs\")","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:16:59.11464Z","iopub.execute_input":"2024-02-27T17:16:59.11501Z","iopub.status.idle":"2024-02-27T17:16:59.541861Z","shell.execute_reply.started":"2024-02-27T17:16:59.114976Z","shell.execute_reply":"2024-02-27T17:16:59.540841Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Analysing Single Parquet file\n  **for sign = \"listen\"**","metadata":{}},{"cell_type":"code","source":"root_dir = \"/kaggle/input/asl-signs/\"","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:16:59.543201Z","iopub.execute_input":"2024-02-27T17:16:59.5435Z","iopub.status.idle":"2024-02-27T17:16:59.549029Z","shell.execute_reply.started":"2024-02-27T17:16:59.54347Z","shell.execute_reply":"2024-02-27T17:16:59.547906Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.query(\"sign == 'listen'\")","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:16:59.550316Z","iopub.execute_input":"2024-02-27T17:16:59.550673Z","iopub.status.idle":"2024-02-27T17:16:59.578177Z","shell.execute_reply.started":"2024-02-27T17:16:59.550642Z","shell.execute_reply":"2024-02-27T17:16:59.577162Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"sign 'listen' contains 415 training samples","metadata":{}},{"cell_type":"code","source":"# single training sample of listen sign\np1 = train_df.query(\"sign == 'listen'\")[\"path\"].iloc[0]\np1_file = pd.read_parquet(root_dir + p1)\np1_file","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:16:59.57917Z","iopub.execute_input":"2024-02-27T17:16:59.579435Z","iopub.status.idle":"2024-02-27T17:16:59.685097Z","shell.execute_reply.started":"2024-02-27T17:16:59.579408Z","shell.execute_reply":"2024-02-27T17:16:59.684083Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"frames = p1_file[\"frame\"]\ntypes = p1_file[\"type\"]\n\nprint(\"frame:\\n\", frames.value_counts())\nprint(f\"this file has {frames.nunique()} unique frames \\n\")\nprint(\"type:\\n\", types.value_counts())\nprint(f\"this file has {types.nunique()} unique types\")","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:16:59.686437Z","iopub.execute_input":"2024-02-27T17:16:59.686746Z","iopub.status.idle":"2024-02-27T17:16:59.703285Z","shell.execute_reply.started":"2024-02-27T17:16:59.686717Z","shell.execute_reply":"2024-02-27T17:16:59.702181Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Comparing parquet files to check what type of data they have","metadata":{}},{"cell_type":"code","source":"# checking unique frames and unique types for each training sample of listen sign\n\nlisten_files = train_df.query(\"sign == 'listen'\")[\"path\"].values\nfor i, j in enumerate(listen_files):\n    parquet_file = pd.read_parquet(root_dir + j)\n    p_frames = parquet_file[\"frame\"]\n    p_types = parquet_file[\"type\"]\n    print(\n        f\"this file has {p_frames.nunique()} unique frames and {p_types.nunique()} unique types \\n\"\n    )\n    if i == 20:\n        break","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:16:59.704737Z","iopub.execute_input":"2024-02-27T17:16:59.705587Z","iopub.status.idle":"2024-02-27T17:17:00.370677Z","shell.execute_reply.started":"2024-02-27T17:16:59.705542Z","shell.execute_reply":"2024-02-27T17:17:00.369515Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"almost all files has same types but each file hase different number of unique frames","metadata":{}},{"cell_type":"code","source":"p1_file.query(\"frame==35\")[\"type\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:00.371916Z","iopub.execute_input":"2024-02-27T17:17:00.372217Z","iopub.status.idle":"2024-02-27T17:17:00.387672Z","shell.execute_reply.started":"2024-02-27T17:17:00.372186Z","shell.execute_reply":"2024-02-27T17:17:00.386493Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"in each frame there are:\n- 468 samples of face\n- 33 samples of pose\n- 21 samples of left_hand\n- 21 samples of right_hand <br>\n(here each sample is one landmark coordinate)\n\nas mediapipe holistic model have the above configuration to detect landmarks ","metadata":{}},{"cell_type":"markdown","source":"# Create MetaData for Training dataset","metadata":{}},{"cell_type":"markdown","source":"**Using only 1000 samples to build it quick. (use all samples to make MetaData)**","metadata":{}},{"cell_type":"code","source":"metadata = {}\nfor i, d in tqdm(train_df.iterrows(), total=len(train_df)):\n    file_path = d[\"path\"]\n    # reading parquet file\n    parquet_file = pd.read_parquet(root_dir + file_path)\n    # get (counting) the number of sample landmarks having x,y,z data per type.\n    # i.e x,y,z values can be null so we are taking those sample data with x,y,z having non null values\n    # that means we are finding which type(pose, face, hand) landmarks are missing in frame for each parquet file\n    meta = parquet_file.dropna(subset=[\"x\", \"y\", \"z\"])[\"type\"].value_counts().to_dict()\n    # finding total number of unique frames each parquet file contains\n    meta[\"frames\"] = parquet_file[\"frame\"].nunique()\n\n    \"\"\"\n    .agg() function is used to aggregate data in a DataFrame. It takes a dictionary as an argument, \n    where keys are column names, and values are the aggregation functions to be \n    applied to those columns.\n    \"\"\"\n    xyz = (\n        (\n            parquet_file[[\"x\", \"y\", \"z\"]].agg(\n                {\n                    \"x\": [\"min\", \"max\", \"mean\"],\n                    \"y\": [\"min\", \"max\", \"mean\"],\n                    \"z\": [\"min\", \"max\", \"mean\"],\n                }\n            )\n        )\n        .unstack()\n        .to_dict()\n    )\n\n    for k in xyz.keys():\n        new_key = k[0] + \"_\" + k[1]\n        meta[new_key] = xyz[k]\n\n    metadata[file_path] = meta\n    if i == 1000:\n        break\n\nmetadata_df = pd.DataFrame(metadata).T.reset_index().rename(columns={\"index\": \"path\"})","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:00.389055Z","iopub.execute_input":"2024-02-27T17:17:00.389445Z","iopub.status.idle":"2024-02-27T17:17:37.509546Z","shell.execute_reply.started":"2024-02-27T17:17:00.389402Z","shell.execute_reply":"2024-02-27T17:17:37.50805Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"metadata_df","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:37.511295Z","iopub.execute_input":"2024-02-27T17:17:37.512319Z","iopub.status.idle":"2024-02-27T17:17:37.546041Z","shell.execute_reply.started":"2024-02-27T17:17:37.512274Z","shell.execute_reply":"2024-02-27T17:17:37.544832Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_with_meta = train_df.merge(metadata_df, how=\"left\")\ntrain_with_meta.to_parquet(\"train_with_meta.parquet\")","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:37.547221Z","iopub.execute_input":"2024-02-27T17:17:37.547562Z","iopub.status.idle":"2024-02-27T17:17:37.697648Z","shell.execute_reply.started":"2024-02-27T17:17:37.547516Z","shell.execute_reply":"2024-02-27T17:17:37.69663Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_with_meta","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:37.707512Z","iopub.execute_input":"2024-02-27T17:17:37.70793Z","iopub.status.idle":"2024-02-27T17:17:37.74735Z","shell.execute_reply.started":"2024-02-27T17:17:37.707896Z","shell.execute_reply":"2024-02-27T17:17:37.746197Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Finding most frequent types of landmarks provided","metadata":{}},{"cell_type":"code","source":"train_with_meta[[\"face\", \"pose\", \"left_hand\", \"right_hand\"]].sum().sort_values().plot(\n    kind=\"barh\", title=\"Sum of Rows by Landmark Type\"\n)","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:37.748505Z","iopub.execute_input":"2024-02-27T17:17:37.748823Z","iopub.status.idle":"2024-02-27T17:17:37.923565Z","shell.execute_reply.started":"2024-02-27T17:17:37.748768Z","shell.execute_reply":"2024-02-27T17:17:37.922457Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"(\n    train_with_meta.query(\"index <= 1001\").fillna(0)[\n        [\"face\", \"pose\", \"right_hand\", \"left_hand\"]\n    ]\n    > 0\n).mean()","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:37.925008Z","iopub.execute_input":"2024-02-27T17:17:37.925528Z","iopub.status.idle":"2024-02-27T17:17:37.946609Z","shell.execute_reply.started":"2024-02-27T17:17:37.925482Z","shell.execute_reply":"2024-02-27T17:17:37.945555Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"(\n    train_with_meta.query(\"index <= 1001\").fillna(0)[\n        [\"face\", \"pose\", \"right_hand\", \"left_hand\"]\n    ]\n    > 0\n).mean().plot(kind=\"barh\", title=\"Rate of Frame/Keypoints with Data\")","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:37.948003Z","iopub.execute_input":"2024-02-27T17:17:37.948483Z","iopub.status.idle":"2024-02-27T17:17:38.109856Z","shell.execute_reply.started":"2024-02-27T17:17:37.948441Z","shell.execute_reply":"2024-02-27T17:17:38.108643Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Visualization","metadata":{}},{"cell_type":"markdown","source":"## Plotting datapoints for single example","metadata":{}},{"cell_type":"code","source":"train_with_meta.dropna().query(\"sign == 'shhh'\")[\"path\"].values[0]","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:38.111556Z","iopub.execute_input":"2024-02-27T17:17:38.112607Z","iopub.status.idle":"2024-02-27T17:17:38.153661Z","shell.execute_reply.started":"2024-02-27T17:17:38.112561Z","shell.execute_reply":"2024-02-27T17:17:38.15243Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"example_path = train_with_meta.dropna().query(\"sign == 'shhh'\")[\"path\"].values[0]\nexample_file = pd.read_parquet(root_dir + example_path)\nmid = int(example_file[\"frame\"].mean())  # finding the middle frame no. for this example\nframe_file = example_file.query(\"frame== @mid\")\nfig = px.scatter_3d(frame_file, x=\"x\", y=\"y\", z=\"z\", color=\"type\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:38.155343Z","iopub.execute_input":"2024-02-27T17:17:38.155669Z","iopub.status.idle":"2024-02-27T17:17:39.376823Z","shell.execute_reply.started":"2024-02-27T17:17:38.155636Z","shell.execute_reply":"2024-02-27T17:17:39.37579Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"frame_file = example_file.query(\"frame== 17\")\nfig = px.scatter_3d(frame_file, x=\"x\", y=\"y\", z=\"z\", color=\"type\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:39.378375Z","iopub.execute_input":"2024-02-27T17:17:39.378943Z","iopub.status.idle":"2024-02-27T17:17:39.467387Z","shell.execute_reply.started":"2024-02-27T17:17:39.378896Z","shell.execute_reply":"2024-02-27T17:17:39.466225Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Using mediaPipe for ploting","metadata":{}},{"cell_type":"code","source":"def landmarklists(sig_n, framenum):\n    random_example_file_path = train_with_meta.query(\"sign==@sig_n\")[\"path\"].iloc[0]\n    random_example_file = pd.read_parquet(root_dir + random_example_file_path)\n    f1 = random_example_file.dropna().query(\"frame==@framenum\")\n\n    # Creating the NormalizedLandmarkList for each type\n    landmark_list_face = landmark_pb2.NormalizedLandmarkList()\n    landmark_list_pose = landmark_pb2.NormalizedLandmarkList()\n    landmark_list_left_hand = landmark_pb2.NormalizedLandmarkList()\n    landmark_list_right_hand = landmark_pb2.NormalizedLandmarkList()\n\n    for i, row in f1.iterrows():\n        if row[\"type\"] == \"face\":\n            landmark = landmark_list_face.landmark.add()\n            landmark.x = row[\"x\"]\n            landmark.y = row[\"y\"]\n            landmark.z = row[\"z\"]\n        elif row[\"type\"] == \"pose\":\n            landmark = landmark_list_pose.landmark.add()\n            landmark.x = row[\"x\"]\n            landmark.y = row[\"y\"]\n            landmark.z = row[\"z\"]\n        elif row[\"type\"] == \"left_hand\":\n            landmark = landmark_list_left_hand.landmark.add()\n            landmark.x = row[\"x\"]\n            landmark.y = row[\"y\"]\n            landmark.z = row[\"z\"]\n        elif row[\"type\"] == \"right_hand\":\n            landmark = landmark_list_right_hand.landmark.add()\n            landmark.x = row[\"x\"]\n            landmark.y = row[\"y\"]\n            landmark.z = row[\"z\"]\n\n    results = {}\n    results[\"pose_landmarks\"] = landmark_list_pose\n    results[\"left_hand_landmarks\"] = landmark_list_left_hand\n    results[\"right_hand_landmarks\"] = landmark_list_right_hand\n    results[\"face_landmarks\"] = landmark_list_face\n    return results","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:39.468764Z","iopub.execute_input":"2024-02-27T17:17:39.469133Z","iopub.status.idle":"2024-02-27T17:17:39.501463Z","shell.execute_reply.started":"2024-02-27T17:17:39.469101Z","shell.execute_reply":"2024-02-27T17:17:39.50026Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def plot_landmarks(results):\n    # creating black background\n    bg_img = np.zeros([1300, 1146, 3])\n\n    # drawing landmarks if landmark_list is not empty\n    if len(results[\"face_landmarks\"].landmark) != 0:\n        mp_drawing.draw_landmarks(\n            bg_img,\n            results[\"face_landmarks\"],\n            mp_holistic.FACEMESH_TESSELATION,\n            landmark_drawing_spec=None,\n            connection_drawing_spec=mp_drawing_styles.get_default_face_mesh_tesselation_style(),\n            #             mp_drawing.DrawingSpec(color=(0, 0, 255), thickness=2, circle_radius=2),\n            #             mp_drawing.DrawingSpec(color=(0, 255, 0), thickness=2, circle_radius=2),\n        )\n    if len(results[\"pose_landmarks\"].landmark) != 0:\n        mp_drawing.draw_landmarks(\n            bg_img,\n            results[\"pose_landmarks\"],\n            mp_holistic.POSE_CONNECTIONS,\n            landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style(),\n            #             mp_drawing.DrawingSpec(color=(255, 0, 0), thickness=5, circle_radius=5),\n            #             mp_drawing.DrawingSpec(color=(0, 50, 100), thickness=5, circle_radius=2),\n        )\n    if len(results[\"right_hand_landmarks\"].landmark) != 0:\n        mp_drawing.draw_landmarks(\n            bg_img,\n            results[\"right_hand_landmarks\"],\n            mp_hands.HAND_CONNECTIONS,\n            mp_drawing_styles.get_default_hand_landmarks_style(),\n            mp_drawing_styles.get_default_hand_connections_style(),\n        )\n    if len(results[\"left_hand_landmarks\"].landmark) != 0:\n        mp_drawing.draw_landmarks(\n            bg_img,\n            results[\"left_hand_landmarks\"],\n            mp_hands.HAND_CONNECTIONS,\n            mp_drawing_styles.get_default_hand_landmarks_style(),\n            mp_drawing_styles.get_default_hand_connections_style(),\n        )\n    # plotting the landmarks\n    plt.imshow(bg_img)","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:39.502699Z","iopub.execute_input":"2024-02-27T17:17:39.503035Z","iopub.status.idle":"2024-02-27T17:17:39.526208Z","shell.execute_reply.started":"2024-02-27T17:17:39.503004Z","shell.execute_reply":"2024-02-27T17:17:39.52484Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"results = landmarklists(\"shhh\", 17)\nplot_landmarks(results)","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:39.527732Z","iopub.execute_input":"2024-02-27T17:17:39.528087Z","iopub.status.idle":"2024-02-27T17:17:40.348611Z","shell.execute_reply.started":"2024-02-27T17:17:39.528053Z","shell.execute_reply":"2024-02-27T17:17:40.347426Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"results = landmarklists(\"look\", 34)\nplot_landmarks(results)","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:40.350215Z","iopub.execute_input":"2024-02-27T17:17:40.350581Z","iopub.status.idle":"2024-02-27T17:17:41.19291Z","shell.execute_reply.started":"2024-02-27T17:17:40.350547Z","shell.execute_reply":"2024-02-27T17:17:41.191844Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"p1_file","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:41.194464Z","iopub.execute_input":"2024-02-27T17:17:41.194824Z","iopub.status.idle":"2024-02-27T17:17:41.215985Z","shell.execute_reply.started":"2024-02-27T17:17:41.194765Z","shell.execute_reply":"2024-02-27T17:17:41.214834Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## data visualization using animation","metadata":{}},{"cell_type":"code","source":"from matplotlib.animation import FuncAnimation\nfrom IPython.display import HTML","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:41.217145Z","iopub.execute_input":"2024-02-27T17:17:41.217424Z","iopub.status.idle":"2024-02-27T17:17:41.227463Z","shell.execute_reply.started":"2024-02-27T17:17:41.217396Z","shell.execute_reply":"2024-02-27T17:17:41.226265Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_hand_points(hand):\n    x = [\n        [\n            hand.iloc[0].x,\n            hand.iloc[1].x,\n            hand.iloc[2].x,\n            hand.iloc[3].x,\n            hand.iloc[4].x,\n        ],  # Thumb\n        [hand.iloc[5].x, hand.iloc[6].x, hand.iloc[7].x, hand.iloc[8].x],  # Index\n        [hand.iloc[9].x, hand.iloc[10].x, hand.iloc[11].x, hand.iloc[12].x],\n        [hand.iloc[13].x, hand.iloc[14].x, hand.iloc[15].x, hand.iloc[16].x],\n        [hand.iloc[17].x, hand.iloc[18].x, hand.iloc[19].x, hand.iloc[20].x],\n        [\n            hand.iloc[0].x,\n            hand.iloc[5].x,\n            hand.iloc[9].x,\n            hand.iloc[13].x,\n            hand.iloc[17].x,\n            hand.iloc[0].x,\n        ],\n    ]\n    y = [\n        [\n            hand.iloc[0].y,\n            hand.iloc[1].y,\n            hand.iloc[2].y,\n            hand.iloc[3].y,\n            hand.iloc[4].y,\n        ],  # Thumb\n        [hand.iloc[5].y, hand.iloc[6].y, hand.iloc[7].y, hand.iloc[8].y],  # Index\n        [hand.iloc[9].y, hand.iloc[10].y, hand.iloc[11].y, hand.iloc[12].y],\n        [hand.iloc[13].y, hand.iloc[14].y, hand.iloc[15].y, hand.iloc[16].y],\n        [hand.iloc[17].y, hand.iloc[18].y, hand.iloc[19].y, hand.iloc[20].y],\n        [\n            hand.iloc[0].y,\n            hand.iloc[5].y,\n            hand.iloc[9].y,\n            hand.iloc[13].y,\n            hand.iloc[17].y,\n            hand.iloc[0].y,\n        ],\n    ]\n    return x, y\n\n\ndef get_hand_points(hand):\n    x = [\n        [\n            hand.iloc[0].x,\n            hand.iloc[1].x,\n            hand.iloc[2].x,\n            hand.iloc[3].x,\n            hand.iloc[4].x,\n        ],  # Thumb\n        [hand.iloc[5].x, hand.iloc[6].x, hand.iloc[7].x, hand.iloc[8].x],  # Index\n        [hand.iloc[9].x, hand.iloc[10].x, hand.iloc[11].x, hand.iloc[12].x],\n        [hand.iloc[13].x, hand.iloc[14].x, hand.iloc[15].x, hand.iloc[16].x],\n        [hand.iloc[17].x, hand.iloc[18].x, hand.iloc[19].x, hand.iloc[20].x],\n        [\n            hand.iloc[0].x,\n            hand.iloc[5].x,\n            hand.iloc[9].x,\n            hand.iloc[13].x,\n            hand.iloc[17].x,\n            hand.iloc[0].x,\n        ],\n    ]\n\n    y = [\n        [\n            hand.iloc[0].y,\n            hand.iloc[1].y,\n            hand.iloc[2].y,\n            hand.iloc[3].y,\n            hand.iloc[4].y,\n        ],  # Thumb\n        [hand.iloc[5].y, hand.iloc[6].y, hand.iloc[7].y, hand.iloc[8].y],  # Index\n        [hand.iloc[9].y, hand.iloc[10].y, hand.iloc[11].y, hand.iloc[12].y],\n        [hand.iloc[13].y, hand.iloc[14].y, hand.iloc[15].y, hand.iloc[16].y],\n        [hand.iloc[17].y, hand.iloc[18].y, hand.iloc[19].y, hand.iloc[20].y],\n        [\n            hand.iloc[0].y,\n            hand.iloc[5].y,\n            hand.iloc[9].y,\n            hand.iloc[13].y,\n            hand.iloc[17].y,\n            hand.iloc[0].y,\n        ],\n    ]\n    return x, y\n\n\ndef get_pose_points(pose):\n    x = [\n        [\n            pose.iloc[8].x,\n            pose.iloc[6].x,\n            pose.iloc[5].x,\n            pose.iloc[4].x,\n            pose.iloc[0].x,\n            pose.iloc[1].x,\n            pose.iloc[2].x,\n            pose.iloc[3].x,\n            pose.iloc[7].x,\n        ],\n        [pose.iloc[10].x, pose.iloc[9].x],\n        [\n            pose.iloc[22].x,\n            pose.iloc[16].x,\n            pose.iloc[20].x,\n            pose.iloc[18].x,\n            pose.iloc[16].x,\n            pose.iloc[14].x,\n            pose.iloc[12].x,\n            pose.iloc[11].x,\n            pose.iloc[13].x,\n            pose.iloc[15].x,\n            pose.iloc[17].x,\n            pose.iloc[19].x,\n            pose.iloc[15].x,\n            pose.iloc[21].x,\n        ],\n        [\n            pose.iloc[12].x,\n            pose.iloc[24].x,\n            pose.iloc[26].x,\n            pose.iloc[28].x,\n            pose.iloc[30].x,\n            pose.iloc[32].x,\n            pose.iloc[28].x,\n        ],\n        [\n            pose.iloc[11].x,\n            pose.iloc[23].x,\n            pose.iloc[25].x,\n            pose.iloc[27].x,\n            pose.iloc[29].x,\n            pose.iloc[31].x,\n            pose.iloc[27].x,\n        ],\n        [pose.iloc[24].x, pose.iloc[23].x],\n    ]\n    y = [\n        [\n            pose.iloc[8].y,\n            pose.iloc[6].y,\n            pose.iloc[5].y,\n            pose.iloc[4].y,\n            pose.iloc[0].y,\n            pose.iloc[1].y,\n            pose.iloc[2].y,\n            pose.iloc[3].y,\n            pose.iloc[7].y,\n        ],\n        [pose.iloc[10].y, pose.iloc[9].y],\n        [\n            pose.iloc[22].y,\n            pose.iloc[16].y,\n            pose.iloc[20].y,\n            pose.iloc[18].y,\n            pose.iloc[16].y,\n            pose.iloc[14].y,\n            pose.iloc[12].y,\n            pose.iloc[11].y,\n            pose.iloc[13].y,\n            pose.iloc[15].y,\n            pose.iloc[17].y,\n            pose.iloc[19].y,\n            pose.iloc[15].y,\n            pose.iloc[21].y,\n        ],\n        [\n            pose.iloc[12].y,\n            pose.iloc[24].y,\n            pose.iloc[26].y,\n            pose.iloc[28].y,\n            pose.iloc[30].y,\n            pose.iloc[32].y,\n            pose.iloc[28].y,\n        ],\n        [\n            pose.iloc[11].y,\n            pose.iloc[23].y,\n            pose.iloc[25].y,\n            pose.iloc[27].y,\n            pose.iloc[29].y,\n            pose.iloc[31].y,\n            pose.iloc[27].y,\n        ],\n        [pose.iloc[24].y, pose.iloc[23].y],\n    ]\n    return x, y","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:41.229154Z","iopub.execute_input":"2024-02-27T17:17:41.229551Z","iopub.status.idle":"2024-02-27T17:17:41.361565Z","shell.execute_reply.started":"2024-02-27T17:17:41.229508Z","shell.execute_reply":"2024-02-27T17:17:41.360289Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def sign_type(DF, root_dirt, sign):\n    file_path = train_df.query(\"sign==@sign\")[\"path\"].loc[256]\n    file = pd.read_parquet(root_dirt + file_path)\n    file.y = file.y * -1\n    return file, sign","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:41.36296Z","iopub.execute_input":"2024-02-27T17:17:41.363293Z","iopub.status.idle":"2024-02-27T17:17:41.372219Z","shell.execute_reply.started":"2024-02-27T17:17:41.363261Z","shell.execute_reply":"2024-02-27T17:17:41.371135Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"file, sign = sign_type(train_df, root_dir, \"pretend\")\n\n\ndef animation_frame(f):\n    frame = file[file.frame == f]\n    left = frame[frame.type == \"left_hand\"]\n    right = frame[frame.type == \"right_hand\"]\n    pose = frame[frame.type == \"pose\"]\n    face = frame[frame.type == \"face\"][[\"x\", \"y\"]].values\n    lx, ly = get_hand_points(left)\n    rx, ry = get_hand_points(right)\n    px, py = get_pose_points(pose)\n    ax.clear()  # removing any existing plots, labels, or other elements.\n    ax.plot(face[:, 0], face[:, 1], \".\")\n    for i in range(len(lx)):\n        ax.plot(lx[i], ly[i])\n    for i in range(len(rx)):\n        ax.plot(rx[i], ry[i])\n    for i in range(len(px)):\n        ax.plot(px[i], py[i])\n    plt.xlim(xmin, xmax)\n    plt.ylim(ymin, ymax)\n\n\nprint(f\"The sign being shown here is: {sign}\")\n\n## These values set the limits on the graph to stabilize the video\nxmin = file.x.min() - 0.2\nxmax = file.x.max() + 0.2\nymin = file.y.min() - 0.2\nymax = file.y.max() + 0.2\n\nfig, ax = plt.subplots()\n(l,) = ax.plot([], [])\n\nanimation = FuncAnimation(fig, func=animation_frame, frames=file.frame.unique())\nHTML(animation.to_html5_video())","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:41.373551Z","iopub.execute_input":"2024-02-27T17:17:41.373881Z","iopub.status.idle":"2024-02-27T17:17:59.075672Z","shell.execute_reply.started":"2024-02-27T17:17:41.373851Z","shell.execute_reply":"2024-02-27T17:17:59.074521Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **DATA Preparation**","metadata":{}},{"cell_type":"markdown","source":"**Code for creating batched data**","metadata":{}},{"cell_type":"code","source":"# # set files directories\n# LANDMARK_FILES_DIR = \"/kaggle/input/asl-signs/train_landmark_files\"\n# TRAIN_FILE = \"/kaggle/input/asl-signs/train.csv\"\n\n# DATA_COLUMNS = [\"x\", \"y\", \"z\"]\n# ROWS_PER_FRAME = 543\n# NUM_SHARDS = 2\n# SAVE_PATH = \"BatchedGSLRdataset\"\n# BATCH_SIZE = 256\n\n# Set constants and pick important landmarks\n# LANDMARK_IDX = list(range(0, 543))  # total landmarks are 543\n# DATA_PATH = \"/kaggle/input/saved-tfdataset-of-google-isl-recognition-data/GoogleISLDatasetBatched\"\n# DS_CARDINALITY = 185\n# VAL_SIZE = 18\n# N_SIGNS = 250\n# ROWS_PER_FRAME = 543","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:59.077097Z","iopub.execute_input":"2024-02-27T17:17:59.077408Z","iopub.status.idle":"2024-02-27T17:17:59.083683Z","shell.execute_reply.started":"2024-02-27T17:17:59.077375Z","shell.execute_reply":"2024-02-27T17:17:59.082558Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# with open(\"/kaggle/input/asl-signs/sign_to_prediction_index_map.json\") as f:\n#     sign_ids = json.load(f)\n# print(sign_ids)","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:59.085278Z","iopub.execute_input":"2024-02-27T17:17:59.086365Z","iopub.status.idle":"2024-02-27T17:17:59.095118Z","shell.execute_reply.started":"2024-02-27T17:17:59.086312Z","shell.execute_reply":"2024-02-27T17:17:59.094111Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def load_relevant_data_subset(pq_path):\n#     data = pd.read_parquet(root_dir + pq_path, columns=DATA_COLUMNS)\n#     n_frames = int(len(data) / ROWS_PER_FRAME)\n#     data = data.values.astype(np.float32)  # for memory saving and performance gain\n#     return data.reshape(n_frames, ROWS_PER_FRAME, len(DATA_COLUMNS))","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:59.096434Z","iopub.execute_input":"2024-02-27T17:17:59.097164Z","iopub.status.idle":"2024-02-27T17:17:59.104648Z","shell.execute_reply.started":"2024-02-27T17:17:59.097128Z","shell.execute_reply":"2024-02-27T17:17:59.103602Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def tf_get_features(ftensor):\n#     def feat_wrapper(ftensor):\n#         return load_relevant_data_subset(ftensor.numpy().decode(\"utf-8\"))\n\n#     return tf.py_function(feat_wrapper, [ftensor], Tout=tf.float32)","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:59.105915Z","iopub.execute_input":"2024-02-27T17:17:59.106195Z","iopub.status.idle":"2024-02-27T17:17:59.11629Z","shell.execute_reply.started":"2024-02-27T17:17:59.106166Z","shell.execute_reply":"2024-02-27T17:17:59.115198Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"TensorFlow Dataset.map() only accepts graph traceable functions, so we wrap the data loading function with tf.py_function","metadata":{}},{"cell_type":"markdown","source":"tf.py_function let's us execute code outside the tf graph, but that means TensorFlow cannot track the shapes of the returned tensors!\n\nHowever, we still know the returned shape from looking at the loading function, so we can use tf.ensure_shape to tell the rest of our pipeline what element shapes to expect. This is necessary for the dense_to_ragged_batch function that will finally perform the batching","metadata":{}},{"cell_type":"code","source":"# def set_shape(x):\n#     # None dimensions can be of any length\n#     return tf.ensure_shape(x, (None, ROWS_PER_FRAME, len(DATA_COLUMNS)))","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:59.11734Z","iopub.execute_input":"2024-02-27T17:17:59.117599Z","iopub.status.idle":"2024-02-27T17:17:59.126444Z","shell.execute_reply.started":"2024-02-27T17:17:59.117574Z","shell.execute_reply":"2024-02-27T17:17:59.125409Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# X_ds = (\n#     tf.data.Dataset.from_tensor_slices(\n#         train_df.path.values  # start with a dataset of the parquet paths\n#     )\n#     .map(tf_get_features)  # load individual sequences\n#     .map(set_shape)  # set and enforce element shape\n#     .apply(\n#         tf.data.experimental.dense_to_ragged_batch(\n#             batch_size=BATCH_SIZE\n#         )  # apply batching function\n#     )\n# )","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:59.127715Z","iopub.execute_input":"2024-02-27T17:17:59.12838Z","iopub.status.idle":"2024-02-27T17:17:59.136506Z","shell.execute_reply.started":"2024-02-27T17:17:59.128339Z","shell.execute_reply":"2024-02-27T17:17:59.135543Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # load and batch the labels\n# y_ds = tf.data.Dataset.from_tensor_slices(\n#     train_df.sign.map(sign_ids).values.reshape(-1, 1)\n# ).batch(BATCH_SIZE)","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:59.137691Z","iopub.execute_input":"2024-02-27T17:17:59.138338Z","iopub.status.idle":"2024-02-27T17:17:59.145803Z","shell.execute_reply.started":"2024-02-27T17:17:59.138297Z","shell.execute_reply":"2024-02-27T17:17:59.144757Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # zip the features and labels\n# train_ds = tf.data.Dataset.zip((X_ds, y_ds))","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:59.146982Z","iopub.execute_input":"2024-02-27T17:17:59.147273Z","iopub.status.idle":"2024-02-27T17:17:59.154514Z","shell.execute_reply.started":"2024-02-27T17:17:59.147237Z","shell.execute_reply":"2024-02-27T17:17:59.153475Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def shard_func(*_):\n#     return tf.random.uniform(shape=[], maxval=NUM_SHARDS, dtype=tf.int64)","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:59.155663Z","iopub.execute_input":"2024-02-27T17:17:59.156481Z","iopub.status.idle":"2024-02-27T17:17:59.164555Z","shell.execute_reply.started":"2024-02-27T17:17:59.15645Z","shell.execute_reply":"2024-02-27T17:17:59.16359Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train_ds.prefetch(tf.data.AUTOTUNE).save(SAVE_PATH, shard_func=shard_func)","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:59.165999Z","iopub.execute_input":"2024-02-27T17:17:59.166305Z","iopub.status.idle":"2024-02-27T17:17:59.17353Z","shell.execute_reply.started":"2024-02-27T17:17:59.166275Z","shell.execute_reply":"2024-02-27T17:17:59.172643Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def check_throughput(ds_path):\n#     for x in tqdm(tf.data.Dataset.load(ds_path)):\n#         pass","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:59.17469Z","iopub.execute_input":"2024-02-27T17:17:59.175371Z","iopub.status.idle":"2024-02-27T17:17:59.182138Z","shell.execute_reply.started":"2024-02-27T17:17:59.17534Z","shell.execute_reply":"2024-02-27T17:17:59.181213Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# check_throughput(SAVE_PATH)","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:59.183286Z","iopub.execute_input":"2024-02-27T17:17:59.183549Z","iopub.status.idle":"2024-02-27T17:17:59.191607Z","shell.execute_reply.started":"2024-02-27T17:17:59.183522Z","shell.execute_reply":"2024-02-27T17:17:59.190652Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"-------------------------------------------------------------------","metadata":{}},{"cell_type":"markdown","source":"# Creating Data In Correct Format","metadata":{}},{"cell_type":"code","source":"# Set constants and pick important landmarks\nLANDMARK_IDX = [0, 9, 11, 13, 14, 17, 117, 118, 119, 199, 346, 347, 348] + list(\n    range(468, 543)\n)","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:59.192892Z","iopub.execute_input":"2024-02-27T17:17:59.193278Z","iopub.status.idle":"2024-02-27T17:17:59.20269Z","shell.execute_reply.started":"2024-02-27T17:17:59.193233Z","shell.execute_reply":"2024-02-27T17:17:59.201685Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# set files directories\nLANDMARK_FILES_DIR = \"/kaggle/input/asl-signs/train_landmark_files\"\nTRAIN_FILE = \"/kaggle/input/asl-signs/train.csv\"\nROOT_DIR = \"/kaggle/input/asl-signs/\"\nDATA_PATH = \"/kaggle/input/saved-tfdataset-of-google-isl-recognition-data/GoogleISLDatasetBatched\"\nDS_CARDINALITY = 185\nVAL_SIZE = 18\nN_SIGNS = 250\nROWS_PER_FRAME = 543","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:59.2041Z","iopub.execute_input":"2024-02-27T17:17:59.204765Z","iopub.status.idle":"2024-02-27T17:17:59.213716Z","shell.execute_reply.started":"2024-02-27T17:17:59.204724Z","shell.execute_reply":"2024-02-27T17:17:59.212826Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def preprocess(ragged_batch, labels):\n    ragged_batch = tf.gather(ragged_batch, LANDMARK_IDX, axis=2)\n    ragged_batch = tf.where(\n        tf.math.is_nan(ragged_batch), tf.zeros_like(ragged_batch), ragged_batch\n    )\n    return tf.concat([ragged_batch[..., i] for i in range(3)], -1), labels\n\n\ndataset = tf.data.Dataset.load(DATA_PATH)\ndataset = dataset.map(preprocess)\nval_ds = dataset.take(VAL_SIZE).cache().prefetch(tf.data.AUTOTUNE)\ntrain_ds = dataset.skip(VAL_SIZE).cache().shuffle(20).prefetch(tf.data.AUTOTUNE)","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:17:59.21507Z","iopub.execute_input":"2024-02-27T17:17:59.215569Z","iopub.status.idle":"2024-02-27T17:18:02.687593Z","shell.execute_reply.started":"2024-02-27T17:17:59.215528Z","shell.execute_reply":"2024-02-27T17:18:02.686383Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **BUILDING MODEL**","metadata":{}},{"cell_type":"code","source":"# include early stopping and reducelr\ndef get_callbacks():\n    return [\n        tf.keras.callbacks.EarlyStopping(\n            monitor=\"val_accuracy\", patience=10, restore_best_weights=True\n        ),\n        tf.keras.callbacks.ReduceLROnPlateau(\n            monitor=\"val_accuracy\", factor=0.5, patience=3\n        ),\n    ]\n\n\n# a single dense block followed by a normalization block and relu activation\ndef dense_block(units, name):\n    fc = layers.Dense(units)\n    norm = layers.LayerNormalization()\n    act = layers.Activation(\"relu\")\n    drop = layers.Dropout(0.1)\n    return lambda x: drop(act(norm(fc(x))))\n\n\n# the lstm block with the final dense block for the classification\ndef classifier(lstm_units):\n    lstm = layers.LSTM(lstm_units)\n    out = layers.Dense(N_SIGNS, activation=\"softmax\")\n    return lambda x: out(lstm(x))","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:18:02.689198Z","iopub.execute_input":"2024-02-27T17:18:02.69011Z","iopub.status.idle":"2024-02-27T17:18:02.708523Z","shell.execute_reply.started":"2024-02-27T17:18:02.690061Z","shell.execute_reply":"2024-02-27T17:18:02.707211Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# choose the number of nodes per layer\nencoder_units = [512, 256]  # tune this\nlstm_units = 500  # tune this\n\n# define the inputs (ragged batches of time series of landmark coordinates)\ninputs = tf.keras.Input(shape=(None, 3 * len(LANDMARK_IDX)), ragged=True)\n\n# dense encoder model\nx = inputs\nfor i, n in enumerate(encoder_units):\n    x = dense_block(n, f\"encoder_{i}\")(x)\n\n# classifier model\nout = classifier(lstm_units)(x)\n\nmodel = tf.keras.Model(inputs=inputs, outputs=out)\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:18:02.709924Z","iopub.execute_input":"2024-02-27T17:18:02.710297Z","iopub.status.idle":"2024-02-27T17:18:03.469568Z","shell.execute_reply.started":"2024-02-27T17:18:02.710265Z","shell.execute_reply":"2024-02-27T17:18:03.468372Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# add a decreasing learning rate scheduler to help convergence\nsteps_per_epoch = DS_CARDINALITY - VAL_SIZE\nboundaries = [steps_per_epoch * n for n in [30, 50, 70]]\nvalues = [1e-3, 1e-4, 1e-5, 1e-6]\nlr_sched = optimizers.schedules.PiecewiseConstantDecay(boundaries, values)\noptimizer = optimizers.Adam(lr_sched)\n\nmodel.compile(\n    optimizer=optimizer,\n    loss=\"sparse_categorical_crossentropy\",\n    metrics=[\"accuracy\", \"sparse_top_k_categorical_accuracy\"],\n)","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:18:03.47119Z","iopub.execute_input":"2024-02-27T17:18:03.471547Z","iopub.status.idle":"2024-02-27T17:18:03.533046Z","shell.execute_reply.started":"2024-02-27T17:18:03.471496Z","shell.execute_reply":"2024-02-27T17:18:03.531829Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# fit the model with 100 epochs iteration\nmodel.fit(train_ds, validation_data=val_ds, callbacks=get_callbacks(), epochs=100)","metadata":{"execution":{"iopub.status.busy":"2024-02-27T17:18:03.534409Z","iopub.execute_input":"2024-02-27T17:18:03.534719Z","iopub.status.idle":"2024-02-27T18:13:17.782077Z","shell.execute_reply.started":"2024-02-27T17:18:03.534687Z","shell.execute_reply":"2024-02-27T18:13:17.781046Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.summary(expand_nested=True)","metadata":{"execution":{"iopub.status.busy":"2024-02-27T18:13:17.783758Z","iopub.execute_input":"2024-02-27T18:13:17.784066Z","iopub.status.idle":"2024-02-27T18:13:17.820761Z","shell.execute_reply.started":"2024-02-27T18:13:17.784037Z","shell.execute_reply":"2024-02-27T18:13:17.819954Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_inference_model(model):\n    inputs = tf.keras.Input(shape=(ROWS_PER_FRAME, 3), name=\"inputs\")\n\n    # drop most of the face mesh\n    x = tf.gather(inputs, LANDMARK_IDX, axis=1)\n\n    # fill nan\n    x = tf.where(tf.math.is_nan(x), tf.zeros_like(x), x)\n\n    # flatten landmark xyz coordinates ()\n    x = tf.concat([x[..., i] for i in range(3)], -1)\n\n    x = tf.expand_dims(x, 0)\n\n    # call trained model\n    out = model(x)\n\n    # explicitly name the final (identity) layer for the submission format\n    out = layers.Activation(\"linear\", name=\"outputs\")(out)\n\n    inference_model = tf.keras.Model(inputs=inputs, outputs=out)\n    inference_model.compile(loss=\"sparse_categorical_crossentropy\", metrics=\"accuracy\")\n    return inference_model","metadata":{"execution":{"iopub.status.busy":"2024-02-27T18:13:17.829295Z","iopub.execute_input":"2024-02-27T18:13:17.829571Z","iopub.status.idle":"2024-02-27T18:13:17.852665Z","shell.execute_reply.started":"2024-02-27T18:13:17.829543Z","shell.execute_reply":"2024-02-27T18:13:17.851552Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inference_model = get_inference_model(model)\ninference_model.summary(expand_nested=True)","metadata":{"execution":{"iopub.status.busy":"2024-02-27T18:13:17.853958Z","iopub.execute_input":"2024-02-27T18:13:17.854257Z","iopub.status.idle":"2024-02-27T18:13:18.318824Z","shell.execute_reply.started":"2024-02-27T18:13:17.854228Z","shell.execute_reply":"2024-02-27T18:13:18.317698Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tensorflow.keras.models import save_model","metadata":{"execution":{"iopub.status.busy":"2024-02-27T18:13:18.32014Z","iopub.execute_input":"2024-02-27T18:13:18.320457Z","iopub.status.idle":"2024-02-27T18:13:18.329362Z","shell.execute_reply.started":"2024-02-27T18:13:18.320425Z","shell.execute_reply":"2024-02-27T18:13:18.32752Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# saving model\nsave_model(inference_model, \"gslrmodel.h5\")","metadata":{"execution":{"iopub.status.busy":"2024-02-27T18:14:54.914241Z","iopub.execute_input":"2024-02-27T18:14:54.915151Z","iopub.status.idle":"2024-02-27T18:14:54.981754Z","shell.execute_reply.started":"2024-02-27T18:14:54.915103Z","shell.execute_reply":"2024-02-27T18:14:54.980834Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # save the tflite version of model\n# converter = tf.lite.TFLiteConverter.from_keras_model(inference_model)\n# tflite_model = converter.convert()e\n# model_path = \"model.tflite\"\n\n# # submit the model\n# with open(model_path, \"wb\") as f:\n#     f.write(tflite_model)\n# !zip submission.zip $model_path","metadata":{"execution":{"iopub.status.busy":"2024-02-27T18:13:18.902911Z","iopub.status.idle":"2024-02-27T18:13:18.903294Z","shell.execute_reply.started":"2024-02-27T18:13:18.903097Z","shell.execute_reply":"2024-02-27T18:13:18.903116Z"},"trusted":true},"outputs":[],"execution_count":null}]}