{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Google - Isolated Sign Language Recognition\nThe goal of this competition is to classify isolated American Sign Language (ASL) signs.\n\nThe landmarks were extracted from raw videos with the MediaPipe holistic model and are asked to predict sign from this data.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.notebook import tqdm\n\nplt.style.use(\"seaborn-colorblind\")","metadata":{"execution":{"iopub.status.busy":"2023-03-15T15:42:10.776472Z","iopub.execute_input":"2023-03-15T15:42:10.777463Z","iopub.status.idle":"2023-03-15T15:42:11.974844Z","shell.execute_reply.started":"2023-03-15T15:42:10.777421Z","shell.execute_reply":"2023-03-15T15:42:11.973663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Install nb_black for autoformating\n!pip install nb_black --quiet\n%load_ext lab_black","metadata":{"execution":{"iopub.status.busy":"2023-03-15T15:42:22.585602Z","iopub.execute_input":"2023-03-15T15:42:22.586034Z","iopub.status.idle":"2023-03-15T15:42:39.090517Z","shell.execute_reply.started":"2023-03-15T15:42:22.585995Z","shell.execute_reply":"2023-03-15T15:42:39.089170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data EDA","metadata":{}},{"cell_type":"code","source":"!ls ../input/asl-signs/ -GFlash --color","metadata":{"execution":{"iopub.status.busy":"2023-03-12T08:59:06.967271Z","iopub.execute_input":"2023-03-12T08:59:06.967828Z","iopub.status.idle":"2023-03-12T08:59:08.067056Z","shell.execute_reply.started":"2023-03-12T08:59:06.967769Z","shell.execute_reply":"2023-03-12T08:59:08.065594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_DIR = \"../input/asl-signs/\"\ntrain = pd.read_csv(f\"{BASE_DIR}/train.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-03-12T08:59:08.070843Z","iopub.execute_input":"2023-03-12T08:59:08.071242Z","iopub.status.idle":"2023-03-12T08:59:08.283858Z","shell.execute_reply.started":"2023-03-12T08:59:08.071199Z","shell.execute_reply":"2023-03-12T08:59:08.282448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train.csv has the path to each parquet file: participant_id, sequence_id and sign.\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-12T08:59:08.285335Z","iopub.execute_input":"2023-03-12T08:59:08.285686Z","iopub.status.idle":"2023-03-12T08:59:08.317523Z","shell.execute_reply.started":"2023-03-12T08:59:08.285652Z","shell.execute_reply":"2023-03-12T08:59:08.316281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# What Signs are we trying to predict?\n- 250 Unique Signs\n- Ranging from 299-415 examples of each","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(8, 8))\ntrain[\"sign\"].value_counts().head(50).sort_values(ascending=True).plot(\n    kind=\"barh\", ax=ax, title=\"Top 50 Signs in Training Dataset\"\n)\nax.set_xlabel(\"Number of Trainig Examples\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-12T08:59:08.319467Z","iopub.execute_input":"2023-03-12T08:59:08.320352Z","iopub.status.idle":"2023-03-12T08:59:09.144462Z","shell.execute_reply.started":"2023-03-12T08:59:08.320303Z","shell.execute_reply":"2023-03-12T08:59:09.143403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(8, 8))\ntrain[\"sign\"].value_counts().tail(50).sort_values(ascending=True).plot(\n    kind=\"barh\", ax=ax, title=\"Bottom 50 Signs in Training Dataset\"\n)\nax.set_xlabel(\"Number of Trainig Examples\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-12T08:59:09.145736Z","iopub.execute_input":"2023-03-12T08:59:09.146253Z","iopub.status.idle":"2023-03-12T08:59:09.876555Z","shell.execute_reply.started":"2023-03-12T08:59:09.146216Z","shell.execute_reply":"2023-03-12T08:59:09.875272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Parquet Landmark Data\n- Each parquet file is in the path:\n   - train_landmark_files/[participant_id]/[sequence_id].parquet\n- The parquet's associated sign can be found in train.csv","metadata":{}},{"cell_type":"markdown","source":"# Pull an exampple parquet file data...\nPull an example landmark file for the sign \"listen\"","metadata":{}},{"cell_type":"code","source":"example_fn = train.query('sign == \"listen\"')[\"path\"].values[0]\n\nexample_landmark = pd.read_parquet(f\"{BASE_DIR}/{example_fn}\")\nexample_landmark.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-12T08:59:09.878374Z","iopub.execute_input":"2023-03-12T08:59:09.879107Z","iopub.status.idle":"2023-03-12T08:59:10.023758Z","shell.execute_reply.started":"2023-03-12T08:59:09.879053Z","shell.execute_reply":"2023-03-12T08:59:10.022818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_frames = example_landmark[\"frame\"].nunique()\nunique_types = example_landmark[\"type\"].nunique()\ntypes_in_video = example_landmark[\"type\"].unique()\nprint(\n    f\"The file has {unique_frames} unique frames and {unique_types} unique types: {types_in_video}\"\n)","metadata":{"execution":{"iopub.status.busy":"2023-03-12T08:59:10.025273Z","iopub.execute_input":"2023-03-12T08:59:10.026057Z","iopub.status.idle":"2023-03-12T08:59:10.040247Z","shell.execute_reply.started":"2023-03-12T08:59:10.026005Z","shell.execute_reply":"2023-03-12T08:59:10.038637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Compare for a bunch of parquet files what type of data we have.\n- the number of frames is not consistent\n- every file has 4 types of landmarks: face, left_hand, pose and right_hand.","metadata":{}},{"cell_type":"code","source":"listen_files = train.query('sign == \"listen\"')[\"path\"].values\nfor i, f in enumerate(listen_files):\n    example_landmark = pd.read_parquet(f\"{BASE_DIR}/{f}\")\n    unique_frames = example_landmark[\"frame\"].nunique()\n    unique_types = example_landmark[\"type\"].nunique()\n    types_in_video = example_landmark[\"type\"].unique()\n    print(\n        f\"The file has {unique_frames} unique frames and {unique_types} unique types: {types_in_video}\"\n    )\n    if i == 20:\n        break","metadata":{"execution":{"iopub.status.busy":"2023-03-12T08:59:10.044490Z","iopub.execute_input":"2023-03-12T08:59:10.045232Z","iopub.status.idle":"2023-03-12T08:59:10.643694Z","shell.execute_reply.started":"2023-03-12T08:59:10.045187Z","shell.execute_reply":"2023-03-12T08:59:10.642698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Metadata for Training Dataset","metadata":{}},{"cell_type":"code","source":"N_PARQUETS_TO_READ = 1_000  # 100_000  # So we don't have to load all 95k\n\ncombined_meta = {}\nfor i, d in tqdm(train.iterrows(), total=len(train)):\n    file_path = d[\"path\"]\n    example_landmark = pd.read_parquet(f\"{BASE_DIR}/{file_path}\")\n    # Get the number of landmarks with x,y,z data per type\n    meta = (\n        example_landmark.dropna(subset=[\"x\", \"y\", \"z\"])[\"type\"].value_counts().to_dict()\n    )\n    meta[\"frames\"] = example_landmark[\"frame\"].nunique()\n    xyz_meta = (\n        example_landmark.agg(\n            {\n                \"x\": [\"min\", \"max\", \"mean\"],\n                \"y\": [\"min\", \"max\", \"mean\"],\n                \"z\": [\"min\", \"max\", \"mean\"],\n            }\n        )\n        .unstack()\n        .to_dict()\n    )\n\n    for key in xyz_meta.keys():\n        new_key = key[0] + \"_\" + key[1]\n        meta[new_key] = xyz_meta[key]\n    combined_meta[file_path] = meta\n    if i == N_PARQUETS_TO_READ:\n        break","metadata":{"execution":{"iopub.status.busy":"2023-03-12T08:59:10.645095Z","iopub.execute_input":"2023-03-12T08:59:10.645669Z","iopub.status.idle":"2023-03-12T08:59:45.651507Z","shell.execute_reply.started":"2023-03-12T08:59:10.645632Z","shell.execute_reply":"2023-03-12T08:59:45.650303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_with_meta = train.merge(\n    pd.DataFrame(combined_meta).T.reset_index().rename(columns={\"index\": \"path\"}),\n    how=\"left\",\n)\ntrain_with_meta.to_parquet(\"train_with_meta.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-03-12T08:59:45.653390Z","iopub.execute_input":"2023-03-12T08:59:45.654170Z","iopub.status.idle":"2023-03-12T08:59:45.863768Z","shell.execute_reply.started":"2023-03-12T08:59:45.654121Z","shell.execute_reply":"2023-03-12T08:59:45.862731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## The most frequent types of landmarks provided\n- Face has a lot more datapoints because mediapipe provides 468 3D datapoints per frame.","metadata":{}},{"cell_type":"code","source":"train_with_meta[[\"face\", \"pose\", \"left_hand\", \"right_hand\"]].sum().sort_values().plot(\n    kind=\"barh\", title=\"Sum of Rows by Landmark Type\"\n)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-12T08:59:45.865106Z","iopub.execute_input":"2023-03-12T08:59:45.865424Z","iopub.status.idle":"2023-03-12T08:59:46.087394Z","shell.execute_reply.started":"2023-03-12T08:59:45.865393Z","shell.execute_reply":"2023-03-12T08:59:46.086178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Every parquet file has at least some datapoints for all four types of landmarks:\n- Face, pose, left hand and right hand.\n","metadata":{}},{"cell_type":"code","source":"# checking to see if the number of landmarks for this type is zero\n(\n    train_with_meta.query(\"index < 1000\").fillna(0)[\n        [\"face\", \"pose\", \"left_hand\", \"right_hand\"]\n    ]\n    > 0\n).mean().plot(kind=\"barh\", title=\"Rate of Frame/Keypoints with Data\")","metadata":{"execution":{"iopub.status.busy":"2023-03-12T08:59:46.091055Z","iopub.execute_input":"2023-03-12T08:59:46.091403Z","iopub.status.idle":"2023-03-12T08:59:46.320840Z","shell.execute_reply.started":"2023-03-12T08:59:46.091370Z","shell.execute_reply":"2023-03-12T08:59:46.319907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Check out one Example","metadata":{}},{"cell_type":"code","source":"example_fn = train_with_meta.dropna().query('sign == \"shhh\"')[\"path\"].values[0]\nexample_landmark = pd.read_parquet(f\"{BASE_DIR}/{example_fn}\")","metadata":{"execution":{"iopub.status.busy":"2023-03-12T08:59:46.322337Z","iopub.execute_input":"2023-03-12T08:59:46.323013Z","iopub.status.idle":"2023-03-12T08:59:46.360814Z","shell.execute_reply.started":"2023-03-12T08:59:46.322944Z","shell.execute_reply":"2023-03-12T08:59:46.359391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark.query(\"frame == 25\")[\"type\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-03-12T08:59:46.362422Z","iopub.execute_input":"2023-03-12T08:59:46.362792Z","iopub.status.idle":"2023-03-12T08:59:46.378588Z","shell.execute_reply.started":"2023-03-12T08:59:46.362756Z","shell.execute_reply":"2023-03-12T08:59:46.377218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark[\"no_xyz\"] = example_landmark[\"x\"].isna()","metadata":{"execution":{"iopub.status.busy":"2023-03-12T08:59:46.380172Z","iopub.execute_input":"2023-03-12T08:59:46.380533Z","iopub.status.idle":"2023-03-12T08:59:46.388497Z","shell.execute_reply.started":"2023-03-12T08:59:46.380498Z","shell.execute_reply":"2023-03-12T08:59:46.387207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark.groupby(\"frame\")[\"no_xyz\"].sum().plot(\n    title=\"missing xyz per frame\", kind=\"bar\"\n)","metadata":{"execution":{"iopub.status.busy":"2023-03-12T08:59:46.390965Z","iopub.execute_input":"2023-03-12T08:59:46.391593Z","iopub.status.idle":"2023-03-12T08:59:46.752279Z","shell.execute_reply.started":"2023-03-12T08:59:46.391536Z","shell.execute_reply":"2023-03-12T08:59:46.750928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3D plot of Landmarks from \"shhh\" example\nPick frame 17 because we have no missing xyz data","metadata":{}},{"cell_type":"code","source":"import plotly.express as px\n\nexample_frame = example_landmark.query(\"frame == 17\")\npx.scatter_3d(example_frame, x=\"x\", y=\"y\", z=\"z\", color=\"type\")","metadata":{"execution":{"iopub.status.busy":"2023-03-12T10:01:05.734591Z","iopub.execute_input":"2023-03-12T10:01:05.735035Z","iopub.status.idle":"2023-03-12T10:01:05.821744Z","shell.execute_reply.started":"2023-03-12T10:01:05.734997Z","shell.execute_reply":"2023-03-12T10:01:05.820467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark[\"y_\"] = example_landmark[\"y\"] * -1\nexample_frame = example_landmark.query(\"frame == 17 and type== 'face'\")\npx.scatter(example_frame, x=\"x\", y=\"y_\", color=\"type\")","metadata":{"execution":{"iopub.status.busy":"2023-03-12T09:32:32.391176Z","iopub.execute_input":"2023-03-12T09:32:32.391652Z","iopub.status.idle":"2023-03-12T09:32:32.569055Z","shell.execute_reply.started":"2023-03-12T09:32:32.391608Z","shell.execute_reply":"2023-03-12T09:32:32.567807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark[\"y_\"] = example_landmark[\"y\"] * -1\nexample_frame = example_landmark.query(\"frame == 17 and type== 'left_hand'\")\npx.scatter(example_frame, x=\"x\", y=\"y_\", color=\"type\")","metadata":{"execution":{"iopub.status.busy":"2023-03-12T09:34:27.551587Z","iopub.execute_input":"2023-03-12T09:34:27.552055Z","iopub.status.idle":"2023-03-12T09:34:27.629517Z","shell.execute_reply.started":"2023-03-12T09:34:27.552011Z","shell.execute_reply":"2023-03-12T09:34:27.628289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Media pipe hand connections\n- draw the example","metadata":{}},{"cell_type":"code","source":"!pip install mediapipe --quiet","metadata":{"execution":{"iopub.status.busy":"2023-03-12T10:08:41.777365Z","iopub.execute_input":"2023-03-12T10:08:41.777847Z","iopub.status.idle":"2023-03-12T10:08:53.284583Z","shell.execute_reply.started":"2023-03-12T10:08:41.777803Z","shell.execute_reply":"2023-03-12T10:08:53.283201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import mediapipe as mp\n\nmp_hands = mp.solutions.hands\n\nexample_landmark[\"y_\"] = example_landmark[\"y\"] * -1\n\nfig, ax = plt.subplots(figsize=(5, 5))\n\nfor hand in [\"left_hand\", \"right_hand\"]:\n    example_hand = example_landmark.query(\"frame == 17 and type== @hand\")\n\n    ax.scatter(example_hand[\"x\"], example_hand[\"y_\"])\n\n    for connection in mp_hands.HAND_CONNECTIONS:\n        point_a = connection[0]\n        point_b = connection[1]\n        x1, y1 = example_hand.query(\"landmark_index == @point_a\")[[\"x\", \"y_\"]].values[0]\n        x2, y2 = example_hand.query(\"landmark_index == @point_b\")[[\"x\", \"y_\"]].values[0]\n        plt.plot([x1, x2], [y1, y2], color=\"purple\")\nax.set_title(\"Shhh - Hands Data\")","metadata":{"execution":{"iopub.status.busy":"2023-03-12T10:12:15.046444Z","iopub.execute_input":"2023-03-12T10:12:15.047658Z","iopub.status.idle":"2023-03-12T10:12:16.255796Z","shell.execute_reply.started":"2023-03-12T10:12:15.047613Z","shell.execute_reply":"2023-03-12T10:12:16.254437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Try to use Mediapipe to plot\n- some example images\n- run mediapipe holistic to see how it produces the results\n- plot them on a image","metadata":{}},{"cell_type":"code","source":"!wget https://previews.123rf.com/images/mimagephotography/mimagephotography1411/mimagephotography141100022/33214722-full-length-portrait-of-a-fashionable-young-man-standing-on-isolated-white-background.jpg --quiet\n!wget https://images.app.goo.gl/N2zHEmcn6GuJRDsf7 --quiet\n!pip install mediapipe --quiet","metadata":{"execution":{"iopub.status.busy":"2023-03-12T09:26:53.789221Z","iopub.execute_input":"2023-03-12T09:26:53.789634Z","iopub.status.idle":"2023-03-12T09:27:07.751577Z","shell.execute_reply.started":"2023-03-12T09:26:53.789597Z","shell.execute_reply":"2023-03-12T09:27:07.750373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nimport os\nimport mediapipe as mp\n\nmp_drawing = mp.solutions.drawing_utils\nmp_drawing_styles = mp.solutions.drawing_styles\nmp_holistic = mp.solutions.holistic\n\n# For static images:\nIMAGE_FILES = [\n    \"33214722-full-length-portrait-of-a-fashionable-young-man-standing-on-isolated-white-background.jpg\"\n]\nBG_COLOR = (192, 192, 192)  # gray\nwith mp_holistic.Holistic(\n    static_image_mode=True,\n    model_complexity=2,\n    enable_segmentation=True,\n    refine_face_landmarks=True,\n) as holistic:\n    for idx, file in enumerate(IMAGE_FILES):\n        if not os.path.isfile(file):\n            print(f\"{file} does not exist\")\n            continue\n\n        image = cv2.imread(file)\n\n        if image is None:\n            print(f\"Failed to read {file}\")\n            continue\n\n        image_height, image_width, _ = image.shape\n        # Convert the BGR image to RGB before processing.\n        results = holistic.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))\n\n        if results.pose_landmarks:\n            print(\n                f\"Nose coordinates: (\"\n                f\"{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].x * image_width}, \"\n                f\"{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].y * image_height})\"\n            )\n\n        annotated_image = image.copy()\n        # Draw segmentation on the image.\n        # To improve segmentation around boundaries, consider applying a joint\n        # bilateral filter to \"results.segmentation_mask\" with \"image\".\n        condition = np.stack((results.segmentation_mask,) * 3, axis=-1) > 0.1\n        bg_image = np.zeros(image.shape, dtype=np.uint8)\n        bg_image[:] = BG_COLOR\n        annotated_image = np.where(condition, annotated_image, bg_image)\n        # Draw pose, left and right hands, and face landmarks on the image.\n        mp_drawing.draw_landmarks(\n            annotated_image,\n            results.face_landmarks,\n            mp_holistic.FACEMESH_TESSELATION,\n            landmark_drawing_spec=None,\n            connection_drawing_spec=mp_drawing_styles.get_default_face_mesh_tesselation_style(),\n        )\n        mp_drawing.draw_landmarks(\n            annotated_image,\n            results.pose_landmarks,\n            mp_holistic.POSE_CONNECTIONS,\n            landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style(),\n        )\n#         cv2.imwrite(\"/tmp/annotated_image\" + str(idx) + \".png\", annotated_image)\n#         # Plot pose world landmarks.\n#         mp_drawing.plot_landmarks(\n#             results.pose_world_landmarks, mp_holistic.POSE_CONNECTIONS\n#         )","metadata":{"execution":{"iopub.status.busy":"2023-03-12T10:01:24.839909Z","iopub.execute_input":"2023-03-12T10:01:24.840363Z","iopub.status.idle":"2023-03-12T10:01:25.562645Z","shell.execute_reply.started":"2023-03-12T10:01:24.840325Z","shell.execute_reply":"2023-03-12T10:01:25.561252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(image)","metadata":{"execution":{"iopub.status.busy":"2023-03-12T09:00:09.315497Z","iopub.execute_input":"2023-03-12T09:00:09.316166Z","iopub.status.idle":"2023-03-12T09:00:09.761135Z","shell.execute_reply.started":"2023-03-12T09:00:09.316128Z","shell.execute_reply":"2023-03-12T09:00:09.760033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.imshow(plt.imread(\"/tmp/annotated_image\" + str(idx) + \".png\"))","metadata":{"execution":{"iopub.status.busy":"2023-03-12T09:00:09.763077Z","iopub.execute_input":"2023-03-12T09:00:09.763588Z","iopub.status.idle":"2023-03-12T09:00:10.286623Z","shell.execute_reply.started":"2023-03-12T09:00:09.763540Z","shell.execute_reply":"2023-03-12T09:00:10.285225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Try to use the same format for plotting of parquet data","metadata":{}},{"cell_type":"code","source":"# train_with_meta[\"x_min\"].values\n# train_with_meta[\"x_max\"].values\n\n# annotated_image.shape  # (1300, 825, 3)\nbackground_image = np.zeros((1300, 825, 3))\n\nexample_frame","metadata":{"execution":{"iopub.status.busy":"2023-03-12T09:05:38.195774Z","iopub.execute_input":"2023-03-12T09:05:38.196229Z","iopub.status.idle":"2023-03-12T09:05:38.228494Z","shell.execute_reply.started":"2023-03-12T09:05:38.196193Z","shell.execute_reply":"2023-03-12T09:05:38.225755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# type(results.face_landmarks)\n\nfrom mediapipe.framework.formats import landmark_pb2\n\nface_landmarks = landmark_pb2.NormalizedLandmarkList()","metadata":{"execution":{"iopub.status.busy":"2023-03-12T09:04:04.489770Z","iopub.execute_input":"2023-03-12T09:04:04.490717Z","iopub.status.idle":"2023-03-12T09:04:04.501042Z","shell.execute_reply.started":"2023-03-12T09:04:04.490666Z","shell.execute_reply":"2023-03-12T09:04:04.498737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"mediapipe.framework.formats.landmark_pb2.NormalizedLandmarkList\n","metadata":{}},{"cell_type":"code","source":"type(face_landmarks)","metadata":{"execution":{"iopub.status.busy":"2023-03-12T09:04:22.405389Z","iopub.execute_input":"2023-03-12T09:04:22.406341Z","iopub.status.idle":"2023-03-12T09:04:22.415572Z","shell.execute_reply.started":"2023-03-12T09:04:22.406289Z","shell.execute_reply":"2023-03-12T09:04:22.414121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"mediapipe.framework.formats.landmark_pb2.NormalizedLandmarkList","metadata":{}},{"cell_type":"code","source":"mp_drawing.draw_landmarks(\n    background_image,\n    results.face_landmarks,\n    mp_holistic.FACEMESH_TESSELATION,\n    landmark_drawing_spec=None,\n    connection_drawing_spec=mp_drawing_styles.get_default_face_mesh_tesselation_style(),\n)\nmp_drawing.draw_landmarks(\n    background_image,\n    results.pose_landmarks,\n    mp_holistic.POSE_CONNECTIONS,\n    landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style(),\n)\nplt.imshow(background_image)","metadata":{"execution":{"iopub.status.busy":"2023-03-12T09:21:37.855866Z","iopub.execute_input":"2023-03-12T09:21:37.856570Z","iopub.status.idle":"2023-03-12T09:21:38.490734Z","shell.execute_reply.started":"2023-03-12T09:21:37.856516Z","shell.execute_reply":"2023-03-12T09:21:38.489086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# results.face_landmarks.landmark[0].x\n\nexample_frame.plot(x=\"x\", y=\"y\", style=\".\")","metadata":{"execution":{"iopub.status.busy":"2023-03-12T09:17:56.966690Z","iopub.execute_input":"2023-03-12T09:17:56.967162Z","iopub.status.idle":"2023-03-12T09:17:57.234703Z","shell.execute_reply.started":"2023-03-12T09:17:56.967097Z","shell.execute_reply":"2023-03-12T09:17:57.233233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_frame.query('type == \"face\"')[[\"x\", \"y\", \"z\"]].values","metadata":{"execution":{"iopub.status.busy":"2023-03-12T09:15:39.950834Z","iopub.execute_input":"2023-03-12T09:15:39.951234Z","iopub.status.idle":"2023-03-12T09:15:39.968111Z","shell.execute_reply.started":"2023-03-12T09:15:39.951200Z","shell.execute_reply":"2023-03-12T09:15:39.966777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(background_image)","metadata":{"execution":{"iopub.status.busy":"2023-03-12T09:00:10.336321Z","iopub.execute_input":"2023-03-12T09:00:10.336700Z","iopub.status.idle":"2023-03-12T09:00:10.937651Z","shell.execute_reply.started":"2023-03-12T09:00:10.336656Z","shell.execute_reply":"2023-03-12T09:00:10.936207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transform the parquet file data into mediapipe '*NormalizedLandmarkList*'\n","metadata":{}},{"cell_type":"markdown","source":"# Evaluation\nThe evaluation metric for this contest is simple classification accuracy.\n\n","metadata":{}},{"cell_type":"code","source":"ROWS_PER_FRAME = 543  # number of landmarks per frame\n\n\ndef load_relevant_data_subset(pq_path):\n    data_columns = [\"x\", \"y\", \"z\"]\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2023-03-12T09:00:10.938960Z","iopub.execute_input":"2023-03-12T09:00:10.939305Z","iopub.status.idle":"2023-03-12T09:00:10.951810Z","shell.execute_reply.started":"2023-03-12T09:00:10.939273Z","shell.execute_reply":"2023-03-12T09:00:10.950160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference \nIgnoring details like how we manage multiple videos","metadata":{}},{"cell_type":"code","source":"# import tflite_runtime.interpreter as tflite\n# def run_model(model_path):\n#     interpreter = tflite.Interpreter(model_path)\n\n#     found_signatures = list(interpreter.get_signature_list().keys())\n\n#     if REQUIRED_SIGNATURE not in found_signatures:\n#         raise KernelEvalException('Required input signature not found.')\n\n#     prediction_fn = interpreter.get_signature_runner(\"serving_default\")\n#     output = prediction_fn(inputs=frames)\n#     sign = np.argmax(output[\"outputs\"])","metadata":{"execution":{"iopub.status.busy":"2023-03-12T09:00:10.953383Z","iopub.execute_input":"2023-03-12T09:00:10.953896Z","iopub.status.idle":"2023-03-12T09:00:10.965509Z","shell.execute_reply.started":"2023-03-12T09:00:10.953828Z","shell.execute_reply":"2023-03-12T09:00:10.963850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ASL Sign Detection - PyTorch Lightning\n\n","metadata":{}},{"cell_type":"markdown","source":"## Versions:\n11: Fix reset_index bug when setting up train_df and val_df in ASLDataModule in combination with __getitem()__, which caused unrealistic high val accuracies:","metadata":{}},{"cell_type":"code","source":"def setup(self, stage=None):\n    val_fold = self.hparams.val_fold\n\n    train_df = self.df[self.df.fold != val_fold].reset_index(\n        drop=True\n    )  # Removed .reset_index in version 11\n    val_df = self.df[self.df.fold == val_fold].reset_index(\n        drop=True\n    )  # Removed .reset_index in version 11\n\n\ndef __getitem__(self, idx):\n    # Use df_index as idx due to folds splitting\n    df_index = self.df.index.values[idx]\n\n    x = self.X[df_index]\n    y = self.y[df_index]","metadata":{"execution":{"iopub.status.busy":"2023-03-15T15:51:39.637464Z","iopub.execute_input":"2023-03-15T15:51:39.637930Z","iopub.status.idle":"2023-03-15T15:51:39.653439Z","shell.execute_reply.started":"2023-03-15T15:51:39.637887Z","shell.execute_reply":"2023-03-15T15:51:39.652567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Installs","metadata":{}},{"cell_type":"code","source":"!pip install onnx-tf tflite-runtime nb_black -q\n!pip install --upgrade -q wandb","metadata":{"execution":{"iopub.status.busy":"2023-03-15T15:47:14.049161Z","iopub.execute_input":"2023-03-15T15:47:14.049602Z","iopub.status.idle":"2023-03-15T15:47:41.666627Z","shell.execute_reply.started":"2023-03-15T15:47:14.049556Z","shell.execute_reply":"2023-03-15T15:47:41.664902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Imports","metadata":{}},{"cell_type":"code","source":"import json\nimport math\nfrom pathlib import Path\n\nimport numpy as np\nimport onnx\nimport pandas as pd\nimport pytorch_lightning as pl\nimport tensorflow as tf\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchmetrics\nimport wandb\nfrom kaggle_secrets import UserSecretsClient\nfrom onnx_tf.backend import prepare\nfrom pytorch_lightning.callbacks import EarlyStopping\nfrom pytorch_lightning.callbacks import LearningRateMonitor\nfrom pytorch_lightning.loggers import WandbLogger\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom timm.optim import create_optimizer_v2\nfrom torchmetrics import MetricCollection","metadata":{"execution":{"iopub.status.busy":"2023-03-15T16:02:44.998914Z","iopub.execute_input":"2023-03-15T16:02:44.999326Z","iopub.status.idle":"2023-03-15T16:02:45.013027Z","shell.execute_reply.started":"2023-03-15T16:02:44.999293Z","shell.execute_reply":"2023-03-15T16:02:45.011716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# WandB\n- Weights & Biases (WandB) is a python package that allows us to monitor our training in real-time","metadata":{}},{"cell_type":"code","source":"# user_secrets = UserSecretsClient()\n# # wandb_api = UserSecretsClient().get_secret(secret_label)\n\n# wandb_api = user_secrets.get_secret(\"wandb_api\")\n# wandb.login(key=wandb_api)","metadata":{"execution":{"iopub.status.busy":"2023-03-15T16:03:26.389085Z","iopub.execute_input":"2023-03-15T16:03:26.389516Z","iopub.status.idle":"2023-03-15T16:03:26.395435Z","shell.execute_reply.started":"2023-03-15T16:03:26.389477Z","shell.execute_reply":"2023-03-15T16:03:26.394146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Paths & Settings\n","metadata":{}},{"cell_type":"code","source":"# KAGGLE_DIR = Path(\"/\") / \"kaggle\"\n# INPUT_DIR = KAGGLE_DIR / \"input\"\n# DATA_ROOT_DIR = INPUT_DIR / \"asl-signs\"\n# NPY_DATA_DIR = INPUT_DIR / \"train_landmark_files\"\n# MY_NPY_DATA_DIR = INPUT_DIR / \"asl-signs-features-npy\"\n\n# TRAIN_CSV_PATH = MY_NPY_DATA_DIR / \"train_prepared.csv\"\n# SIGN_TO_IDX_PATH = INPUT_DIR / \"asl-signs\" / \"sign_to_prediction_index_map.json\"\n\n# with open(SIGN_TO_IDX_PATH, \"r\") as f:\n#     SIGN_TO_IDX = json.load(f)\n\n# N_SPLITS = 5\n# SEED = 2023\n# ROWS_PER_FRAME = 543\n\n# IN_FEATURES = 5796","metadata":{"execution":{"iopub.status.busy":"2023-03-15T16:29:37.607309Z","iopub.execute_input":"2023-03-15T16:29:37.608342Z","iopub.status.idle":"2023-03-15T16:29:37.620469Z","shell.execute_reply.started":"2023-03-15T16:29:37.608296Z","shell.execute_reply":"2023-03-15T16:29:37.619209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df = pd.read_csv(TRAIN_CSV_PATH)\n# df","metadata":{"execution":{"iopub.status.busy":"2023-03-15T16:30:01.013253Z","iopub.execute_input":"2023-03-15T16:30:01.014317Z","iopub.status.idle":"2023-03-15T16:30:01.067834Z","shell.execute_reply.started":"2023-03-15T16:30:01.014270Z","shell.execute_reply":"2023-03-15T16:30:01.066312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}