{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install mediapipe --quiet","metadata":{"execution":{"iopub.status.busy":"2023-02-28T08:49:53.167319Z","iopub.execute_input":"2023-02-28T08:49:53.167720Z","iopub.status.idle":"2023-02-28T08:50:08.372076Z","shell.execute_reply.started":"2023-02-28T08:49:53.167684Z","shell.execute_reply":"2023-02-28T08:50:08.370772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nfrom pathlib import Path\nDATA_PATH = Path('/kaggle/input/asl-signs/')\n\ndf = pd.read_csv(DATA_PATH/'train.csv')","metadata":{"execution":{"iopub.status.busy":"2023-02-28T08:49:47.933456Z","iopub.execute_input":"2023-02-28T08:49:47.933822Z","iopub.status.idle":"2023-02-28T08:49:48.083068Z","shell.execute_reply.started":"2023-02-28T08:49:47.933788Z","shell.execute_reply":"2023-02-28T08:49:48.081736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2023-02-28T08:49:48.086244Z","iopub.execute_input":"2023-02-28T08:49:48.086714Z","iopub.status.idle":"2023-02-28T08:49:48.110098Z","shell.execute_reply.started":"2023-02-28T08:49:48.086676Z","shell.execute_reply":"2023-02-28T08:49:48.101451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df2 = pd.read_parquet(DATA_PATH/'tdrain_landmark_files/26734/1000035562.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-02-28T08:49:48.111378Z","iopub.execute_input":"2023-02-28T08:49:48.111875Z","iopub.status.idle":"2023-02-28T08:49:48.128719Z","shell.execute_reply.started":"2023-02-28T08:49:48.111826Z","shell.execute_reply":"2023-02-28T08:49:48.127758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df2","metadata":{"execution":{"iopub.status.busy":"2023-02-28T08:49:48.131994Z","iopub.execute_input":"2023-02-28T08:49:48.132992Z","iopub.status.idle":"2023-02-28T08:49:48.156919Z","shell.execute_reply.started":"2023-02-28T08:49:48.132940Z","shell.execute_reply":"2023-02-28T08:49:48.155591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df2 = pd.read_parquet(DATA_PATH/'train_landmark_files/28656/1000106739.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-02-28T08:49:48.158186Z","iopub.execute_input":"2023-02-28T08:49:48.158648Z","iopub.status.idle":"2023-02-28T08:49:48.179587Z","shell.execute_reply.started":"2023-02-28T08:49:48.158601Z","shell.execute_reply":"2023-02-28T08:49:48.178262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df2","metadata":{"execution":{"iopub.status.busy":"2023-02-28T08:49:48.181436Z","iopub.execute_input":"2023-02-28T08:49:48.181912Z","iopub.status.idle":"2023-02-28T08:49:48.220632Z","shell.execute_reply.started":"2023-02-28T08:49:48.181868Z","shell.execute_reply":"2023-02-28T08:49:48.219319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import mediapipe as mp\nimport matplotlib.pyplot as plt\nimport tempfile\n\nfrom PIL import Image as PIL_Image\nfrom IPython.display import Image as IPython_Image\nfrom pathlib import Path\n\n\ndef plot_frame(df, frame, point_type, connections=None, ax=None):\n    \"\"\"Plots a single frame of 3d landmark data of a specific type.\n    \n    Args:\n        df (pd.DataFrame): The dataframe containing landmark data.\n        frame (int): The frame number within the dataframe to plot.\n        point_type (str): The landmark type to plot.\n        connections: The mediapipe connections data for the point_type.\n        ax (matplotlib.axes.Axes): The axes to use to plot. Will create one with some useful defaults if not provided.\n    \n    Returns:\n        (matplotlib.axes.Axes): The resulting axes after plotting.\n    \"\"\"\n    connections = connections if connections is not None else []\n    frame_df = df[(df.frame == frame) & (df.type == point_type)]\n    if ax is None:\n        fig = plt.figure()\n        ax = fig.add_subplot(projection='3d')\n        ax.view_init(elev=270, azim=-90)\n    ax.scatter(frame_df.x, frame_df.y, frame_df.z)\n    points = frame_df[['landmark_index', 'x', 'y', 'z']].set_index('landmark_index').to_dict(orient='index')\n    for pt1, pt2 in connections:\n        try:\n            pt1 = points[pt1]\n            pt2 = points[pt2]\n            ax.plot([pt1['x'], pt2['x']], [pt1['y'], pt2['y']], [pt1['z'], pt2['z']])\n        except:\n            pass\n    return ax\n\ndef plot_face(df, frame, plot_connections=True, ax=None):\n    \"\"\"Plots a single frame of 3d landmark face data.\n    \n    Args:\n        df (pd.DataFrame): The dataframe containing landmark data.\n        frame (int): The frame number within the dataframe to plot.\n        plot_connections (bool): If true, plots connections between points.\n        ax (matplotlib.axes.Axes): The axes to use to plot. Will create one with some useful defaults if not provided.\n    \n    Returns:\n        (matplotlib.axes.Axes): The resulting axes after plotting.\n    \"\"\"\n    return plot_frame(df, frame, 'face', mp.solutions.face_mesh.FACEMESH_TESSELATION if plot_connections else None, ax=ax)\n    \ndef plot_left_hand(df, frame, plot_connections=True, ax=None):\n    \"\"\"Plots a single frame of 3d landmark left_hand data.\n    \n    Args:\n        df (pd.DataFrame): The dataframe containing landmark data.\n        frame (int): The frame number within the dataframe to plot.\n        plot_connections (bool): If true, plots connections between points.\n        ax (matplotlib.axes.Axes): The axes to use to plot. Will create one with some useful defaults if not provided.\n    \n    Returns:\n        (matplotlib.axes.Axes): The resulting axes after plotting.\n    \"\"\"\n    return plot_frame(df, frame, 'left_hand', mp.solutions.hands.HAND_CONNECTIONS if plot_connections else None, ax=ax)\n    \ndef plot_right_hand(df, frame, plot_connections=True, ax=None):\n    \"\"\"Plots a single frame of 3d landmark right_hand data.\n    \n    Args:\n        df (pd.DataFrame): The dataframe containing landmark data.\n        frame (int): The frame number within the dataframe to plot.\n        plot_connections (bool): If true, plots connections between points.\n        ax (matplotlib.axes.Axes): The axes to use to plot. Will create one with some useful defaults if not provided.\n    \n    Returns:\n        (matplotlib.axes.Axes): The resulting axes after plotting.\n    \"\"\"\n    return plot_frame(df, frame, 'right_hand', mp.solutions.hands.HAND_CONNECTIONS if plot_connections else None, ax=ax)\n    \ndef plot_pose(df, frame, plot_connections=True, ax=None):\n    \"\"\"Plots a single frame of 3d landmark pose data.\n    \n    Args:\n        df (pd.DataFrame): The dataframe containing landmark data.\n        frame (int): The frame number within the dataframe to plot.\n        plot_connections (bool): If true, plots connections between points.\n        ax (matplotlib.axes.Axes): The axes to use to plot. Will create one with some useful defaults if not provided.\n    \n    Returns:\n        (matplotlib.axes.Axes): The resulting axes after plotting.\n    \"\"\"\n    return plot_frame(df, frame, 'pose', mp.solutions.pose.POSE_CONNECTIONS if plot_connections else None, ax=ax)\n\ndef plot_multiple(df, frame, *funcs, plot_connections=True, ax=None, fig_kwargs=None):\n    \"\"\"Convenience function for running multiple functions on a single plot. \n    Warning: Hands seem to be on a different scale.\n    \n    Args:\n        df (pd.DataFrame): The dataframe containing landmark data.\n        frame (int): The frame number within the dataframe to plot.\n        funcs: The various functions to generate the single plot for.\n        plot_connections (bool): If true, plots connections between points.\n        ax (matplotlib.axes.Axes): The axes to use to plot. Will create one with some useful defaults if not provided.\n        fig_kwargs (dict): Any keyword arguments to provide to matplotlib.pyplot.figure.\n    \n    Returns:\n        (matplotlib.axes.Axes): The resulting axes after plotting.\n    \"\"\"\n    if ax is None:\n        fig_kwargs = fig_kwargs if fig_kwargs is not None else {}\n        fig = plt.figure(**fig_kwargs)\n        ax = fig.add_subplot(projection='3d')\n        ax.view_init(elev=270, azim=-90)\n    for func in funcs:\n        ax = func(df, frame, plot_connections=plot_connections, ax=ax)\n    return ax\n    \ndef plot_animation(func, df, plot_connections=True, fig_kwargs=None):\n    \"\"\"Displays an animation for a specific landmark type.\n    \n    Args:\n        func: The function to run for each frame (plot_pose, plot_face, plot_left_hand, plot_right_hand)\n        df (pd.DataFrame): The dataframe containing landmark data.\n        plot_connections (bool): If true, plots connections between points.\n        fig_kwargs (dict): Any keyword arguments to provide to matplotlib.pyplot.figure.\n    \n    Returns:\n        (IPython.display.Image): The class containing the gif data.\n    \"\"\"\n    fig_kwargs = fig_kwargs if fig_kwargs is not None else {}\n    with tempfile.TemporaryDirectory() as tempdir:\n        for ix, frame in enumerate(df.frame.sort_values().unique()):\n            fig = plt.figure(**fig_kwargs)\n            ax = fig.add_subplot(projection='3d')\n            ax.view_init(elev=270, azim=-90)\n            func(df, frame, plot_connections=plot_connections, ax=ax)\n            plt.savefig(Path(tempdir)/f'{ix}.png')\n            plt.close()\n        images = [PIL_Image.open(Path(tempdir)/f\"{n}.png\") for n in range(ix)]\n        images[0].save(Path(tempdir)/'output.gif', save_all=True, append_images=images[1:], duration=100, loop=0)\n        ret = IPython_Image(open(Path(tempdir)/'output.gif','rb').read())\n    return ret","metadata":{"execution":{"iopub.status.busy":"2023-02-28T09:29:09.918064Z","iopub.execute_input":"2023-02-28T09:29:09.919244Z","iopub.status.idle":"2023-02-28T09:29:09.938770Z","shell.execute_reply.started":"2023-02-28T09:29:09.919182Z","shell.execute_reply":"2023-02-28T09:29:09.937857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_animation(plot_pose, df2)","metadata":{"execution":{"iopub.status.busy":"2023-02-28T09:09:52.404411Z","iopub.execute_input":"2023-02-28T09:09:52.404812Z","iopub.status.idle":"2023-02-28T09:09:56.797375Z","shell.execute_reply.started":"2023-02-28T09:09:52.404777Z","shell.execute_reply":"2023-02-28T09:09:56.794813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_animation(plot_face, df2)","metadata":{"execution":{"iopub.status.busy":"2023-02-28T09:10:21.867469Z","iopub.execute_input":"2023-02-28T09:10:21.867947Z","iopub.status.idle":"2023-02-28T09:11:12.168865Z","shell.execute_reply.started":"2023-02-28T09:10:21.867904Z","shell.execute_reply":"2023-02-28T09:11:12.167778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_animation(plot_left_hand, df2)","metadata":{"execution":{"iopub.status.busy":"2023-02-28T09:11:17.558046Z","iopub.execute_input":"2023-02-28T09:11:17.558903Z","iopub.status.idle":"2023-02-28T09:11:20.586681Z","shell.execute_reply.started":"2023-02-28T09:11:17.558852Z","shell.execute_reply":"2023-02-28T09:11:20.585315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_animation(plot_right_hand, df2)","metadata":{"execution":{"iopub.status.busy":"2023-02-28T09:11:26.614929Z","iopub.execute_input":"2023-02-28T09:11:26.616019Z","iopub.status.idle":"2023-02-28T09:11:29.747471Z","shell.execute_reply.started":"2023-02-28T09:11:26.615957Z","shell.execute_reply":"2023-02-28T09:11:29.745998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df2[(df2.frame == 37) & (df2.type == 'right_hand')]","metadata":{"execution":{"iopub.status.busy":"2023-02-28T09:16:23.780117Z","iopub.execute_input":"2023-02-28T09:16:23.780713Z","iopub.status.idle":"2023-02-28T09:16:23.800795Z","shell.execute_reply.started":"2023-02-28T09:16:23.780665Z","shell.execute_reply":"2023-02-28T09:16:23.799334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = df.sample(n=1)\ndf3 = pd.read_parquet(DATA_PATH/sample.path.iloc[0])","metadata":{"execution":{"iopub.status.busy":"2023-02-28T09:18:41.181013Z","iopub.execute_input":"2023-02-28T09:18:41.181866Z","iopub.status.idle":"2023-02-28T09:18:41.275435Z","shell.execute_reply.started":"2023-02-28T09:18:41.181807Z","shell.execute_reply":"2023-02-28T09:18:41.274431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample","metadata":{"execution":{"iopub.status.busy":"2023-02-28T09:19:27.893753Z","iopub.execute_input":"2023-02-28T09:19:27.894139Z","iopub.status.idle":"2023-02-28T09:19:27.907114Z","shell.execute_reply.started":"2023-02-28T09:19:27.894105Z","shell.execute_reply":"2023-02-28T09:19:27.904966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_animation(plot_left_hand, df3)","metadata":{"execution":{"iopub.status.busy":"2023-02-28T09:19:10.949210Z","iopub.execute_input":"2023-02-28T09:19:10.949651Z","iopub.status.idle":"2023-02-28T09:19:15.062738Z","shell.execute_reply.started":"2023-02-28T09:19:10.949613Z","shell.execute_reply":"2023-02-28T09:19:15.061467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_animation(plot_pose, df3)","metadata":{"execution":{"iopub.status.busy":"2023-02-28T09:20:03.680049Z","iopub.execute_input":"2023-02-28T09:20:03.680542Z","iopub.status.idle":"2023-02-28T09:20:07.627407Z","shell.execute_reply.started":"2023-02-28T09:20:03.680497Z","shell.execute_reply":"2023-02-28T09:20:07.626294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_animation(plot_face, df3, plot_connections=False)","metadata":{"execution":{"iopub.status.busy":"2023-02-28T09:20:29.021735Z","iopub.execute_input":"2023-02-28T09:20:29.022600Z","iopub.status.idle":"2023-02-28T09:20:31.985715Z","shell.execute_reply.started":"2023-02-28T09:20:29.022558Z","shell.execute_reply":"2023-02-28T09:20:31.984599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_landmark_animation(df, landmark_type='pose', iloc=None, sequence_id=None, path_prefix='/kaggle/input/asl-signs/', plot_connections=True, fig_kwargs=None):\n    \"\"\"Reads in the data for a given train.csv iloc or sequence id and generates an animation for a given landmark type.\n    This function is mainly a wrapper for convenience.\n    \n    Args:\n        df (pd.DataFrame): The train.csv dataframe.\n        landmark_type (str): The landmark type to plot.\n        iloc (int): The train.csv iloc containing the row to plot the data for.\n        sequence_id (int): The train.csv sequence_id to plot the data for.\n        path_prefix (str): The prefix to prepend to the paths in train.csv.\n        plot_connections (bool): If true, plots connections between points.\n        fig_kwargs (dict): Any keyword arguments to provide to matplotlib.pyplot.figure.\n    \n    Returns:\n        (IPython.display.Image): The class containing the gif data.\n    \"\"\"\n    type_to_func = {\n        'pose': plot_pose,\n        'face': plot_face,\n        'left_hand': plot_left_hand,\n        'right_hand': plot_right_hand\n    }\n    if landmark_type not in type_to_func:\n        raise ValueError(\"Please provide landmark_type in (pose, face, left_hand, right_hand).\")\n    if iloc is None and sequence_id is None:\n        raise ValueError(\"Please provide either an iloc or sequence_id in the base df to generate a plot for.\")\n    if iloc is not None:\n        row = df.iloc[iloc]\n    else:\n        row = df[df.sequence_id == sequence_id].iloc[0]\n    df = pd.read_parquet(Path(path_prefix)/row.path)\n    func = type_to_func[landmark_type]\n    return plot_animation(func, df, plot_connections=plot_connections, fig_kwargs=fig_kwargs)","metadata":{"execution":{"iopub.status.busy":"2023-02-28T09:56:37.749930Z","iopub.execute_input":"2023-02-28T09:56:37.750389Z","iopub.status.idle":"2023-02-28T09:56:37.759484Z","shell.execute_reply.started":"2023-02-28T09:56:37.750348Z","shell.execute_reply":"2023-02-28T09:56:37.758241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_landmark_animation(df, iloc=0)","metadata":{"execution":{"iopub.status.busy":"2023-02-28T09:55:35.186894Z","iopub.execute_input":"2023-02-28T09:55:35.187792Z","iopub.status.idle":"2023-02-28T09:55:41.450992Z","shell.execute_reply.started":"2023-02-28T09:55:35.187745Z","shell.execute_reply":"2023-02-28T09:55:41.450114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_landmark_animation(df, landmark_type='face', plot_connections=False, sequence_id=1000210073)","metadata":{"execution":{"iopub.status.busy":"2023-02-28T09:56:39.995960Z","iopub.execute_input":"2023-02-28T09:56:39.996372Z","iopub.status.idle":"2023-02-28T09:56:43.111973Z","shell.execute_reply.started":"2023-02-28T09:56:39.996335Z","shell.execute_reply":"2023-02-28T09:56:43.110519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}