{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Display the frames of a sequence.\n\nThe goal is to undertand better the data.\n\n\nWe need to predict a sign from a sequence based on Mediapipe data.","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport pandas as pd","metadata":{"execution":{"iopub.status.busy":"2023-03-08T09:21:22.507366Z","iopub.execute_input":"2023-03-08T09:21:22.507820Z","iopub.status.idle":"2023-03-08T09:21:22.513371Z","shell.execute_reply.started":"2023-03-08T09:21:22.507783Z","shell.execute_reply":"2023-03-08T09:21:22.512068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load the data","metadata":{}},{"cell_type":"code","source":"dir = '/kaggle/input/asl-signs'\n\ntrain_df = pd.read_csv(f'{dir}/train.csv')\n\n# sign = pd.read_json(f'{dir}/sign_to_prediction_index_map.json', orient = 'columns', typ='series')\n# sign_df = pd.DataFrame(sign).reset_index()\n# sign_df = sign_df.rename(columns={\"index\":\"sign\", 0:'id'})","metadata":{"execution":{"iopub.status.busy":"2023-03-08T09:22:19.498859Z","iopub.execute_input":"2023-03-08T09:22:19.499320Z","iopub.status.idle":"2023-03-08T09:22:19.775238Z","shell.execute_reply.started":"2023-03-08T09:22:19.499282Z","shell.execute_reply":"2023-03-08T09:22:19.773952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Functions to print ","metadata":{}},{"cell_type":"markdown","source":"Based on the work of https://www.kaggle.com/code/mayukh18/sign-language-eda-visualization","metadata":{}},{"cell_type":"code","source":"def plot_frame(df, frame_id, ax, count, printlabel):\n    \"\"\"\n    Display for one frame the parts: face, pose, left_hand, right_hand and all.\n    \n    \"\"\"\n\n    # Plot on at the time.\n    plot_part(df, \"face\", frame_id, ax[count][1], printlabel)\n    plot_part(df, \"pose\", frame_id, ax[count][2], printlabel)\n    plot_part(df, \"left_hand\", frame_id, ax[count][3], printlabel)\n    plot_part(df, \"right_hand\", frame_id, ax[count][4], printlabel)\n    \n    # Merge all the data.\n    plot_part(df, \"face\", frame_id, ax[count][0], printlabel)\n    plot_part(df, \"pose\", frame_id, ax[count][0], printlabel)\n    plot_part(df, \"left_hand\", frame_id, ax[count][0], printlabel)\n    plot_part(df, \"right_hand\", frame_id, ax[count][0], printlabel)\n\n    \n\ndef plot_part(df, part, frame_id, ax, printlabel=False):\n    \"\"\"\n    Display a part, edges are displayed.\n    \"\"\"\n\n    df = df[(df.type == part) & (df.frame == frame_id)].sort_values(['landmark_index'])\n\n    x = list(df.x)\n    # Y values are inverted.\n    y = list(-df.y)\n    z = list(df.z)\n    landmark_index = list(df.landmark_index)\n    \n    ax.scatter(x, y, color='dodgerblue', s=1)\n    ax.set_xlabel(part)\n    ax.set_ylabel(\"\")\n    ax.set_xticks([])\n    ax.set_yticks([])\n    ax.set_xticklabels([])\n    ax.set_yticklabels([])\n\n    if printlabel:\n        for i in range(len(x)):\n            ax.text(x[i], y[i], landmark_index[i])\n\n    \n    if part == \"face\":\n        edges = []\n        ax.set_ylabel(f\"Frame no. {frame_id}\")\n        # Color of the edges.\n        color='red'\n        \n    elif part == \"pose\":\n        edges = [#(7,3), \n        (3,2), (2,1),\n        #(1,0), (0,4),\n        (4,5), (5,6), \n        #(6,8), \n        (9,10), #mouth\n        (17,19), (19,15), (17,15), (15,21), (15,13), (13,11), #right_arm\n        (20,18), (18,16), (20,16), (16,22), (14,16), (14,12), #left_arm\n        (11,12), (11,23), (23, 24), (24,12), (12,11),\n        (31,29), (29,27), (31,27), (27,25), (25,23),\n        (30, 32), (30,28), (32,28), (28,26), (26,24)\n        ]\n        # Color of the edges.\n        color='salmon'\n    else:\n        edges = [(0,1),(1,2),(2,3),(3,4),(0,5),(0,17),(5,6),(6,7),(7,8),(5,9),(9,10),(10,11),(11,12),\n         (9,13),(13,14),(14,15),(15,16),(13,17),(17,18),(18,19),(19,20)]\n        color='green'\n        \n    for edge in edges:\n       ax.plot([x[edge[0]], x[edge[1]]], [y[edge[0]], y[edge[1]]], color=color)\n\n    \n    \ndef plot_sequence(df, sequence, printlabel=False):\n    \"\"\"\n    For a sequence display all the part on all the frame.\n    \n    \"\"\"\n\n    sequence_path = df.iloc[sequence, 0]\n    parq = pd.read_parquet(f'{dir}/{sequence_path}')\n    label = df.iloc[sequence, 3]\n\n    list_frames = list(parq[\"frame\"].unique())\n    n_frames = len(list_frames)\n\n    fig, ax = plt.subplots(n_frames, 5, figsize=(7,n_frames*4))\n    print(label)\n\n    \n    for count, frame_id in enumerate(list_frames):\n        plot_frame(parq, frame_id, ax, count, printlabel)\n        \n    plt.show()\n\n","metadata":{"execution":{"iopub.status.busy":"2023-03-08T10:14:16.085058Z","iopub.execute_input":"2023-03-08T10:14:16.086446Z","iopub.status.idle":"2023-03-08T10:14:16.109585Z","shell.execute_reply.started":"2023-03-08T10:14:16.086384Z","shell.execute_reply":"2023-03-08T10:14:16.108259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Display sequence","metadata":{}},{"cell_type":"code","source":"# Change the sequence number to display another sequence.\nsequence = 0\nplot_sequence(train_df, sequence)","metadata":{"execution":{"iopub.status.busy":"2023-03-08T10:14:19.672571Z","iopub.execute_input":"2023-03-08T10:14:19.673011Z","iopub.status.idle":"2023-03-08T10:14:31.791305Z","shell.execute_reply.started":"2023-03-08T10:14:19.672956Z","shell.execute_reply":"2023-03-08T10:14:31.790088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ","metadata":{}},{"cell_type":"markdown","source":"Create some 3D plots, some static and some with animation thanks to plotly.","metadata":{}},{"cell_type":"markdown","source":"# 3d plot Static","metadata":{}},{"cell_type":"code","source":"import plotly.graph_objs as go\nimport plotly.express as px","metadata":{"execution":{"iopub.status.busy":"2023-03-08T09:22:33.932990Z","iopub.execute_input":"2023-03-08T09:22:33.933609Z","iopub.status.idle":"2023-03-08T09:22:36.321888Z","shell.execute_reply.started":"2023-03-08T09:22:33.933569Z","shell.execute_reply":"2023-03-08T09:22:36.320504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sequence = 1\nsequence_path = train_df.iloc[sequence, 0]\nparq = pd.read_parquet(f'{dir}/{sequence_path}')\nlist_frames = list(parq[\"frame\"].unique())\n\n# Use the first frame.\nparq = parq[parq[\"frame\"]==list_frames[0]]\n\npart = \"pose\"\ndf = parq[parq.type == part].sort_values(['landmark_index'])\ndf[\"y\"] = df[\"y\"]\n\n\n\nif part == \"face\":\n    edges = []\n    ax.set_ylabel(f\"Frame no. {frame_id}\")\n    # Color of the edges.\n    color='red'\n        \nelif part == \"pose\":\n    edges = [\n     (3,2), (2,1),\n     (4,5), (5,6), \n    (9,10), #mouth\n    (17,19), (19,15), (17,15), (15,21), (15,13), (13,11), #right_arm\n    (20,18), (18,16), (20,16), (16,22), (14,16), (14,12), #left_arm\n    (11,12), (11,23), (23, 24), (24,12), (12,11),\n    (31,29), (29,27), (31,27), (27,25), (25,23),\n    (30, 32), (30,28), (32,28), (28,26), (26,24)\n    ]\n    # Color of the edges.\n    color='salmon'\nelse:\n    edges = [(0,1),(1,2),(2,3),(3,4),(0,5),(0,17),(5,6),(6,7),(7,8),(5,9),(9,10),(10,11),(11,12),\n     (9,13),(13,14),(14,15),(15,16),(13,17),(17,18),(18,19),(19,20)]\n    color='pink'\n\nx = list(df.x)\ny = list(df.y)\nz = list(df.z)\n\nXe=[]\nYe=[]\nZe=[]\n\nfor edge in edges:\n    Xe+=[x[edge[0]],x[edge[1]], None]\n    Ye+=[y[edge[0]],y[edge[1]], None]\n    Ze+=[z[edge[0]],z[edge[1]], None]\n\n\n    \n#import plotly.plotly as py\nimport plotly.graph_objs as go\n\ntrace1=go.Scatter3d(x=Xe,\n               y=Ye,\n               z=Ze,\n               mode='lines',\n               line=dict(color='rgb(125,125,125)', width=1),\n               hoverinfo='none'\n               )\n\ntrace2=go.Scatter3d(x=df[\"x\"],\n               y=df[\"y\"],\n               z=df[\"z\"],\n               mode='markers',\n               marker=dict(symbol='circle',\n                             size=3,\n                             #color=group,\n                             colorscale='Viridis',\n                             line=dict(color='rgb(50,50,50)', width=0.5)\n                             ),\n                \n               text=df[\"row_id\"],\n               )\n\n\naxis=dict(showbackground=False,\n          showline=False,\n          zeroline=False,\n          showgrid=False,\n          showticklabels=False,\n          title=''\n          )\n\nlayout = go.Layout(\n         title=\"3D visualization\",\n         width=800,\n         height=800,\n         showlegend=False,\n         scene=dict(\n             aspectratio=dict(x=1, y=1, z=1),\n                xaxis = dict(visible=False),\n                yaxis = dict(visible=False),\n                zaxis =dict(visible=False)\n         ),\n     margin=dict(\n        t=100\n    ),\n#     hovermode='closest',\n)\n\n\n\ndata=[trace1, trace2]\nfig=go.Figure(data=data, layout=layout)\n\ncamera_params = dict(\n    up=dict(x=0,y=0,z=0),\n    center=dict(x=0,y=0,z=0),\n    eye=dict(x=0,y=0,z=-2)\n)\nfig.update_layout(scene_camera=camera_params)\n\nfig.update_scenes(aspectmode='data')\n\nfig","metadata":{"execution":{"iopub.status.busy":"2023-03-08T10:10:47.717919Z","iopub.execute_input":"2023-03-08T10:10:47.718391Z","iopub.status.idle":"2023-03-08T10:10:47.792808Z","shell.execute_reply.started":"2023-03-08T10:10:47.718352Z","shell.execute_reply":"2023-03-08T10:10:47.791614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3d plot with Animation","metadata":{}},{"cell_type":"code","source":"sequence = 10\nsequence_path = train_df.iloc[sequence, 0]\nparq = pd.read_parquet(f'{dir}/{sequence_path}')\n\npart = \"pose\"\ndf = parq[parq.type == part].sort_values(['landmark_index'])\ndf[\"y\"] = df[\"y\"]\n\nfig = px.scatter_3d(df, x=\"x\", y=\"y\", z=\"z\", animation_frame=\"frame\", hover_name=\"row_id\", width=800, height=800)\nfig.update_traces(marker_size=3)\nfig.update_layout(\n    scene = dict(\n        xaxis = dict(visible=False),\n        yaxis = dict(visible=False),\n        zaxis =dict(visible=False)\n        )\n    )\ncamera_params = dict(\n    up=dict(x=0,y=0,z=0),\n    center=dict(x=0,y=0,z=0),\n    eye=dict(x=0,y=0,z=-2)\n)\n\nfig.update_layout(scene_camera=camera_params)","metadata":{"execution":{"iopub.status.busy":"2023-03-07T21:41:37.339889Z","iopub.execute_input":"2023-03-07T21:41:37.340559Z","iopub.status.idle":"2023-03-07T21:41:39.423572Z","shell.execute_reply.started":"2023-03-07T21:41:37.340515Z","shell.execute_reply":"2023-03-07T21:41:39.422236Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nsequence = 10\nsequence_path = train_df.iloc[sequence, 0]\nparq = pd.read_parquet(f'{dir}/{sequence_path}')\n\npart = \"face\"\ndf = parq[parq.type == part].sort_values(['landmark_index'])\ndf[\"y\"] = df[\"y\"]\n\nfig = px.scatter_3d(df, x=\"x\", y=\"y\", z=\"z\", animation_frame=\"frame\", hover_name=\"row_id\", width=800, height=800)\nfig.update_traces(marker_size=3)\nfig.update_layout(\n    scene = dict(\n        xaxis = dict(visible=False),\n        yaxis = dict(visible=False),\n        zaxis =dict(visible=False)\n        )\n    )\ncamera_params = dict(\n    up=dict(x=0,y=0,z=0),\n    center=dict(x=0,y=0,z=0),\n    eye=dict(x=0,y=0,z=-2)\n)\nfig.update_layout(scene_camera=camera_params)","metadata":{"execution":{"iopub.status.busy":"2023-03-07T21:41:39.425525Z","iopub.execute_input":"2023-03-07T21:41:39.426334Z","iopub.status.idle":"2023-03-07T21:41:39.769450Z","shell.execute_reply.started":"2023-03-07T21:41:39.426292Z","shell.execute_reply":"2023-03-07T21:41:39.768212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nsequence = 10\nsequence_path = train_df.iloc[sequence, 0]\nparq = pd.read_parquet(f'{dir}/{sequence_path}')\n\npart = \"right_hand\"\ndf = parq[parq.type == part].sort_values(['landmark_index'])\ndf[\"y\"] = df[\"y\"]\n\nfig = px.scatter_3d(df, x=\"x\", y=\"y\", z=\"z\", animation_frame=\"frame\", hover_name=\"row_id\", width=800, height=800)\nfig.update_traces(marker_size=3)\nfig.update_layout(\n    scene = dict(\n        xaxis = dict(visible=False),\n        yaxis = dict(visible=False),\n        zaxis =dict(visible=False)\n        )\n    )\ncamera_params = dict(\n    up=dict(x=0,y=0,z=0),\n    center=dict(x=0,y=0,z=0),\n    eye=dict(x=0,y=0,z=-2)\n)\nfig.update_layout(scene_camera=camera_params)","metadata":{"execution":{"iopub.status.busy":"2023-03-05T16:47:08.600939Z","iopub.execute_input":"2023-03-05T16:47:08.601472Z","iopub.status.idle":"2023-03-05T16:47:09.421220Z","shell.execute_reply.started":"2023-03-05T16:47:08.601425Z","shell.execute_reply":"2023-03-05T16:47:09.419756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}