{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install ffmpeg-python nb-black","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-03-17T03:09:20.379420Z","iopub.execute_input":"2023-03-17T03:09:20.379968Z","iopub.status.idle":"2023-03-17T03:09:34.376871Z","shell.execute_reply.started":"2023-03-17T03:09:20.379918Z","shell.execute_reply":"2023-03-17T03:09:34.375601Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Abstract\n\nTo get more intuitive picture of each hand signs, I convined landmark data of pose, face and both hands.\n\n# Reference\n\n- [previous notebook](https://www.kaggle.com/code/tatamikenn/islr-eda-let-s-get-animated) - you can see another animated pictures\n- [ISLR: Intuitive Animations of 250 signs](https://www.kaggle.com/datasets/tatamikenn/islr-animation-250-signs-intuitive) - you can fully access to all the sampled 250 signs","metadata":{}},{"cell_type":"code","source":"from pathlib import Path\nfrom types import SimpleNamespace\n\nimport cv2\nimport numpy as np\nimport polars as pl\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nimport ffmpeg\nfrom IPython.core.display import Video\n\nplt.style.use(\"default\")\n\ncfg = SimpleNamespace()\ncfg.INPUT = Path(\"../input/asl-signs\")\ncfg.OUTPUT = Path(\"./animation\")\ncfg.DEBUG = True\n\nif not cfg.OUTPUT.exists():\n    cfg.OUTPUT.mkdir()\n\n\n%load_ext lab_black\n%load_ext autoreload\n%autoreload 2","metadata":{"execution":{"iopub.status.busy":"2023-03-17T03:09:34.380562Z","iopub.execute_input":"2023-03-17T03:09:34.380962Z","iopub.status.idle":"2023-03-17T03:09:35.076338Z","shell.execute_reply.started":"2023-03-17T03:09:34.380931Z","shell.execute_reply":"2023-03-17T03:09:35.075177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pl.read_csv(cfg.INPUT / \"train.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-03-17T03:09:35.077558Z","iopub.execute_input":"2023-03-17T03:09:35.077875Z","iopub.status.idle":"2023-03-17T03:09:35.227658Z","shell.execute_reply.started":"2023-03-17T03:09:35.077843Z","shell.execute_reply":"2023-03-17T03:09:35.226596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pose_edges = [\n    {\"name\": \"other\", \"path\": [(9, 10), (11, 12), (23, 24)], \"color\": (255, 255, 255)},\n    {\n        \"name\": \"right_eye\",\n        \"path\": [(0, 4), (4, 5), (5, 6), (6, 8)],\n        \"color\": (255, 0, 255),\n    },\n    {\n        \"name\": \"left_eye\",\n        \"path\": [(0, 1), (1, 2), (2, 3), (3, 7)],\n        \"color\": (0, 0, 255),\n    },\n    {\"name\": \"right_hand\", \"path\": [(14, 16), (12, 14)], \"color\": (255, 40, 255)},\n    {\n        \"name\": \"left_hand\",\n        \"path\": [(11, 13), (13, 15)],\n        \"color\": (40, 40, 255),\n    },\n    {\"name\": \"right_torso\", \"path\": [(12, 24)], \"color\": (255, 80, 255)},\n    {\"name\": \"left_torso\", \"path\": [(11, 23)], \"color\": (80, 80, 255)},\n]\nhand_edges = [\n    {\n        \"name\": \"thumb\",\n        \"path\": [(0, 1), (1, 2), (2, 3), (3, 4)],\n        \"color\": (255, 255, 255),\n    },\n    {\"name\": \"index_finter\", \"path\": [(5, 6), (6, 7), (7, 8)], \"color\": (120, 40, 150)},\n    {\n        \"name\": \"middle_finger\",\n        \"path\": [(9, 10), (10, 11), (11, 12)],\n        \"color\": (0, 255, 255),\n    },\n    {\n        \"name\": \"ring_finger\",\n        \"path\": [(13, 14), (14, 15), (15, 16)],\n        \"color\": (0, 255, 0),\n    },\n    {\n        \"name\": \"little_finger\",\n        \"path\": [(17, 18), (18, 19), (19, 20)],\n        \"color\": (255, 0, 0),\n    },\n    {\n        \"name\": \"palm\",\n        \"path\": [(0, 5), (5, 9), (9, 13), (13, 17), (17, 0)],\n        \"color\": (140, 140, 140),\n    },\n    {\"name\": \"\", \"path\": [], \"color\": ()},\n]","metadata":{"execution":{"iopub.status.busy":"2023-03-17T03:09:35.448216Z","iopub.execute_input":"2023-03-17T03:09:35.448572Z","iopub.status.idle":"2023-03-17T03:09:35.506721Z","shell.execute_reply.started":"2023-03-17T03:09:35.448541Z","shell.execute_reply":"2023-03-17T03:09:35.504738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def draw_frame(frame, lms_all):\n    image = np.zeros((512, 512, 3), np.uint8)\n\n    # process pose\n    right_wrist = (None, None)\n    left_wrist = (None, None)\n    lm = lms_all[frame].filter(\n        (pl.col(\"type\") == \"pose\") & (pl.col(\"landmark_index\") < 25)\n    )\n    beta = 0.8\n    lm = lm.with_columns(\n        [\n            (pl.col(\"x\") - pl.col(\"x\").first() + 0.5 / beta).alias(\"x\"),\n            (pl.col(\"y\") - pl.col(\"y\").first() + 0.15 / beta).alias(\"y\"),\n        ]\n    )\n    right_wrist = (lm[\"x\"][16] * beta, lm[\"y\"][16] * beta)\n    left_wrist = (lm[\"x\"][15] * beta, lm[\"y\"][15] * beta)\n    upper_lip = (\n        (lm[\"x\"][9] + lm[\"x\"][10]) / 2 * beta,\n        (lm[\"y\"][9] + lm[\"y\"][9]) / 2 * beta,\n    )\n    for edge in pose_edges:\n        name, paths, color = edge.values()\n        for i, j in paths:\n            x1, x2, y1, y2 = (\n                lm[\"x\"][i] * beta,\n                lm[\"x\"][j] * beta,\n                lm[\"y\"][i] * beta,\n                lm[\"y\"][j] * beta,\n            )\n            if not ((x1 is None) | (x2 is None) | (y1 is None) | (y2 is None)):\n                x1 = int(x1 * 512)\n                x2 = int(x2 * 512)\n                y1 = int(y1 * 512)\n                y2 = int(y2 * 512)\n                image = cv2.line(image, (x1, y1), (x2, y2), color, thickness=2)\n\n    # process face\n    lm = lms_all[frame].filter(pl.col(\"type\") == \"face\")\n    lm = lm.with_columns(\n        [\n            (pl.col(\"x\") - pl.col(\"x\").first()).alias(\"x\"),\n            (pl.col(\"y\") - pl.col(\"y\").first()).alias(\"y\"),\n        ]\n    )\n    x0, y0 = upper_lip\n    if (\n        (lm[\"x\"][0] is not None)\n        & (lm[\"y\"][0] is not None)\n        & (x0 is not None)\n        & (y0 is not None)\n    ):\n        gamma = 0.7\n        point_color = (255, 255, 255)\n        radius = 1\n        for x, y in lm.select([\"x\", \"y\"]).to_numpy():\n            x, y = x * gamma + x0, y * gamma + y0\n            x = int(x * 512)\n            y = int(y * 512)\n            cv2.circle(image, (x, y), radius, point_color, -1)\n\n    # process hands\n    lms = (\n        lms_all[frame]\n        .filter((pl.col(\"type\") == \"right_hand\") | (pl.col(\"type\") == \"left_hand\"))\n        .partition_by(\"type\")\n    )\n\n    for lm in lms:\n        lm = lm.with_columns(\n            [\n                (pl.col(\"x\") - pl.col(\"x\").first()).alias(\"x\"),\n                (pl.col(\"y\") - pl.col(\"y\").first()).alias(\"y\"),\n            ]\n        )\n        lm_type = lm.row(0)[2]\n        x0, y0 = right_wrist if (lm_type == \"right_hand\") else left_wrist\n        if (lm[\"x\"][0] is None) | (lm[\"y\"][0] is None) | (x0 is None) | (y0 is None):\n            continue\n        alpha = 0.6\n        for edge in hand_edges:\n            name, paths, color = edge.values()\n            for i, j in paths:\n                x1, x2, y1, y2 = (\n                    (lm[\"x\"][i] * alpha + x0),\n                    (lm[\"x\"][j] * alpha + x0),\n                    (lm[\"y\"][i] * alpha + y0),\n                    (lm[\"y\"][j] * alpha + y0),\n                )\n                if not ((x1 is None) | (x2 is None) | (y1 is None) | (y2 is None)):\n                    x1 = int(x1 * 512)\n                    x2 = int(x2 * 512)\n                    y1 = int(y1 * 512)\n                    y2 = int(y2 * 512)\n                    image = cv2.line(image, (x1, y1), (x2, y2), color, thickness=2)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    return image","metadata":{"execution":{"iopub.status.busy":"2023-03-17T03:09:35.508481Z","iopub.execute_input":"2023-03-17T03:09:35.509029Z","iopub.status.idle":"2023-03-17T03:09:35.596270Z","shell.execute_reply.started":"2023-03-17T03:09:35.508980Z","shell.execute_reply":"2023-03-17T03:09:35.594221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Drawing Sample Sequences\n\nlandmark data of face and both hands are joined to pose at these key points:\n\n- face: upper lip\n- both hands: wrist\n\nScales are ajusted manually as follows:\n\n- face: 0.7\n- both hands: 0.6\n- pose: 0.8","metadata":{}},{"cell_type":"code","source":"OUTPUT = Path(\"lm\")\nif not OUTPUT.exists():\n    OUTPUT.mkdir()\n\nlm_data = {\n    k: v\n    for k, v in zip(\n        train.columns,\n        train.filter(\n            (pl.col(\"participant_id\") == 2044) & (pl.col(\"sequence_id\") == 3356604026)\n        ).row(0),\n    )\n}\ndf_landmark = pl.read_parquet(cfg.INPUT / lm_data[\"path\"])\n\ndf_frames = df_landmark.partition_by(\"frame\")\nfor idx, frame in enumerate(range(len(df_frames))):\n    image = draw_frame(frame, df_frames)\n\n    cv2.imwrite((OUTPUT / f\"lm{idx:03d}.png\").as_posix(), image)\n    plt.imshow(image)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T03:09:35.598265Z","iopub.execute_input":"2023-03-17T03:09:35.598677Z","iopub.status.idle":"2023-03-17T03:09:37.304919Z","shell.execute_reply.started":"2023-03-17T03:09:35.598625Z","shell.execute_reply":"2023-03-17T03:09:37.302741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Making Animation","metadata":{}},{"cell_type":"code","source":"def make_animation(row, columns, fps: int = 10):\n    data = {k: v for k, v in zip(columns, row)}\n    sign, participant_id, sequence_id = (\n        data[\"sign\"],\n        data[\"participant_id\"],\n        data[\"sequence_id\"],\n    )\n\n    df_landmark = pl.read_parquet(cfg.INPUT / data[\"path\"])\n    df_landmark = df_landmark.sort([\"frame\", \"type\", \"landmark_index\"])\n    lms_all = df_landmark.partition_by(\"frame\")\n\n    height, width = 512, 512\n    if not (cfg.OUTPUT / sign).exists():\n        (cfg.OUTPUT / sign).mkdir()\n\n    output_filename = (\n        cfg.OUTPUT / sign / f\"{participant_id}_{sequence_id}.mp4\"\n    ).as_posix()\n    process = (\n        ffmpeg.input(\"pipe:\", format=\"rawvideo\", pix_fmt=\"rgb24\", s=f\"{width}x{height}\")\n        .output(\n            output_filename,\n            pix_fmt=\"yuv420p\",\n            vcodec=\"libx264\",\n            r=fps,\n            loglevel=\"quiet\",\n        )\n        .overwrite_output()\n        .run_async(pipe_stdin=True)\n    )\n\n    for frame in range(len(lms_all)):\n        image = draw_frame(frame, lms_all)\n        process.stdin.write(image.tobytes())\n\n    process.stdin.close()\n    process.wait()","metadata":{"execution":{"iopub.status.busy":"2023-03-17T03:09:37.309903Z","iopub.execute_input":"2023-03-17T03:09:37.310281Z","iopub.status.idle":"2023-03-17T03:09:37.356192Z","shell.execute_reply.started":"2023-03-17T03:09:37.310246Z","shell.execute_reply":"2023-03-17T03:09:37.354481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_unique_signs = train.filter(\n    (pl.arange(0, pl.count())).shuffle(seed=42).over(\"sign\") < 5\n)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T03:09:37.358413Z","iopub.execute_input":"2023-03-17T03:09:37.358858Z","iopub.status.idle":"2023-03-17T03:09:37.410591Z","shell.execute_reply.started":"2023-03-17T03:09:37.358821Z","shell.execute_reply":"2023-03-17T03:09:37.408968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not cfg.OUTPUT.exists():\n    cfg.OUTPUT.mkdir()\n\nif cfg.DEBUG:\n    df = train_unique_signs.head(5)\nelse:\n    df = train_unique_signs\n\nfor row in tqdm(df.iter_rows(), total=len(df)):\n    make_animation(row, df.columns)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T03:10:22.491275Z","iopub.execute_input":"2023-03-17T03:10:22.491846Z","iopub.status.idle":"2023-03-17T03:10:24.574825Z","shell.execute_reply.started":"2023-03-17T03:10:22.491796Z","shell.execute_reply":"2023-03-17T03:10:24.573234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!tree animation","metadata":{"execution":{"iopub.status.busy":"2023-03-17T03:10:26.646931Z","iopub.execute_input":"2023-03-17T03:10:26.648606Z","iopub.status.idle":"2023-03-17T03:10:26.955294Z","shell.execute_reply.started":"2023-03-17T03:10:26.648500Z","shell.execute_reply":"2023-03-17T03:10:26.953805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Video(\"animation/TV/28656_125816896.mp4\")","metadata":{"execution":{"iopub.status.busy":"2023-03-17T03:09:40.240377Z","iopub.execute_input":"2023-03-17T03:09:40.240701Z","iopub.status.idle":"2023-03-17T03:09:40.277816Z","shell.execute_reply.started":"2023-03-17T03:09:40.240666Z","shell.execute_reply":"2023-03-17T03:09:40.276137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Video(\"animation/all/26734_1247514751.mp4\")","metadata":{"execution":{"iopub.status.busy":"2023-03-17T03:10:58.786118Z","iopub.execute_input":"2023-03-17T03:10:58.786496Z","iopub.status.idle":"2023-03-17T03:10:58.831431Z","shell.execute_reply.started":"2023-03-17T03:10:58.786458Z","shell.execute_reply":"2023-03-17T03:10:58.829129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Video(\"animation/bug/61333_1268993802.mp4\")","metadata":{"execution":{"iopub.status.busy":"2023-03-17T03:09:40.280394Z","iopub.execute_input":"2023-03-17T03:09:40.280937Z","iopub.status.idle":"2023-03-17T03:09:40.313146Z","shell.execute_reply.started":"2023-03-17T03:09:40.280887Z","shell.execute_reply":"2023-03-17T03:09:40.311653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Video(\"animation/lion/29302_1271911796.mp4\")","metadata":{"execution":{"iopub.status.busy":"2023-03-17T03:09:40.315055Z","iopub.execute_input":"2023-03-17T03:09:40.315414Z","iopub.status.idle":"2023-03-17T03:09:40.349748Z","shell.execute_reply.started":"2023-03-17T03:09:40.315386Z","shell.execute_reply":"2023-03-17T03:09:40.348207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Future works\n\n* scales and center position should be normalized\n* iterporate missing frames","metadata":{}}]}