{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Abstract\n\nIn this notebook, I tried basic EDA.\nIn the latter part, I made animated GIF of sampled landmarks.\n\n## Change History\n\n* version1: plot whole body\n* version2: plot only upper body (since leg part sometimes contains noise)\n* version3: output in mp4 format\n* version4: interpolate missing frames","metadata":{}},{"cell_type":"code","source":"!pip install nb-black","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-04-07T03:15:25.040048Z","iopub.execute_input":"2023-04-07T03:15:25.040630Z","iopub.status.idle":"2023-04-07T03:15:38.382785Z","shell.execute_reply.started":"2023-04-07T03:15:25.040580Z","shell.execute_reply":"2023-04-07T03:15:38.381390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from typing import Dict\nfrom pathlib import Path\nfrom types import SimpleNamespace\nfrom multiprocessing import Pool\nfrom functools import partial\n\nimport polars as pl\nimport matplotlib.pyplot as plt\nimport matplotlib.animation as animation\nfrom IPython.core.display import Video\nfrom tqdm import tqdm\n\nplt.style.use(\"ggplot\")\n\ncfg = SimpleNamespace()\ncfg.INPUT = Path(\"/kaggle/input/asl-signs\")\ncfg.OUTPUT = Path(\"/kaggle/working/animation\")\ncfg.DEBUG = True\n\n%load_ext lab_black\n%load_ext autoreload\n%autoreload 2","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2023-04-07T03:15:38.384940Z","iopub.execute_input":"2023-04-07T03:15:38.385329Z","iopub.status.idle":"2023-04-07T03:15:38.691446Z","shell.execute_reply.started":"2023-04-07T03:15:38.385292Z","shell.execute_reply":"2023-04-07T03:15:38.689717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pl.read_csv(cfg.INPUT / \"train.csv\")","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2023-04-07T03:15:38.692883Z","iopub.execute_input":"2023-04-07T03:15:38.694345Z","iopub.status.idle":"2023-04-07T03:15:38.813648Z","shell.execute_reply.started":"2023-04-07T03:15:38.694293Z","shell.execute_reply":"2023-04-07T03:15:38.812816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"#unique participants: {len(train['participant_id'].unique())}\")\nprint(f\"#unique sequence: {len(train['sequence_id'].unique()):,}\")\nprint(f\"#unique signs: {len(train['sign'].unique()):,}\")","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2023-04-07T03:15:38.815714Z","iopub.execute_input":"2023-04-07T03:15:38.816153Z","iopub.status.idle":"2023-04-07T03:15:38.881707Z","shell.execute_reply.started":"2023-04-07T03:15:38.816124Z","shell.execute_reply":"2023-04-07T03:15:38.880929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sequence_per_participant = (\n    train.groupby(\"participant_id\")\n    .agg(pl.col(\"sequence_id\").unique().count())\n    .with_columns(pl.col(\"participant_id\").cast(str))\n)\n\n_, ax = plt.subplots()\nax.barh(\n    y=sequence_per_participant[\"participant_id\"],\n    width=sequence_per_participant[\"sequence_id\"],\n)\nax.set(xlabel=\"#sequences\", ylabel=\"participant_id\", title=\"#sequences per participant\")\nplt.show()","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2023-04-07T03:15:38.882851Z","iopub.execute_input":"2023-04-07T03:15:38.883328Z","iopub.status.idle":"2023-04-07T03:15:39.324294Z","shell.execute_reply.started":"2023-04-07T03:15:38.883294Z","shell.execute_reply":"2023-04-07T03:15:39.322601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sequence_per_sign = (\n    train.groupby(\"sign\").agg(pl.col(\"sequence_id\").count()).sort(\"sequence_id\")\n)\n_, ax = plt.subplots()\nax.hist(sequence_per_sign[\"sequence_id\"], bins=20)\nax.set(xlabel=\"#sequences/sign\", ylabel=\"count\", title=\"#sequences per sign\")\nplt.show()","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2023-04-07T03:15:39.325731Z","iopub.execute_input":"2023-04-07T03:15:39.326129Z","iopub.status.idle":"2023-04-07T03:15:39.559556Z","shell.execute_reply.started":"2023-04-07T03:15:39.326094Z","shell.execute_reply":"2023-04-07T03:15:39.558349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Skeletical data","metadata":{}},{"cell_type":"code","source":"edges = {\n    \"left_hand\": [\n        (0, 1),\n        (1, 2),\n        (2, 3),\n        (3, 4),\n        (0, 5),\n        (0, 17),\n        (5, 6),\n        (6, 7),\n        (7, 8),\n        (5, 9),\n        (9, 10),\n        (10, 11),\n        (11, 12),\n        (9, 13),\n        (13, 14),\n        (14, 15),\n        (15, 16),\n        (13, 17),\n        (17, 18),\n        (18, 19),\n        (19, 20),\n    ],\n    \"right_hand\": [\n        (0, 1),\n        (1, 2),\n        (2, 3),\n        (3, 4),\n        (0, 5),\n        (0, 17),\n        (5, 6),\n        (6, 7),\n        (7, 8),\n        (5, 9),\n        (9, 10),\n        (10, 11),\n        (11, 12),\n        (9, 13),\n        (13, 14),\n        (14, 15),\n        (15, 16),\n        (13, 17),\n        (17, 18),\n        (18, 19),\n        (19, 20),\n    ],\n    \"pose\": [\n        (8, 6),\n        (6, 5),\n        (6, 4),\n        (4, 0),\n        (0, 1),\n        (1, 2),\n        (2, 3),\n        (3, 7),\n        (10, 9),\n        (11, 12),\n        (11, 13),\n        (11, 23),\n        (13, 15),\n        (15, 21),\n        (15, 17),\n        (15, 19),\n        (17, 19),\n        (12, 14),\n        (12, 24),\n        (14, 16),\n        (16, 22),\n        (16, 20),\n        (16, 18),\n        (18, 20),\n        (23, 24),\n        # discard landmarks of lower body\n        # (24, 26),\n        # (26, 28),\n        # (28, 30),\n        # (28, 32),\n        # (30, 32),\n        # (23, 25),\n        # (25, 27),\n        # (27, 29),\n        # (27, 31),\n        # (29, 31),\n    ],\n}","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-04-07T03:15:39.561813Z","iopub.execute_input":"2023-04-07T03:15:39.562497Z","iopub.status.idle":"2023-04-07T03:15:39.614527Z","shell.execute_reply.started":"2023-04-07T03:15:39.562454Z","shell.execute_reply":"2023-04-07T03:15:39.613081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lm_data = {k: v for k, v in zip(train.columns, train.row(0))}\n\ndf_landmark = pl.read_parquet(cfg.INPUT / lm_data[\"path\"])\nlm_first_frame = df_landmark.partition_by(\"frame\")[0]\nlms = lm_first_frame.partition_by(\"type\")\n\n_, axes = plt.subplots(2, 2, figsize=(8, 8))\naxes = axes.ravel()\n\nfor lm, ax in zip(lms, axes):\n    lm = lm.filter((pl.col(\"type\") != \"pose\") | (pl.col(\"landmark_index\") < 25))\n    lm_type = lm.row(0)[2]\n    ax.scatter(lm[\"x\"], lm[\"y\"])\n    if lm_type != \"face\":\n        for row in lm.iter_rows():\n            dt = {k: v for k, v in zip(lm.columns, row)}\n            x, y, idx = dt[\"x\"], dt[\"y\"], dt[\"landmark_index\"]\n\n            if (x is not None) & (y is not None):\n                ax.text(x, y, idx)\n    if lm_type in [\"left_hand\", \"right_hand\", \"pose\"]:\n        for edge in edges[lm_type]:\n            i, j = edge\n            x1, x2, y1, y2 = lm[\"x\"][i], lm[\"x\"][j], lm[\"y\"][i], lm[\"y\"][j]\n            if not ((x1 is None) | (x2 is None) | (y1 is None) | (y2 is None)):\n                ax.plot((x1, x2), (y1, y2), color=\"gray\")\n    ax.set(title=f\"{lm_type}\")\n    ax.invert_yaxis()","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2023-04-07T03:17:08.006238Z","iopub.execute_input":"2023-04-07T03:17:08.006627Z","iopub.status.idle":"2023-04-07T03:17:08.802724Z","shell.execute_reply.started":"2023-04-07T03:17:08.006594Z","shell.execute_reply":"2023-04-07T03:17:08.801887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, axes = plt.subplots(1, 3, figsize=(12, 4))\n\nfor ax, col in zip(axes, [\"x\", \"y\", \"z\"]):\n    ax.hist(df_landmark[col], bins=20, alpha=0.5, label=col)\n    ax.set(xlabel=col)\nplt.suptitle(\"Distribution of axes of normalized coordinate\")\nplt.show()","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2023-04-07T03:17:37.519221Z","iopub.execute_input":"2023-04-07T03:17:37.519626Z","iopub.status.idle":"2023-04-07T03:17:37.923708Z","shell.execute_reply.started":"2023-04-07T03:17:37.519592Z","shell.execute_reply":"2023-04-07T03:17:37.922379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Making Amination","metadata":{}},{"cell_type":"code","source":"def make_animation(row, columns, fps: int = 10):\n    data = {k: v for k, v in zip(columns, row)}\n    sign, participant_id, sequence_id = (\n        data[\"sign\"],\n        data[\"participant_id\"],\n        data[\"sequence_id\"],\n    )\n\n    df_landmark = pl.read_parquet(cfg.INPUT / data[\"path\"])\n    use_cols = [\"x\", \"y\", \"z\"]\n    df_landmark = df_landmark.sort([\"frame\", \"type\", \"landmark_index\"]).with_columns(\n        [pl.col(col).interpolate().over([\"type\", \"landmark_index\"]) for col in use_cols]\n    )\n\n    fig, axes = plt.subplots(2, 2, figsize=(8, 8))\n    axes = axes.ravel()\n\n    lms_all = df_landmark.partition_by(\"frame\")\n\n    def draw_frame(frame):\n        lms = lms_all[frame].partition_by(\"type\")\n\n        for lm, ax in zip(lms, axes):\n            lm = lm.filter((pl.col(\"type\") != \"pose\") | (pl.col(\"landmark_index\") < 25))\n            ax.cla()\n            lm_type = lm.row(0)[2]\n            frame = lm.row(0)[0]\n\n            ax.scatter(lm[\"x\"], lm[\"y\"])\n            if lm_type != \"face\":\n                for row in lm.iter_rows():\n                    dt = {k: v for k, v in zip(lm.columns, row)}\n                    if (dt[\"x\"] is not None) & (dt[\"y\"] is not None):\n                        ax.text(dt[\"x\"], dt[\"y\"], dt[\"landmark_index\"])\n            if lm_type in [\"left_hand\", \"right_hand\", \"pose\"]:\n                for edge in edges[lm_type]:\n                    i, j = edge\n                    x1, x2, y1, y2 = lm[\"x\"][i], lm[\"x\"][j], lm[\"y\"][i], lm[\"y\"][j]\n                    if not ((x1 is None) | (x2 is None) | (y1 is None) | (y2 is None)):\n                        ax.plot((x1, x2), (y1, y2), color=\"gray\")\n            ax.set(title=f\"{lm_type}\")\n            ax.invert_yaxis()\n        plt.suptitle(f'sign: \"{sign}\" [frame={frame}]')\n\n    ani = animation.FuncAnimation(\n        fig, draw_frame, frames=range(len(lms_all)), interval=1000 / fps\n    )\n\n    if not (cfg.OUTPUT / sign).exists():\n        (cfg.OUTPUT / sign).mkdir()\n    ani.save(\n        cfg.OUTPUT / sign / f\"{participant_id}_{sequence_id}.mp4\",\n        writer=\"ffmpeg\",\n        fps=fps,\n        codec=\"h264\",\n    )\n    plt.close(fig)","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2023-04-07T03:18:14.921866Z","iopub.execute_input":"2023-04-07T03:18:14.922302Z","iopub.status.idle":"2023-04-07T03:18:14.980267Z","shell.execute_reply.started":"2023-04-07T03:18:14.922237Z","shell.execute_reply":"2023-04-07T03:18:14.978648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Choose 5 samples per each group","metadata":{}},{"cell_type":"code","source":"train_unique_signs = train.filter(\n    (pl.arange(0, pl.count())).shuffle(seed=42).over(\"sign\") < 5\n)","metadata":{"execution":{"iopub.status.busy":"2023-04-07T03:18:15.674145Z","iopub.execute_input":"2023-04-07T03:18:15.674527Z","iopub.status.idle":"2023-04-07T03:18:15.713076Z","shell.execute_reply.started":"2023-04-07T03:18:15.674494Z","shell.execute_reply":"2023-04-07T03:18:15.712029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_unique_signs.groupby(\"sign\").agg(pl.count()).head()","metadata":{"execution":{"iopub.status.busy":"2023-04-07T03:18:15.889417Z","iopub.execute_input":"2023-04-07T03:18:15.889793Z","iopub.status.idle":"2023-04-07T03:18:15.921772Z","shell.execute_reply.started":"2023-04-07T03:18:15.889758Z","shell.execute_reply":"2023-04-07T03:18:15.920610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not cfg.OUTPUT.exists():\n    cfg.OUTPUT.mkdir()\n\nif cfg.DEBUG:\n    df = train_unique_signs.head(4)\nelse:\n    df = train_unique_signs\n\nfor row in tqdm(df.iter_rows(), total=len(df)):\n    make_animation(row, df.columns)","metadata":{"execution":{"iopub.status.busy":"2023-04-07T03:18:16.108379Z","iopub.execute_input":"2023-04-07T03:18:16.110295Z","iopub.status.idle":"2023-04-07T03:19:30.377358Z","shell.execute_reply.started":"2023-04-07T03:18:16.110204Z","shell.execute_reply":"2023-04-07T03:19:30.376251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!du -sh animation","metadata":{"execution":{"iopub.status.busy":"2023-04-07T03:19:30.379127Z","iopub.execute_input":"2023-04-07T03:19:30.379478Z","iopub.status.idle":"2023-04-07T03:19:30.671370Z","shell.execute_reply.started":"2023-04-07T03:19:30.379447Z","shell.execute_reply":"2023-04-07T03:19:30.669975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!tree animation | head","metadata":{"execution":{"iopub.status.busy":"2023-04-07T03:19:30.672858Z","iopub.execute_input":"2023-04-07T03:19:30.673829Z","iopub.status.idle":"2023-04-07T03:19:30.967682Z","shell.execute_reply.started":"2023-04-07T03:19:30.673789Z","shell.execute_reply":"2023-04-07T03:19:30.966491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Generated Animations\n\nYou can access to the full 250 class of signs in [ISLR: Animated 250 Sampled Signs](https://www.kaggle.com/datasets/tatamikenn/islr-animation-250-signs).\n","metadata":{}},{"cell_type":"code","source":"!cp animation/all/26734_1247514751.mp4 sample001.mp4\n!cp animation/bug/61333_1268993802.mp4 sample002.mp4\n!cp animation/lion/29302_1271911796.mp4 sample003.mp4\n!cp animation/TV/28656_125816896.mp4 sample004.mp4","metadata":{"execution":{"iopub.status.busy":"2023-04-07T03:19:30.970301Z","iopub.execute_input":"2023-04-07T03:19:30.970593Z","iopub.status.idle":"2023-04-07T03:19:32.046588Z","shell.execute_reply.started":"2023-04-07T03:19:30.970561Z","shell.execute_reply":"2023-04-07T03:19:32.045121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Video(\"sample001.mp4\")","metadata":{"execution":{"iopub.status.busy":"2023-04-07T03:19:32.047992Z","iopub.execute_input":"2023-04-07T03:19:32.048960Z","iopub.status.idle":"2023-04-07T03:19:32.081451Z","shell.execute_reply.started":"2023-04-07T03:19:32.048918Z","shell.execute_reply":"2023-04-07T03:19:32.080414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Video(\"sample002.mp4\")","metadata":{"execution":{"iopub.status.busy":"2023-04-07T03:19:32.082612Z","iopub.execute_input":"2023-04-07T03:19:32.083239Z","iopub.status.idle":"2023-04-07T03:19:32.115315Z","shell.execute_reply.started":"2023-04-07T03:19:32.083209Z","shell.execute_reply":"2023-04-07T03:19:32.113758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Video(\"sample003.mp4\")","metadata":{"execution":{"iopub.status.busy":"2023-04-07T03:19:32.116725Z","iopub.execute_input":"2023-04-07T03:19:32.117499Z","iopub.status.idle":"2023-04-07T03:19:32.152580Z","shell.execute_reply.started":"2023-04-07T03:19:32.117463Z","shell.execute_reply":"2023-04-07T03:19:32.151449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Video(\"sample004.mp4\")","metadata":{"execution":{"iopub.status.busy":"2023-04-07T03:19:32.153828Z","iopub.execute_input":"2023-04-07T03:19:32.154129Z","iopub.status.idle":"2023-04-07T03:19:32.187424Z","shell.execute_reply.started":"2023-04-07T03:19:32.154096Z","shell.execute_reply":"2023-04-07T03:19:32.186003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip -r animation.zip {cfg.OUTPUT.relative_to(\"/kaggle/working\")} && rm -rf {cfg.OUTPUT}","metadata":{"execution":{"iopub.status.busy":"2023-04-07T03:17:58.010754Z","iopub.status.idle":"2023-04-07T03:17:58.011868Z","shell.execute_reply.started":"2023-04-07T03:17:58.011583Z","shell.execute_reply":"2023-04-07T03:17:58.011623Z"},"trusted":true},"execution_count":null,"outputs":[]}]}