{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":130287,"databundleVersionId":15633993}],"dockerImageVersionId":31286,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\"\"\"\n================================================================================\n   SIGN LANGUAGE MOTION — EDA & VISUALIZATION NOTEBOOK\n   Share this on Kaggle community to get upvotes!\n   \n   Covers:\n   1. Dataset overview & statistics\n   2. Token distribution analysis\n   3. Gloss vocabulary analysis\n   4. Motion length analysis\n   5. Sample submission visualization\n================================================================================\n\"\"\"\n\n# ── Cell 1: Imports & Setup ──\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as mpatches\nfrom matplotlib.gridspec import GridSpec\nfrom matplotlib.colors import LinearSegmentedColormap\nimport seaborn as sns\nfrom collections import Counter\nimport re\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# Beautiful dark theme\nplt.rcParams.update({\n    \"figure.facecolor\":  \"#0d1117\",\n    \"axes.facecolor\":    \"#161b22\",\n    \"axes.edgecolor\":    \"#30363d\",\n    \"axes.labelcolor\":   \"#e6edf3\",\n    \"axes.titlecolor\":   \"#e6edf3\",\n    \"xtick.color\":       \"#8b949e\",\n    \"ytick.color\":       \"#8b949e\",\n    \"text.color\":        \"#e6edf3\",\n    \"grid.color\":        \"#21262d\",\n    \"grid.linestyle\":    \"--\",\n    \"grid.alpha\":        0.5,\n    \"font.family\":       \"monospace\",\n    \"figure.dpi\":        120,\n})\n\nACCENT   = \"#58a6ff\"   # blue\nACCENT2  = \"#3fb950\"   # green\nACCENT3  = \"#f78166\"   # red/orange\nACCENT4  = \"#d2a8ff\"   # purple\nACCENT5  = \"#ffa657\"   # orange\n\nDATA = \"/kaggle/input/competitions/motion-s-hierarchical-text-to-motion-generation-for-sign-language\"\nTOKEN_COLS = [\"base_tokens\", \"residual_1\", \"residual_2\",\n              \"residual_3\", \"residual_4\", \"residual_5\"]\n\nprint(\"Loading data...\")\ntrain  = pd.read_csv(f\"{DATA}/train.csv\")\ntest   = pd.read_csv(f\"{DATA}/test.csv\")\nsample = pd.read_csv(f\"{DATA}/sample_submission.csv\")\nprint(f\"Train: {train.shape} | Test: {test.shape}\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-02-28T12:27:50.746359Z","iopub.execute_input":"2026-02-28T12:27:50.746597Z","iopub.status.idle":"2026-02-28T12:27:54.195135Z","shell.execute_reply.started":"2026-02-28T12:27:50.746570Z","shell.execute_reply":"2026-02-28T12:27:54.194251Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ── Cell 2: Helper functions ──\ndef get_token_length(s):\n    if pd.isna(s): return 0\n    return len(str(s).strip().split())\n\ndef parse_tokens(s):\n    if pd.isna(s): return []\n    return [int(x) for x in str(s).strip().split()]\n\ndef clean_gloss(g):\n    if pd.isna(g): return \"\"\n    return re.sub(r\"//+\", \" \", str(g)).strip()\n\ntrain[\"token_len\"] = train[\"base_tokens\"].apply(get_token_length)\ntrain[\"gloss_clean\"] = train[\"gloss\"].apply(clean_gloss)\ntrain[\"gloss_words\"] = train[\"gloss_clean\"].apply(lambda g: len(g.split()))\ntrain[\"sentence_words\"] = train[\"sentence\"].apply(lambda s: len(str(s).split()))\n\nprint(\"Feature engineering done\")\nprint(train[[\"token_len\", \"gloss_words\", \"sentence_words\"]].describe().round(1))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-28T12:28:12.988139Z","iopub.execute_input":"2026-02-28T12:28:12.988427Z","iopub.status.idle":"2026-02-28T12:28:13.107270Z","shell.execute_reply.started":"2026-02-28T12:28:12.988404Z","shell.execute_reply":"2026-02-28T12:28:13.106495Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ── Cell 3: FIGURE 1 — Dataset Overview ──\nfig = plt.figure(figsize=(18, 10))\nfig.patch.set_facecolor(\"#0d1117\")\ngs = GridSpec(2, 3, figure=fig, hspace=0.45, wspace=0.35)\n\n# Title\nfig.text(0.5, 0.97,\n         \"Motion-S: Sign Language Generation — Dataset Overview\",\n         ha=\"center\", va=\"top\", fontsize=18, fontweight=\"bold\",\n         color=\"#e6edf3\", fontfamily=\"monospace\")\n\nfig.text(0.5, 0.93,\n         f\"Train: {len(train):,} samples  |  Test: {len(test):,} samples  |  \"\n         f\"6 RVQ Token Layers  |  Tokens ∈ [0, 511]\",\n         ha=\"center\", va=\"top\", fontsize=11, color=\"#8b949e\")\n\n# 1. Token Length Distribution\nax1 = fig.add_subplot(gs[0, 0])\nax1.hist(train[\"token_len\"], bins=60, color=ACCENT, alpha=0.85, edgecolor=\"#0d1117\")\nax1.axvline(train[\"token_len\"].median(), color=ACCENT3, lw=2,\n            linestyle=\"--\", label=f\"Median: {train['token_len'].median():.0f}\")\nax1.axvline(train[\"token_len\"].mean(), color=ACCENT2, lw=2,\n            linestyle=\":\", label=f\"Mean: {train['token_len'].mean():.0f}\")\nax1.set_title(\"Motion Token Length Distribution\", fontweight=\"bold\", pad=10)\nax1.set_xlabel(\"Sequence Length (tokens)\")\nax1.set_ylabel(\"Count\")\nax1.legend(fontsize=9)\nax1.grid(True, alpha=0.3)\n\n# 2. Gloss Word Count Distribution\nax2 = fig.add_subplot(gs[0, 1])\nax2.hist(train[\"gloss_words\"], bins=40, color=ACCENT2, alpha=0.85, edgecolor=\"#0d1117\")\nax2.axvline(train[\"gloss_words\"].median(), color=ACCENT3, lw=2,\n            linestyle=\"--\", label=f\"Median: {train['gloss_words'].median():.0f}\")\nax2.set_title(\"Gloss Word Count Distribution\", fontweight=\"bold\", pad=10)\nax2.set_xlabel(\"Number of Gloss Words\")\nax2.set_ylabel(\"Count\")\nax2.legend(fontsize=9)\nax2.grid(True, alpha=0.3)\n\n# 3. Tokens per Gloss Word (efficiency)\nax3 = fig.add_subplot(gs[0, 2])\ntpw = train[\"token_len\"] / (train[\"gloss_words\"] + 1e-6)\nax3.hist(tpw.clip(0, 50), bins=50, color=ACCENT4, alpha=0.85, edgecolor=\"#0d1117\")\nax3.axvline(tpw.median(), color=ACCENT3, lw=2,\n            linestyle=\"--\", label=f\"Median: {tpw.median():.1f} tok/word\")\nax3.set_title(\"Tokens per Gloss Word\", fontweight=\"bold\", pad=10)\nax3.set_xlabel(\"Tokens / Gloss Word\")\nax3.set_ylabel(\"Count\")\nax3.legend(fontsize=9)\nax3.grid(True, alpha=0.3)\n\n# 4. Token length vs gloss words scatter\nax4 = fig.add_subplot(gs[1, 0])\nsample_idx = np.random.choice(len(train), min(2000, len(train)), replace=False)\nax4.scatter(train[\"gloss_words\"].iloc[sample_idx],\n            train[\"token_len\"].iloc[sample_idx],\n            alpha=0.3, s=8, color=ACCENT, rasterized=True)\n# Trend line\nz = np.polyfit(train[\"gloss_words\"], train[\"token_len\"], 1)\np = np.poly1d(z)\nxs = np.linspace(train[\"gloss_words\"].min(), train[\"gloss_words\"].max(), 100)\nax4.plot(xs, p(xs), color=ACCENT3, lw=2, label=f\"slope={z[0]:.1f}\")\nax4.set_title(\"Token Length vs Gloss Words\", fontweight=\"bold\", pad=10)\nax4.set_xlabel(\"Gloss Word Count\")\nax4.set_ylabel(\"Token Length\")\nax4.legend(fontsize=9)\nax4.grid(True, alpha=0.3)\n\n# 5. Length bins\nax5 = fig.add_subplot(gs[1, 1])\nbins = [0, 100, 200, 300, 400, 500, 600, 800]\nlabels = [\"0-100\", \"100-200\", \"200-300\", \"300-400\",\n          \"400-500\", \"500-600\", \"600-800\"]\ncounts = pd.cut(train[\"token_len\"], bins=bins, labels=labels).value_counts().sort_index()\ncolors = [ACCENT, ACCENT2, ACCENT4, ACCENT3, ACCENT5, \"#79c0ff\", \"#56d364\"]\nbars = ax5.bar(labels, counts.values, color=colors, alpha=0.85, edgecolor=\"#0d1117\")\nax5.set_title(\"Token Length Buckets\", fontweight=\"bold\", pad=10)\nax5.set_xlabel(\"Length Range\")\nax5.set_ylabel(\"Count\")\nax5.tick_params(axis=\"x\", rotation=35)\nax5.grid(True, alpha=0.3, axis=\"y\")\nfor bar, val in zip(bars, counts.values):\n    ax5.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 30,\n             f\"{val:,}\", ha=\"center\", va=\"bottom\", fontsize=8, color=\"#8b949e\")\n\n# 6. Key stats table\nax6 = fig.add_subplot(gs[1, 2])\nax6.axis(\"off\")\nstats = [\n    [\"Metric\", \"Value\"],\n    [\"Train Samples\", f\"{len(train):,}\"],\n    [\"Test Samples\", f\"{len(test):,}\"],\n    [\"Avg Token Length\", f\"{train['token_len'].mean():.0f}\"],\n    [\"Median Token Length\", f\"{train['token_len'].median():.0f}\"],\n    [\"Min / Max Length\", f\"{train['token_len'].min()} / {train['token_len'].max()}\"],\n    [\"Avg Gloss Words\", f\"{train['gloss_words'].mean():.1f}\"],\n    [\"Token Layers\", \"6 (base + 5 residual)\"],\n    [\"Codebook Size\", \"512 per layer\"],\n]\ntable = ax6.table(cellText=stats[1:], colLabels=stats[0],\n                  loc=\"center\", cellLoc=\"left\")\ntable.auto_set_font_size(False)\ntable.set_fontsize(9)\ntable.scale(1.2, 1.8)\nfor (r, c), cell in table.get_celld().items():\n    cell.set_facecolor(\"#161b22\" if r % 2 == 0 else \"#21262d\")\n    cell.set_edgecolor(\"#30363d\")\n    cell.set_text_props(color=\"#e6edf3\")\n    if r == 0:\n        cell.set_facecolor(\"#1f6feb\")\n        cell.set_text_props(fontweight=\"bold\", color=\"white\")\nax6.set_title(\"Dataset Statistics\", fontweight=\"bold\", pad=15)\n\nplt.savefig(\"fig1_overview.png\", bbox_inches=\"tight\",\n            facecolor=\"#0d1117\", dpi=120)\nplt.show()\nprint(\" Figure 1 saved\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-28T12:28:17.013514Z","iopub.execute_input":"2026-02-28T12:28:17.014021Z","iopub.status.idle":"2026-02-28T12:28:19.038926Z","shell.execute_reply.started":"2026-02-28T12:28:17.013996Z","shell.execute_reply":"2026-02-28T12:28:19.038080Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ── Cell 4: FIGURE 2 — Token Distribution Analysis ──\nfig2, axes = plt.subplots(2, 3, figsize=(18, 10))\nfig2.patch.set_facecolor(\"#0d1117\")\nfig2.suptitle(\"RVQ Token Distribution Across All 6 Layers\",\n              fontsize=16, fontweight=\"bold\", color=\"#e6edf3\", y=0.98)\n\nlayer_colors = [ACCENT, ACCENT2, ACCENT4, ACCENT3, ACCENT5, \"#56d364\"]\n\nfor i, (col, ax, color) in enumerate(zip(TOKEN_COLS, axes.flat, layer_colors)):\n    # Sample tokens from first 500 samples for speed\n    all_tokens = []\n    for val in train[col].iloc[:500]:\n        all_tokens.extend(parse_tokens(val))\n\n    if not all_tokens:\n        continue\n\n    all_tokens = np.array(all_tokens)\n    ax.hist(all_tokens, bins=64, color=color, alpha=0.85, edgecolor=\"#0d1117\")\n    ax.set_title(f\"{'Base Tokens' if i==0 else f'Residual {i}'}\",\n                 fontweight=\"bold\", pad=8)\n    ax.set_xlabel(\"Token Index [0-511]\")\n    ax.set_ylabel(\"Frequency\")\n    ax.grid(True, alpha=0.3)\n\n    # Stats annotation\n    ax.text(0.97, 0.95,\n            f\"mean={all_tokens.mean():.0f}\\nstd={all_tokens.std():.0f}\\n\"\n            f\"unique={len(np.unique(all_tokens))}\",\n            transform=ax.transAxes, ha=\"right\", va=\"top\",\n            fontsize=8, color=\"#8b949e\",\n            bbox=dict(facecolor=\"#21262d\", edgecolor=\"#30363d\",\n                      boxstyle=\"round,pad=0.3\"))\n\nplt.tight_layout()\nplt.savefig(\"fig2_token_dist.png\", bbox_inches=\"tight\",\n            facecolor=\"#0d1117\", dpi=120)\nplt.show()\nprint(\"Figure 2 saved\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-28T12:28:27.794658Z","iopub.execute_input":"2026-02-28T12:28:27.794959Z","iopub.status.idle":"2026-02-28T12:28:29.873156Z","shell.execute_reply.started":"2026-02-28T12:28:27.794936Z","shell.execute_reply":"2026-02-28T12:28:29.872113Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ── Cell 5: FIGURE 3 — Gloss Vocabulary Analysis ──\n# Extract all gloss words\nall_gloss_words = []\nfor g in train[\"gloss_clean\"]:\n    words = str(g).upper().replace(\"//\", \" \").split()\n    all_gloss_words.extend(words)\n\nword_counts = Counter(all_gloss_words)\ntop_words = word_counts.most_common(30)\nwords, freqs = zip(*top_words)\n\nfig3, axes3 = plt.subplots(1, 2, figsize=(18, 8))\nfig3.patch.set_facecolor(\"#0d1117\")\nfig3.suptitle(\"Gloss Vocabulary Analysis\",\n              fontsize=16, fontweight=\"bold\", color=\"#e6edf3\")\n\n# Top 30 gloss words\nax = axes3[0]\ncmap_vals = np.linspace(0.3, 1.0, len(words))\nbar_colors = plt.cm.Blues(cmap_vals)\nbars = ax.barh(list(reversed(words)), list(reversed(freqs)),\n               color=list(reversed(bar_colors)), edgecolor=\"#0d1117\")\nax.set_title(\"Top 30 Most Common Gloss Words\", fontweight=\"bold\", pad=10)\nax.set_xlabel(\"Frequency\")\nax.grid(True, alpha=0.3, axis=\"x\")\nfor bar, val in zip(bars, reversed(freqs)):\n    ax.text(bar.get_width() + 5, bar.get_y() + bar.get_height()/2,\n            f\"{val:,}\", va=\"center\", fontsize=8, color=\"#8b949e\")\n\n# Vocabulary frequency distribution (log scale)\nax2 = axes3[1]\nall_freqs = sorted(word_counts.values(), reverse=True)\nax2.plot(range(1, len(all_freqs)+1), all_freqs,\n         color=ACCENT, lw=2, alpha=0.9)\nax2.fill_between(range(1, len(all_freqs)+1), all_freqs,\n                 alpha=0.15, color=ACCENT)\nax2.set_xscale(\"log\")\nax2.set_yscale(\"log\")\nax2.set_title(\"Vocabulary Frequency Distribution (Zipf's Law)\",\n              fontweight=\"bold\", pad=10)\nax2.set_xlabel(\"Rank (log scale)\")\nax2.set_ylabel(\"Frequency (log scale)\")\nax2.grid(True, alpha=0.3)\nax2.text(0.05, 0.15,\n         f\"Total unique glosses: {len(word_counts):,}\\n\"\n         f\"Total gloss tokens: {sum(all_freqs):,}\\n\"\n         f\"Top-10 cover: {sum(all_freqs[:10])/sum(all_freqs)*100:.1f}%\",\n         transform=ax2.transAxes, fontsize=10, color=\"#8b949e\",\n         bbox=dict(facecolor=\"#21262d\", edgecolor=\"#30363d\",\n                   boxstyle=\"round,pad=0.5\"))\n\nplt.tight_layout()\nplt.savefig(\"fig3_vocabulary.png\", bbox_inches=\"tight\",\n            facecolor=\"#0d1117\", dpi=120)\nplt.show()\nprint(\"Figure 3 saved\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-28T12:28:33.525506Z","iopub.execute_input":"2026-02-28T12:28:33.525784Z","iopub.status.idle":"2026-02-28T12:28:35.034148Z","shell.execute_reply.started":"2026-02-28T12:28:33.525756Z","shell.execute_reply":"2026-02-28T12:28:35.033403Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ── Cell 6: FIGURE 4 — Token Heatmap for sample sequences ──\nfig4, axes4 = plt.subplots(5, 1, figsize=(18, 12))\nfig4.patch.set_facecolor(\"#0d1117\")\nfig4.suptitle(\"Token Heatmap — 5 Sample Training Sequences (All 6 Layers)\",\n              fontsize=15, fontweight=\"bold\", color=\"#e6edf3\")\n\n# Custom colormap\ncmap = LinearSegmentedColormap.from_list(\n    \"motion\", [\"#0d1117\", \"#1f6feb\", \"#58a6ff\", \"#cae8ff\"], N=512\n)\n\nfor idx in range(5):\n    ax = axes4[idx]\n    row = train.iloc[idx * 200]  # spread across dataset\n\n    # Build matrix: 6 layers × sequence_length\n    matrix = []\n    min_len = None\n    for col in TOKEN_COLS:\n        toks = parse_tokens(row[col])\n        if min_len is None:\n            min_len = len(toks)\n        min_len = min(min_len, len(toks))\n        matrix.append(toks)\n\n    # Trim to common length\n    matrix = np.array([m[:min_len] for m in matrix])\n\n    # Subsample if too long for display\n    if matrix.shape[1] > 200:\n        step = matrix.shape[1] // 200\n        matrix = matrix[:, ::step]\n\n    im = ax.imshow(matrix, aspect=\"auto\", cmap=cmap,\n                   vmin=0, vmax=511, interpolation=\"nearest\")\n    ax.set_yticks(range(6))\n    ax.set_yticklabels([\"Base\", \"Res1\", \"Res2\", \"Res3\", \"Res4\", \"Res5\"],\n                       fontsize=8)\n    gloss_short = str(row[\"gloss_clean\"])[:60]\n    ax.set_title(f'Sample {idx+1}: \"{gloss_short}\"  '\n                 f'[len={min_len}]',\n                 fontsize=9, pad=4, loc=\"left\")\n    ax.set_xlabel(\"Token Position\", fontsize=8)\n    plt.colorbar(im, ax=ax, fraction=0.01, pad=0.01,\n                 label=\"Token Index\")\n\nplt.tight_layout()\nplt.savefig(\"fig4_heatmap.png\", bbox_inches=\"tight\",\n            facecolor=\"#0d1117\", dpi=120)\nplt.show()\nprint(\"Figure 4 saved\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-28T12:28:38.615941Z","iopub.execute_input":"2026-02-28T12:28:38.616680Z","iopub.status.idle":"2026-02-28T12:28:40.328509Z","shell.execute_reply.started":"2026-02-28T12:28:38.616648Z","shell.execute_reply":"2026-02-28T12:28:40.327576Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ── Cell 7: FIGURE 5 — Competition Task Summary ──\nfig5, ax5 = plt.subplots(figsize=(16, 7))\nfig5.patch.set_facecolor(\"#0d1117\")\nax5.set_facecolor(\"#0d1117\")\nax5.axis(\"off\")\n\n# Draw pipeline diagram\ndef draw_box(ax, x, y, w, h, text, color, fontsize=11):\n    rect = mpatches.FancyBboxPatch((x, y), w, h,\n        boxstyle=\"round,pad=0.02\",\n        facecolor=color, edgecolor=\"#e6edf3\",\n        linewidth=1.5, alpha=0.9)\n    ax.add_patch(rect)\n    ax.text(x + w/2, y + h/2, text,\n            ha=\"center\", va=\"center\",\n            fontsize=fontsize, fontweight=\"bold\",\n            color=\"white\", multialignment=\"center\")\n\ndef draw_arrow(ax, x1, x2, y):\n    ax.annotate(\"\", xy=(x2, y), xytext=(x1, y),\n                arrowprops=dict(arrowstyle=\"->\", color=\"#8b949e\",\n                                lw=2, mutation_scale=20))\n\nax5.set_xlim(0, 10)\nax5.set_ylim(0, 4)\n\ndraw_box(ax5, 0.1, 1.2, 1.8, 1.6,\n         \"Input\\nEnglish Text\\n+\\nGloss\", \"#1f6feb\")\ndraw_arrow(ax5, 1.9, 2.4, 2.0)\n\ndraw_box(ax5, 2.4, 1.2, 2.0, 1.6,\n         \"Your\\nModel\\n(Text→Motion)\", \"#388bfd\")\ndraw_arrow(ax5, 4.4, 4.9, 2.0)\n\ndraw_box(ax5, 4.9, 0.3, 2.2, 3.4,\n         \"6 Token Layers\\n\\nBase Tokens\\nResidual 1-5\\n\\nEach ∈ [0-511]\", \"#1f6feb\", 10)\ndraw_arrow(ax5, 7.1, 7.6, 2.0)\n\ndraw_box(ax5, 7.6, 1.2, 2.2, 1.6,\n         \"Output\\nSign Language\\nAnimation\", \"#238636\")\n\n# Metric boxes\nfor i, (metric, weight, color) in enumerate([\n    (\"R-Precision\\n(Text Alignment)\", \"50%\", ACCENT2),\n    (\"FID\\n(Realism)\", \"30%\", ACCENT),\n    (\"Diversity\\n(Variety)\", \"20%\", ACCENT4),\n]):\n    draw_box(ax5, 0.3 + i*3.2, -0.1, 2.6, 0.8, f\"{metric}  {weight}\", color, 9)\n\nax5.text(5.0, -0.45, \"Evaluation Metrics\",\n         ha=\"center\", fontsize=10, color=\"#8b949e\")\n\nax5.set_title(\"Competition Pipeline: Text → Sign Language Motion Tokens\",\n              fontsize=14, fontweight=\"bold\", color=\"#e6edf3\", pad=15)\n\nplt.tight_layout()\nplt.savefig(\"fig5_pipeline.png\", bbox_inches=\"tight\",\n            facecolor=\"#0d1117\", dpi=120)\nplt.show()\nprint(\"Figure 5 saved\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-28T12:28:46.960016Z","iopub.execute_input":"2026-02-28T12:28:46.960689Z","iopub.status.idle":"2026-02-28T12:28:47.322263Z","shell.execute_reply.started":"2026-02-28T12:28:46.960663Z","shell.execute_reply":"2026-02-28T12:28:47.321542Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ── Cell 8: Summary Print ──\nprint(\"\\n\" + \"=\"*60)\nprint(\"EDA COMPLETE — KEY INSIGHTS\")\nprint(\"=\"*60)\nprint(f\"\"\"\nDataset:\n  • {len(train):,} training samples, {len(test):,} test samples\n  • Average sequence: {train['token_len'].mean():.0f} tokens\n  • Range: {train['token_len'].min()} – {train['token_len'].max()} tokens\n\nTokens:\n  • 6 hierarchical RVQ layers (base + 5 residuals)\n  • Each token ∈ [0, 511] — 512-entry codebook\n  • Base layer captures coarse motion\n  • Residuals refine progressively (fine detail)\n\nVocabulary:\n  • {len(word_counts):,} unique gloss signs\n  • Strong Zipf distribution — top 10 signs cover\n    {sum(list(word_counts.values())[:10])/sum(word_counts.values())*100:.1f}% of all occurrences\n\nApproach Ideas:\n  1. Retrieval baseline — copy tokens from similar train samples\n  2. Fine-tuned T5/BART — seq2seq gloss → token generation\n  3. Autoregressive GPT — predict tokens one by one\n  4. Non-autoregressive — predict all tokens in parallel\n\"\"\")\nprint(\"=\"*60)\nprint(\"Figures saved: fig1_overview.png, fig2_token_dist.png,\")\nprint(\"               fig3_vocabulary.png, fig4_heatmap.png,\")\nprint(\"               fig5_pipeline.png\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-28T12:29:04.293358Z","iopub.execute_input":"2026-02-28T12:29:04.293934Z","iopub.status.idle":"2026-02-28T12:29:04.300209Z","shell.execute_reply.started":"2026-02-28T12:29:04.293910Z","shell.execute_reply":"2026-02-28T12:29:04.299601Z"}},"outputs":[],"execution_count":null}]}