{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.12"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":130287,"databundleVersionId":15633993,"sourceType":"competition"}],"dockerImageVersionId":31259,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":514.908468,"end_time":"2026-02-09T12:09:33.546298","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2026-02-09T12:00:58.637830","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"a1edcb1e","cell_type":"markdown","source":"# Motion-S Dataset Analysis\n\nThis notebook provides comprehensive analysis of the sign language motion dataset including:\n- Dataset structure and statistics\n- Text and gloss analysis\n- Motion feature analysis\n- Quality checks and anomaly detection","metadata":{"papermill":{"duration":0.006369,"end_time":"2026-02-09T12:01:02.093442","exception":false,"start_time":"2026-02-09T12:01:02.087073","status":"completed"},"tags":[]}},{"id":"234b1b8e","cell_type":"markdown","source":"## 1. Setup and Configuration","metadata":{"papermill":{"duration":0.005245,"end_time":"2026-02-09T12:01:02.104071","exception":false,"start_time":"2026-02-09T12:01:02.098826","status":"completed"},"tags":[]}},{"id":"3052d092","cell_type":"code","source":"%%capture \nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"papermill":{"duration":77.765432,"end_time":"2026-02-09T12:02:19.874731","exception":false,"start_time":"2026-02-09T12:01:02.109299","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"a481df85","cell_type":"code","source":"import sys\nfrom pathlib import Path\nfrom collections import Counter, defaultdict\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm import tqdm\nimport warnings\n\n# --- Environment and Visuals ---\nwarnings.filterwarnings('ignore')\nsns.set_style(\"whitegrid\")\nplt.rcParams['figure.figsize'] = (12, 6)\nplt.rcParams['font.size'] = 11\n\n# --- Updated Path Configuration ---\nIS_KAGGLE = Path(\"/kaggle/working\").exists()\n\nif IS_KAGGLE:\n    INPUT_DIR = Path(\"/kaggle/input/competitions/motion-s-hierarchical-text-to-motion-generation-for-sign-language\")\n    MODEL_DIR = Path(\"/kaggle/input/motion-s-vae-rvq/pytorch/default/3\")\n    \n    DATASET_ROOT = INPUT_DIR / \"Train\"\n    # If you are READING pre-processed features:\n    MOTION_FEATS_DIR = INPUT_DIR / \"Motion-Features\" \n    # If you are GENERATING new features, use this instead:\n    # MOTION_FEATS_DIR = Path(\"/kaggle/working/Motion-Features\")\n    \n    TRAIN_CSV = INPUT_DIR / \"train.csv\"\n    MODEL_PATH = MODEL_DIR / \"rvq_vae_best.pth\"\n    NORM_PATH = INPUT_DIR / \"normalization.npz\"\n    print(\"Running on Kaggle\")\nelse:\n    PROJECT_ROOT = Path(\"..\").resolve()\n    DATASET_ROOT = PROJECT_ROOT / \"Train\"\n    MOTION_FEATS_DIR = PROJECT_ROOT / \"Motion-Features\"\n    TRAIN_CSV = PROJECT_ROOT / \"train.csv\"\n    MODEL_PATH = PROJECT_ROOT / \"models\" / \"rvq_vae_best.pth\"\n    NORM_PATH = PROJECT_ROOT / \"normalization.npz\"\n    print(f\"Running locally at {PROJECT_ROOT}\")\n\n# --- Helper Function ---\ndef get_bvh_path(inp):\n    \"\"\"Handles both raw IDs and the //dataset/ paths from the CSV.\"\"\"\n    inp_str = str(inp)\n    if \"dataset\" in inp_str:\n        clean_path = inp_str.replace(\"//dataset/\", \"\").replace(\"dataset/\", \"\").lstrip(\"/\")\n        return DATASET_ROOT / clean_path\n    return DATASET_ROOT / inp_str / f\"{inp_str}.bvh\"\n\n\n","metadata":{"papermill":{"duration":3.292676,"end_time":"2026-02-09T12:02:23.172859","exception":false,"start_time":"2026-02-09T12:02:19.880183","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"2f428017","cell_type":"code","source":"# Helper functions\n\ndef load_metadata(sample_dir: Path) -> dict:\n    \"\"\"Load metadata.txt from a sample directory.\"\"\"\n    metadata_path = sample_dir / \"metadata.txt\"\n    if not metadata_path.exists():\n        return None\n    \n    result = {}\n    try:\n        with open(metadata_path, 'r', encoding='utf-8') as f:\n            for line in f:\n                line = line.strip()\n                if line.startswith(\"SENTENCE:\"):\n                    result['sentence'] = line.replace(\"SENTENCE:\", \"\").strip()\n                elif line.startswith(\"GLOSS:\"):\n                    result['gloss'] = line.replace(\"GLOSS:\", \"\").strip()\n    except Exception as e:\n        return None\n    \n    return result if 'sentence' in result and 'gloss' in result else None\n\n\ndef count_bvh_files(sample_dir: Path) -> int:\n    \"\"\"Count BVH files in a sample directory.\"\"\"\n    return len(list(sample_dir.glob(\"*.bvh\")))\n\n\ndef parse_glosses(gloss_str: str) -> list:\n    \"\"\"Parse gloss string into list of individual glosses.\"\"\"\n    # Remove sentence boundary markers and split\n    cleaned = gloss_str.replace(\"//\", \"\").strip()\n    return [g.strip() for g in cleaned.split() if g.strip()]\n\n\ndef is_fingerspelling(gloss: str) -> bool:\n    \"\"\"Check if a gloss is a fingerspelled letter (single uppercase letter).\"\"\"\n    return len(gloss) == 1 and gloss.isupper()","metadata":{"papermill":{"duration":0.016534,"end_time":"2026-02-09T12:02:23.194858","exception":false,"start_time":"2026-02-09T12:02:23.178324","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"0f81391a","cell_type":"markdown","source":"## 2. Dataset Overview","metadata":{"papermill":{"duration":0.005241,"end_time":"2026-02-09T12:02:23.205476","exception":false,"start_time":"2026-02-09T12:02:23.200235","status":"completed"},"tags":[]}},{"id":"2ded5f94","cell_type":"code","source":"# Load all sample data into a dataframe\nsamples_data = []\n\nif DATASET_ROOT.exists():\n    sample_dirs = [d for d in DATASET_ROOT.iterdir() if d.is_dir()]\n    \n    for sample_dir in tqdm(sample_dirs, desc=\"Loading samples\"):\n        sample_id = sample_dir.name\n        metadata = load_metadata(sample_dir)\n        bvh_count = count_bvh_files(sample_dir)\n        \n        if metadata:\n            glosses = parse_glosses(metadata['gloss'])\n            fingerspell_count = sum(1 for g in glosses if is_fingerspelling(g))\n            \n            samples_data.append({\n                'sample_id': sample_id,\n                'sentence': metadata['sentence'],\n                'gloss': metadata['gloss'],\n                'gloss_list': glosses,\n                'gloss_count': len(glosses),\n                'fingerspell_count': fingerspell_count,\n                'bvh_count': bvh_count,\n                'sentence_word_count': len(metadata['sentence'].split()),\n                'sentence_char_count': len(metadata['sentence']),\n            })\n        else:\n            samples_data.append({\n                'sample_id': sample_id,\n                'sentence': None,\n                'gloss': None,\n                'gloss_list': [],\n                'gloss_count': 0,\n                'fingerspell_count': 0,\n                'bvh_count': bvh_count,\n                'sentence_word_count': 0,\n                'sentence_char_count': 0,\n            })\n\ndf = pd.DataFrame(samples_data)\nprint(f\"Loaded {len(df)} samples\")","metadata":{"papermill":{"duration":66.085346,"end_time":"2026-02-09T12:03:29.297078","exception":false,"start_time":"2026-02-09T12:02:23.211732","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"5f840c91","cell_type":"code","source":"# Basic statistics\nvalid_df = df[df['sentence'].notna()]\n\nprint(\"=\" * 50)\nprint(\"DATASET STATISTICS\")\nprint(\"=\" * 50)\nprint(f\"Total samples:           {len(df):,}\")\nprint(f\"Valid samples (w/ meta): {len(valid_df):,}\")\nprint(f\"Missing metadata:        {len(df) - len(valid_df):,}\")\nprint(f\"Total BVH files:         {df['bvh_count'].sum():,}\")\nprint()\nprint(\"BVH files per sample:\")\nprint(f\"  Mean:   {df['bvh_count'].mean():.1f}\")\nprint(f\"  Median: {df['bvh_count'].median():.1f}\")\nprint(f\"  Min:    {df['bvh_count'].min()}\")\nprint(f\"  Max:    {df['bvh_count'].max()}\")","metadata":{"papermill":{"duration":0.051008,"end_time":"2026-02-09T12:03:29.377697","exception":false,"start_time":"2026-02-09T12:03:29.326689","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"1b8f6579","cell_type":"markdown","source":"## 3. Text and Linguistic Analysis\n\n### 3.1 Sentence Statistics","metadata":{"papermill":{"duration":0.030086,"end_time":"2026-02-09T12:03:29.437683","exception":false,"start_time":"2026-02-09T12:03:29.407597","status":"completed"},"tags":[]}},{"id":"94932a0a","cell_type":"code","source":"# Sentence length statistics\nprint(\"SENTENCE STATISTICS\")\nprint(\"-\" * 40)\nprint(f\"Word count:\")\nprint(f\"  Mean:   {valid_df['sentence_word_count'].mean():.1f}\")\nprint(f\"  Median: {valid_df['sentence_word_count'].median():.1f}\")\nprint(f\"  Min:    {valid_df['sentence_word_count'].min()}\")\nprint(f\"  Max:    {valid_df['sentence_word_count'].max()}\")\nprint()\nprint(f\"Character count:\")\nprint(f\"  Mean:   {valid_df['sentence_char_count'].mean():.1f}\")\nprint(f\"  Median: {valid_df['sentence_char_count'].median():.1f}\")\nprint(f\"  Min:    {valid_df['sentence_char_count'].min()}\")\nprint(f\"  Max:    {valid_df['sentence_char_count'].max()}\")\n\n# Build vocabulary\nall_words = []\nfor sentence in valid_df['sentence']:\n    words = sentence.lower().split()\n    # Remove punctuation\n    words = [''.join(c for c in w if c.isalnum()) for w in words]\n    words = [w for w in words if w]\n    all_words.extend(words)\n\nword_counts = Counter(all_words)\nprint(f\"\\nVocabulary size: {len(word_counts):,} unique words\")","metadata":{"papermill":{"duration":0.136387,"end_time":"2026-02-09T12:03:29.605431","exception":false,"start_time":"2026-02-09T12:03:29.469044","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"fc50c7c2","cell_type":"code","source":"# Sentence length distribution\nfig, axes = plt.subplots(1, 2, figsize=(14, 5))\n\naxes[0].hist(valid_df['sentence_word_count'], bins=30, edgecolor='black', alpha=0.7)\naxes[0].set_xlabel('Word Count')\naxes[0].set_ylabel('Frequency')\naxes[0].set_title('Sentence Length Distribution (Words)')\naxes[0].axvline(valid_df['sentence_word_count'].mean(), color='red', linestyle='--', label=f'Mean: {valid_df[\"sentence_word_count\"].mean():.1f}')\naxes[0].legend()\n\naxes[1].hist(valid_df['sentence_char_count'], bins=30, edgecolor='black', alpha=0.7, color='orange')\naxes[1].set_xlabel('Character Count')\naxes[1].set_ylabel('Frequency')\naxes[1].set_title('Sentence Length Distribution (Characters)')\naxes[1].axvline(valid_df['sentence_char_count'].mean(), color='red', linestyle='--', label=f'Mean: {valid_df[\"sentence_char_count\"].mean():.1f}')\naxes[1].legend()\n\nplt.tight_layout()\nplt.show()","metadata":{"papermill":{"duration":0.610711,"end_time":"2026-02-09T12:03:30.246112","exception":false,"start_time":"2026-02-09T12:03:29.635401","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"fa12ee78","cell_type":"code","source":"# Top 30 most common words\ntop_words = word_counts.most_common(30)\nwords, counts = zip(*top_words)\n\nplt.figure(figsize=(14, 6))\nplt.bar(words, counts, edgecolor='black', alpha=0.7)\nplt.xlabel('Word')\nplt.ylabel('Frequency')\nplt.title('Top 30 Most Common Words in Sentences')\nplt.xticks(rotation=45, ha='right')\nplt.tight_layout()\nplt.show()\n\nprint(\"\\nTop 30 words:\")\nfor i, (word, count) in enumerate(top_words, 1):\n    print(f\"  {i:2}. {word:15} {count:,}\")","metadata":{"papermill":{"duration":0.352188,"end_time":"2026-02-09T12:03:30.630263","exception":false,"start_time":"2026-02-09T12:03:30.278075","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"1c05f8e4","cell_type":"markdown","source":"### 3.2 Gloss Statistics","metadata":{"papermill":{"duration":0.03202,"end_time":"2026-02-09T12:03:30.693647","exception":false,"start_time":"2026-02-09T12:03:30.661627","status":"completed"},"tags":[]}},{"id":"fa8d13fe","cell_type":"code","source":"# Gloss statistics\nprint(\"GLOSS STATISTICS\")\nprint(\"-\" * 40)\nprint(f\"Gloss count per sample:\")\nprint(f\"  Mean:   {valid_df['gloss_count'].mean():.1f}\")\nprint(f\"  Median: {valid_df['gloss_count'].median():.1f}\")\nprint(f\"  Min:    {valid_df['gloss_count'].min()}\")\nprint(f\"  Max:    {valid_df['gloss_count'].max()}\")\n\n# Build gloss vocabulary\nall_glosses = []\nfor gloss_list in valid_df['gloss_list']:\n    all_glosses.extend(gloss_list)\n\ngloss_counts = Counter(all_glosses)\nprint(f\"\\nUnique glosses: {len(gloss_counts):,}\")\nprint(f\"Total gloss occurrences: {len(all_glosses):,}\")\n\n# Fingerspelling analysis\nfingerspell_glosses = [g for g in all_glosses if is_fingerspelling(g)]\nprint(f\"\\nFingerspelling:\")\nprint(f\"  Total fingerspelled letters: {len(fingerspell_glosses):,}\")\nprint(f\"  Percentage of all glosses: {100*len(fingerspell_glosses)/len(all_glosses):.1f}%\")\nprint(f\"  Samples with fingerspelling: {(valid_df['fingerspell_count'] > 0).sum():,} ({100*(valid_df['fingerspell_count'] > 0).mean():.1f}%)\")","metadata":{"papermill":{"duration":0.063536,"end_time":"2026-02-09T12:03:30.788467","exception":false,"start_time":"2026-02-09T12:03:30.724931","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"c1725096","cell_type":"code","source":"# Gloss distribution\nfig, axes = plt.subplots(1, 2, figsize=(14, 5))\n\naxes[0].hist(valid_df['gloss_count'], bins=30, edgecolor='black', alpha=0.7, color='green')\naxes[0].set_xlabel('Gloss Count')\naxes[0].set_ylabel('Frequency')\naxes[0].set_title('Gloss Count Distribution per Sample')\naxes[0].axvline(valid_df['gloss_count'].mean(), color='red', linestyle='--', label=f'Mean: {valid_df[\"gloss_count\"].mean():.1f}')\naxes[0].legend()\n\n# Top glosses (excluding single letters)\nnon_fingerspell = {g: c for g, c in gloss_counts.items() if not is_fingerspelling(g)}\ntop_glosses = sorted(non_fingerspell.items(), key=lambda x: x[1], reverse=True)[:20]\nglosses, counts = zip(*top_glosses)\n\naxes[1].barh(list(reversed(glosses)), list(reversed(counts)), edgecolor='black', alpha=0.7, color='green')\naxes[1].set_xlabel('Frequency')\naxes[1].set_ylabel('Gloss')\naxes[1].set_title('Top 20 Most Common Glosses (excluding fingerspelling)')\n\nplt.tight_layout()\nplt.show()","metadata":{"papermill":{"duration":0.483535,"end_time":"2026-02-09T12:03:31.303602","exception":false,"start_time":"2026-02-09T12:03:30.820067","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"86e1ec73","cell_type":"markdown","source":"### 3.3 Sentence-Gloss Relationship","metadata":{"papermill":{"duration":0.037852,"end_time":"2026-02-09T12:03:31.374471","exception":false,"start_time":"2026-02-09T12:03:31.336619","status":"completed"},"tags":[]}},{"id":"fd217349","cell_type":"code","source":"# Sentence-gloss compression analysis\nvalid_df['compression_ratio'] = valid_df['gloss_count'] / valid_df['sentence_word_count']\n\nprint(\"SENTENCE-GLOSS RELATIONSHIP\")\nprint(\"-\" * 40)\nprint(f\"Compression ratio (gloss_count / word_count):\")\nprint(f\"  Mean:   {valid_df['compression_ratio'].mean():.2f}\")\nprint(f\"  Median: {valid_df['compression_ratio'].median():.2f}\")\nprint(f\"  Min:    {valid_df['compression_ratio'].min():.2f}\")\nprint(f\"  Max:    {valid_df['compression_ratio'].max():.2f}\")\n\n# Correlation\ncorr = valid_df['sentence_word_count'].corr(valid_df['gloss_count'])\nprint(f\"\\nCorrelation (word_count vs gloss_count): {corr:.3f}\")","metadata":{"papermill":{"duration":0.049692,"end_time":"2026-02-09T12:03:31.458013","exception":false,"start_time":"2026-02-09T12:03:31.408321","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"4a10063f","cell_type":"code","source":"# Scatter plot: sentence length vs gloss count\nfig, axes = plt.subplots(1, 2, figsize=(14, 5))\n\naxes[0].scatter(valid_df['sentence_word_count'], valid_df['gloss_count'], alpha=0.3, s=10)\naxes[0].set_xlabel('Sentence Word Count')\naxes[0].set_ylabel('Gloss Count')\naxes[0].set_title(f'Sentence Length vs Gloss Count (r={corr:.3f})')\n\n# Add trend line\nz = np.polyfit(valid_df['sentence_word_count'], valid_df['gloss_count'], 1)\np = np.poly1d(z)\nx_line = np.linspace(valid_df['sentence_word_count'].min(), valid_df['sentence_word_count'].max(), 100)\naxes[0].plot(x_line, p(x_line), 'r--', linewidth=2, label='Trend')\naxes[0].legend()\n\n# Compression ratio distribution\naxes[1].hist(valid_df['compression_ratio'], bins=50, edgecolor='black', alpha=0.7, color='purple')\naxes[1].set_xlabel('Compression Ratio (gloss/word)')\naxes[1].set_ylabel('Frequency')\naxes[1].set_title('Compression Ratio Distribution')\naxes[1].axvline(1.0, color='red', linestyle='--', label='Ratio = 1.0')\naxes[1].axvline(valid_df['compression_ratio'].mean(), color='green', linestyle='--', label=f'Mean: {valid_df[\"compression_ratio\"].mean():.2f}')\naxes[1].legend()\n\nplt.tight_layout()\nplt.show()","metadata":{"papermill":{"duration":0.57259,"end_time":"2026-02-09T12:03:32.062960","exception":false,"start_time":"2026-02-09T12:03:31.490370","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"ad659720","cell_type":"markdown","source":"## 4. Motion Data Analysis\n\n### 4.1 Motion Feature Files","metadata":{"papermill":{"duration":0.033582,"end_time":"2026-02-09T12:03:32.130225","exception":false,"start_time":"2026-02-09T12:03:32.096643","status":"completed"},"tags":[]}},{"id":"d2a68703","cell_type":"code","source":"# Load motion feature statistics\nmotion_stats = []\n\nif MOTION_FEATS_DIR.exists():\n    npy_files = list(MOTION_FEATS_DIR.glob(\"*.npy\"))\n    print(f\"Found {len(npy_files)} motion feature files\")\n    \n    for npy_file in tqdm(npy_files, desc=\"Analyzing motion files\"):\n        try:\n            motion = np.load(npy_file, mmap_mode='r')\n            motion_stats.append({\n                'sample_id': npy_file.stem,\n                'num_frames': len(motion),\n                'feature_dim': motion.shape[1] if len(motion.shape) > 1 else 1,\n            })\n        except Exception as e:\n            print(f\"Error loading {npy_file}: {e}\")\n            continue\n    \n    motion_df = pd.DataFrame(motion_stats)\n    print(f\"Loaded stats for {len(motion_df)} motion files\")\nelse:\n    motion_df = pd.DataFrame()\n    print(\"Motion features directory not found\")","metadata":{"papermill":{"duration":150.723677,"end_time":"2026-02-09T12:06:02.889239","exception":false,"start_time":"2026-02-09T12:03:32.165562","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"6ec47202","cell_type":"code","source":"# Motion statistics\nif len(motion_df) > 0:\n    FRAME_RATE = 24  # fps\n    motion_df['duration_sec'] = motion_df['num_frames'] / FRAME_RATE\n    \n    total_frames = motion_df['num_frames'].sum()\n    total_duration_sec = motion_df['duration_sec'].sum()\n    \n    print(\"MOTION STATISTICS\")\n    print(\"-\" * 40)\n    print(f\"Total motion files: {len(motion_df):,}\")\n    print(f\"Feature dimension:  {motion_df['feature_dim'].iloc[0]}\")\n    print(\"-\" * 40)\n    \n    print(f\"TOTAL DATASET SIZE:\")\n    print(f\"  Total Frames:     {total_frames:,}\")\n    print(f\"  Total Duration:   {total_duration_sec/3600:.2f} hours \" \n          f\"({total_duration_sec/60:.1f} minutes)\")\n    print(\"-\" * 40)\n    \n    print(f\"FRAME COUNT PER FILE:\")\n    print(f\"  Mean:    {motion_df['num_frames'].mean():.1f}\")\n    print(f\"  Median:  {motion_df['num_frames'].median():.1f}\")\n    print(f\"  Min:     {motion_df['num_frames'].min()}\")\n    print(f\"  Max:     {motion_df['num_frames'].max()}\")\n    print()\n    \n    print(f\"DURATION PER FILE (seconds):\")\n    print(f\"  Mean:    {motion_df['duration_sec'].mean():.2f}s\")\n    print(f\"  Median:  {motion_df['duration_sec'].median():.2f}s\")\n    print(f\"  Min:     {motion_df['duration_sec'].min():.2f}s\")\n    print(f\"  Max:     {motion_df['duration_sec'].max():.2f}s\")","metadata":{"papermill":{"duration":0.115532,"end_time":"2026-02-09T12:06:03.096533","exception":false,"start_time":"2026-02-09T12:06:02.981001","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"8beafda4","cell_type":"code","source":"# Motion duration distribution\nif len(motion_df) > 0:\n    fig, axes = plt.subplots(1, 2, figsize=(14, 5))\n    \n    axes[0].hist(motion_df['num_frames'], bins=50, edgecolor='black', alpha=0.7, color='coral')\n    axes[0].set_xlabel('Number of Frames')\n    axes[0].set_ylabel('Frequency')\n    axes[0].set_title('Motion Length Distribution (Frames)')\n    axes[0].axvline(motion_df['num_frames'].mean(), color='red', linestyle='--', label=f'Mean: {motion_df[\"num_frames\"].mean():.0f}')\n    axes[0].legend()\n    \n    axes[1].hist(motion_df['duration_sec'], bins=50, edgecolor='black', alpha=0.7, color='coral')\n    axes[1].set_xlabel('Duration (seconds)')\n    axes[1].set_ylabel('Frequency')\n    axes[1].set_title('Motion Duration Distribution')\n    axes[1].axvline(motion_df['duration_sec'].mean(), color='red', linestyle='--', label=f'Mean: {motion_df[\"duration_sec\"].mean():.1f}s')\n    axes[1].legend()\n    \n    plt.tight_layout()\n    plt.show()","metadata":{"papermill":{"duration":0.579529,"end_time":"2026-02-09T12:06:03.765698","exception":false,"start_time":"2026-02-09T12:06:03.186169","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"d29cb4e2","cell_type":"markdown","source":"### 4.2 Normalization Statistics","metadata":{"papermill":{"duration":0.255622,"end_time":"2026-02-09T12:06:04.118791","exception":false,"start_time":"2026-02-09T12:06:03.863169","status":"completed"},"tags":[]}},{"id":"639f1353","cell_type":"code","source":"import numpy as np\n\n# 1. Skip if we already have the result\nif NORM_PATH.exists():\n    norm_data = np.load(NORM_PATH)\n    mean, std = norm_data['mean'], norm_data['std']\n    print(f\"✅ Normalization stats loaded from: {NORM_PATH}\")\nelse:\n    print(\"⚠️ Normalization file not found. Calculating from data (this requires reading the files)...\")\n    \n    sum_feats = None\n    sum_sq_feats = None\n    total_frames = 0\n\n    # 2. Iterate through the files you already found\n    for npy_file in tqdm(npy_files, desc=\"Computing Mean/Std\"):\n        # WE DO NOT USE mmap_mode HERE. We need the real values.\n        data = np.load(npy_file).astype(np.float64) \n        \n        if sum_feats is None:\n            sum_feats = np.zeros(data.shape[1])\n            sum_sq_feats = np.zeros(data.shape[1])\n            \n        sum_feats += data.sum(axis=0)\n        sum_sq_feats += (data**2).sum(axis=0)\n        total_frames += data.shape[0]\n\n    # 3. Final Math\n    mean = sum_feats / total_frames\n    variance = (sum_sq_feats / total_frames) - (mean**2)\n    std = np.sqrt(np.maximum(variance, 1e-8))\n    \n    # Precision guard for static joints\n    std[std < 1e-6] = 1.0\n\n    # 4. Save so you never have to do this 2-minute wait again\n    SAVE_PATH = Path(\"/kaggle/working/normalization.npz\")\n    np.savez(SAVE_PATH, mean=mean, std=std)\n    print(f\"Normalization stats saved to {SAVE_PATH}\")","metadata":{"papermill":{"duration":200.516159,"end_time":"2026-02-09T12:09:24.726006","exception":false,"start_time":"2026-02-09T12:06:04.209847","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"dd81e9c7","cell_type":"code","source":"# Visualize normalization statistics\n\nfig, axes = plt.subplots(2, 2, figsize=(14, 10))\n\n# Mean distribution\naxes[0, 0].bar(range(len(mean)), mean, width=1.0, alpha=0.7)\naxes[0, 0].set_xlabel('Feature Index')\naxes[0, 0].set_ylabel('Mean')\naxes[0, 0].set_title('Feature Mean Values')\n\n# Std distribution\naxes[0, 1].bar(range(len(std)), std, width=1.0, alpha=0.7, color='orange')\naxes[0, 1].set_xlabel('Feature Index')\naxes[0, 1].set_ylabel('Std')\naxes[0, 1].set_title('Feature Standard Deviation')\n\n# Mean histogram\naxes[1, 0].hist(mean, bins=50, edgecolor='black', alpha=0.7)\naxes[1, 0].set_xlabel('Mean Value')\naxes[1, 0].set_ylabel('Frequency')\naxes[1, 0].set_title('Distribution of Feature Means')\n\n# Std histogram\naxes[1, 1].hist(std, bins=50, edgecolor='black', alpha=0.7, color='orange')\naxes[1, 1].set_xlabel('Std Value')\naxes[1, 1].set_ylabel('Frequency')\naxes[1, 1].set_title('Distribution of Feature Stds')\n\nplt.tight_layout()\nplt.show()","metadata":{"papermill":{"duration":2.494015,"end_time":"2026-02-09T12:09:27.390514","exception":false,"start_time":"2026-02-09T12:09:24.896499","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"98de68d5","cell_type":"markdown","source":"### 4.3 Text-Motion Relationship","metadata":{"papermill":{"duration":0.162071,"end_time":"2026-02-09T12:09:27.722689","exception":false,"start_time":"2026-02-09T12:09:27.560618","status":"completed"},"tags":[]}},{"id":"02e5a418","cell_type":"code","source":"# Merge motion data with text data\nif len(motion_df) > 0:\n    merged_df = valid_df.merge(motion_df, on='sample_id', how='inner')\n    print(f\"Matched {len(merged_df)} samples with both text and motion data\")\n    \n    # Correlations\n    corr_word_frames = merged_df['sentence_word_count'].corr(merged_df['num_frames'])\n    corr_gloss_frames = merged_df['gloss_count'].corr(merged_df['num_frames'])\n    \n    print(f\"\\nCorrelations:\")\n    print(f\"  Word count vs Frames: {corr_word_frames:.3f}\")\n    print(f\"  Gloss count vs Frames: {corr_gloss_frames:.3f}\")\nelse:\n    merged_df = pd.DataFrame()\n    print(\"No motion data available for merging\")","metadata":{"papermill":{"duration":0.205308,"end_time":"2026-02-09T12:09:28.090448","exception":false,"start_time":"2026-02-09T12:09:27.885140","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"b85955b1","cell_type":"code","source":"# Text-motion relationship plots\nif len(merged_df) > 0:\n    fig, axes = plt.subplots(1, 2, figsize=(14, 5))\n    \n    # Gloss count vs frames (should be stronger correlation)\n    axes[0].scatter(merged_df['gloss_count'], merged_df['num_frames'], alpha=0.3, s=10)\n    axes[0].set_xlabel('Gloss Count')\n    axes[0].set_ylabel('Number of Frames')\n    axes[0].set_title(f'Gloss Count vs Motion Length (r={corr_gloss_frames:.3f})')\n    \n    # Add trend line\n    z = np.polyfit(merged_df['gloss_count'], merged_df['num_frames'], 1)\n    p = np.poly1d(z)\n    x_line = np.linspace(merged_df['gloss_count'].min(), merged_df['gloss_count'].max(), 100)\n    axes[0].plot(x_line, p(x_line), 'r--', linewidth=2, label='Trend')\n    axes[0].legend()\n    \n    # Word count vs frames\n    axes[1].scatter(merged_df['sentence_word_count'], merged_df['num_frames'], alpha=0.3, s=10, color='orange')\n    axes[1].set_xlabel('Sentence Word Count')\n    axes[1].set_ylabel('Number of Frames')\n    axes[1].set_title(f'Word Count vs Motion Length (r={corr_word_frames:.3f})')\n    \n    z = np.polyfit(merged_df['sentence_word_count'], merged_df['num_frames'], 1)\n    p = np.poly1d(z)\n    x_line = np.linspace(merged_df['sentence_word_count'].min(), merged_df['sentence_word_count'].max(), 100)\n    axes[1].plot(x_line, p(x_line), 'r--', linewidth=2, label='Trend')\n    axes[1].legend()\n    \n    plt.tight_layout()\n    plt.show()","metadata":{"papermill":{"duration":0.669381,"end_time":"2026-02-09T12:09:28.928123","exception":false,"start_time":"2026-02-09T12:09:28.258742","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"b3263855","cell_type":"code","source":"# Motion anomalies\nif len(motion_df) > 0:\n    print(\"\\nMOTION ANOMALIES\")\n    print(\"-\" * 40)\n    \n    # Very short motions\n    short_motions = motion_df[motion_df['num_frames'] < 24]\n    print(f\"6. VERY SHORT MOTIONS (<24 frames)\")\n    print(f\"   Count: {len(short_motions)}\")\n    \n    # Very long motions\n    q99 = motion_df['num_frames'].quantile(0.99)\n    long_motions = motion_df[motion_df['num_frames'] > q99]\n    print(f\"\\n7. VERY LONG MOTIONS (>99th percentile: {q99:.0f} frames)\")\n    print(f\"   Count: {len(long_motions)}\")\n    \n    # Outlier detection using IQR\n    Q1 = motion_df['num_frames'].quantile(0.25)\n    Q3 = motion_df['num_frames'].quantile(0.75)\n    IQR = Q3 - Q1\n    outliers = motion_df[(motion_df['num_frames'] < Q1 - 1.5*IQR) | (motion_df['num_frames'] > Q3 + 1.5*IQR)]\n    print(f\"\\n8. FRAME COUNT OUTLIERS (IQR method)\")\n    print(f\"   Q1: {Q1:.0f}, Q3: {Q3:.0f}, IQR: {IQR:.0f}\")\n    print(f\"   Outlier count: {len(outliers)}\")","metadata":{"papermill":{"duration":0.338017,"end_time":"2026-02-09T12:09:29.434867","exception":false,"start_time":"2026-02-09T12:09:29.096850","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"cac15eeb","cell_type":"markdown","source":"## 6. Sample Examples","metadata":{"papermill":{"duration":0.164994,"end_time":"2026-02-09T12:09:29.765481","exception":false,"start_time":"2026-02-09T12:09:29.600487","status":"completed"},"tags":[]}},{"id":"0a02d5d9","cell_type":"code","source":"# Display sample examples\nprint(\"SAMPLE EXAMPLES\")\nprint(\"=\" * 50)\n\n# Random samples\nsample_examples = valid_df.sample(5, random_state=42)\n\nfor idx, row in sample_examples.iterrows():\n    print(f\"\\nSample ID: {row['sample_id']}\")\n    print(f\"  Sentence: {row['sentence']}\")\n    print(f\"  Gloss:    {row['gloss']}\")\n    print(f\"  Words: {row['sentence_word_count']}, Glosses: {row['gloss_count']}, BVH files: {row['bvh_count']}\")\n    print(f\"  Fingerspelling letters: {row['fingerspell_count']}\")","metadata":{"papermill":{"duration":0.177343,"end_time":"2026-02-09T12:09:30.112287","exception":false,"start_time":"2026-02-09T12:09:29.934944","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"ecf248cc","cell_type":"code","source":"# Analyze a sample motion file in detail\nif MOTION_FEATS_DIR.exists():\n    sample_npy = list(MOTION_FEATS_DIR.glob(\"*.npy\"))[:1]\n    if sample_npy:\n        motion = np.load(sample_npy[0])\n        \n        print(\"\\nSAMPLE MOTION FILE ANALYSIS\")\n        print(\"-\" * 40)\n        print(f\"File: {sample_npy[0].name}\")\n        print(f\"Shape: {motion.shape}\")\n        print(f\"Dtype: {motion.dtype}\")\n        print()\n        print(f\"Value range: [{motion.min():.4f}, {motion.max():.4f}]\")\n        print(f\"Mean: {motion.mean():.4f}\")\n        print(f\"Std:  {motion.std():.4f}\")\n        \n        # Per-feature statistics\n        print(f\"\\nPer-feature statistics (first 10 features):\")\n        for i in range(min(10, motion.shape[1])):\n            print(f\"  Feature {i:3d}: mean={motion[:, i].mean():8.4f}, std={motion[:, i].std():8.4f}\")","metadata":{"papermill":{"duration":0.415173,"end_time":"2026-02-09T12:09:30.697774","exception":false,"start_time":"2026-02-09T12:09:30.282601","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"0a40eaac-8fd1-4ee5-9d2e-31b1c9c8a8dc","cell_type":"markdown","source":"# Motion Visualization","metadata":{}},{"id":"1943a1a0-3caf-4795-847a-71dcb340cb89","cell_type":"code","source":"!pip uninstall kiseki -y \n!pip install git+https://github.com/1997MarsRover/kiseki.git","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-11T10:58:04.578390Z","iopub.execute_input":"2026-02-11T10:58:04.578742Z","iopub.status.idle":"2026-02-11T10:58:20.296495Z","shell.execute_reply.started":"2026-02-11T10:58:04.578711Z","shell.execute_reply":"2026-02-11T10:58:20.295191Z"}},"outputs":[],"execution_count":null},{"id":"cf89f3c5-e3f3-409a-8d3c-8ed189c32c7c","cell_type":"code","source":"from kiseki import visualize\n\n\n# Focus on hands with front view\nvisualize(\"/kaggle/input/competitions/motion-s-hierarchical-text-to-motion-generation-for-sign-language/Motion-Features/1000648.npy\",\n          focus_joints='both_hands',\n          fixed_view='front',\n          fps=60, display=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-11T10:58:22.850611Z","iopub.execute_input":"2026-02-11T10:58:22.851048Z","iopub.status.idle":"2026-02-11T10:58:46.716731Z","shell.execute_reply.started":"2026-02-11T10:58:22.851008Z","shell.execute_reply":"2026-02-11T10:58:46.715744Z"}},"outputs":[],"execution_count":null},{"id":"4cb2e4a6-5f11-4c0f-ac2e-79ea1956f51c","cell_type":"markdown","source":"# Compare Motions","metadata":{}},{"id":"d9253e8e-0a87-4145-bced-6fbec3ffb174","cell_type":"code","source":"# Compare Motions\nfrom kiseki import compare\n\n\nm_a = \"/kaggle/input/competitions/motion-s-hierarchical-text-to-motion-generation-for-sign-language/Motion-Features/1005354.npy\"\nm_b = \"/kaggle/input/competitions/motion-s-hierarchical-text-to-motion-generation-for-sign-language/Motion-Features/1000648.npy\"\n# Overlay -- both skeletons on the same axes\ncompare(m_a,\n        m_b\", \n        mode=\"overlay\", display=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-11T11:01:10.270668Z","iopub.execute_input":"2026-02-11T11:01:10.271099Z","iopub.status.idle":"2026-02-11T11:01:45.952141Z","shell.execute_reply.started":"2026-02-11T11:01:10.271057Z","shell.execute_reply":"2026-02-11T11:01:45.951145Z"}},"outputs":[],"execution_count":null},{"id":"75b8b8c6","cell_type":"markdown","source":"## 7. Summary Statistics","metadata":{"papermill":{"duration":0.162216,"end_time":"2026-02-09T12:09:31.023823","exception":false,"start_time":"2026-02-09T12:09:30.861607","status":"completed"},"tags":[]}},{"id":"7c26c54e","cell_type":"code","source":"# Summary table\nsummary_data = {\n    'Metric': [\n        'Total Samples',\n        'Valid Samples (with metadata)',\n        'Total BVH Files',\n        'Unique Words (vocabulary)',\n        'Unique Glosses',\n        'Avg Sentence Length (words)',\n        'Avg Gloss Count',\n        'Fingerspelling Rate',\n    ],\n    'Value': [\n        f\"{len(df):,}\",\n        f\"{len(valid_df):,}\",\n        f\"{df['bvh_count'].sum():,}\",\n        f\"{len(word_counts):,}\",\n        f\"{len(gloss_counts):,}\",\n        f\"{valid_df['sentence_word_count'].mean():.1f}\",\n        f\"{valid_df['gloss_count'].mean():.1f}\",\n        f\"{100*len(fingerspell_glosses)/len(all_glosses):.1f}%\",\n    ]\n}\n\nif len(motion_df) > 0:\n    summary_data['Metric'].extend([\n        'Motion Files',\n        'Feature Dimension',\n        'Avg Motion Length (frames)',\n        'Avg Duration (seconds)',\n    ])\n    summary_data['Value'].extend([\n        f\"{len(motion_df):,}\",\n        f\"{motion_df['feature_dim'].iloc[0]}\",\n        f\"{motion_df['num_frames'].mean():.0f}\",\n        f\"{motion_df['duration_sec'].mean():.2f}s\",\n    ])\n\nsummary_df = pd.DataFrame(summary_data)\nprint(\"DATASET SUMMARY\")\nprint(\"=\" * 50)\nprint(summary_df.to_string(index=False))","metadata":{"papermill":{"duration":0.191056,"end_time":"2026-02-09T12:09:31.378936","exception":false,"start_time":"2026-02-09T12:09:31.187880","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"1a2459ea","cell_type":"code","source":"# Correlation heatmap for merged data\nif len(merged_df) > 0:\n    corr_cols = ['sentence_word_count', 'sentence_char_count', 'gloss_count', \n                 'fingerspell_count', 'bvh_count', 'num_frames']\n    corr_matrix = merged_df[corr_cols].corr()\n    \n    plt.figure(figsize=(10, 8))\n    sns.heatmap(corr_matrix, annot=True, cmap='coolwarm', center=0, \n                fmt='.2f', square=True, linewidths=0.5)\n    plt.title('Feature Correlation Matrix')\n    plt.tight_layout()\n    plt.show()","metadata":{"papermill":{"duration":0.459321,"end_time":"2026-02-09T12:09:32.007978","exception":false,"start_time":"2026-02-09T12:09:31.548657","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"66cb821b","cell_type":"code","source":"# Export analyzed data for future use (optional)\nexport_cols = ['sample_id', 'sentence', 'gloss', 'gloss_count', 'sentence_word_count', \n               'sentence_char_count', 'fingerspell_count', 'bvh_count']\n\nif len(merged_df) > 0:\n    export_df = merged_df[export_cols + ['num_frames', 'duration_sec']]\nelse:\n    export_df = valid_df[export_cols]\n\n# Uncomment to export\nexport_df.to_csv('dataset_analysis.csv', index=False)\n# print(f\"Exported {len(export_df)} samples to dataset_analysis.csv\")\n\nprint(f\"DataFrame ready for export: {len(export_df)} samples, {len(export_df.columns)} columns\")\nexport_df.head()","metadata":{"papermill":{"duration":0.276416,"end_time":"2026-02-09T12:09:32.450470","exception":false,"start_time":"2026-02-09T12:09:32.174054","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null}]}