{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# %% [markdown]\n# # American Sign Language (ASL) Translation from English Sentences\n# \n# This notebook builds a neural machine translation system to convert English sentences into ASL gloss sequences using the provided dataset (12,000+ examples). It includes:\n# - Data loading and exploration\n# - Vocabulary building and preprocessing\n# - Sequence-to-sequence model with attention\n# - Training and validation\n# - BLEU score evaluation\n# \n# **Dataset columns:**\n# - `id`: Unique identifier\n# - `sentence`: English sentence\n# - `gloss`: ASL gloss representation (tokens in uppercase, ends with `//`)\n# - `bvh_path`: Path to motion capture file (not used here)\n# - `base_tokens` and `residual_1` to `residual_5`: numeric motion tokens (optional future use)\n# \n# We focus on text‑based translation (English → gloss).\n\n# %% [markdown]\n# ## 1. Setup and Imports\n\n# %%\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom collections import Counter\nimport re\nimport random\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\n\nfrom sklearn.model_selection import train_test_split\n\nimport nltk\nfrom nltk.translate.bleu_score import sentence_bleu, SmoothingFunction\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Set random seeds for reproducibility\ndef set_seed(seed=42):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\nset_seed(42)\n\n# Check for GPU\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"Using device: {device}\")\n\n# %% [markdown]\n# ## 2. Load the Data\n\n# %%\n# Adjust path according to your Kaggle dataset location\ndf = pd.read_csv('/kaggle/input/asl-data/train.csv')   # example path\nprint(f\"Dataset shape: {df.shape}\")\ndf.head()\n\n# %% [markdown]\n# ## 3. Exploratory Data Analysis (EDA)\n\n# %%\n# Basic info\ndf.info()\n\n# %%\n# Check for missing values\ndf.isnull().sum()\n\n# %%\n# Sentence lengths (in words)\ndf['sentence_word_count'] = df['sentence'].astype(str).apply(lambda x: len(x.split()))\ndf['gloss_token_count'] = df['gloss'].astype(str).apply(lambda x: len(x.replace('//', '').split()))\n\nfig, axes = plt.subplots(1, 2, figsize=(12,4))\nsns.histplot(df['sentence_word_count'], bins=30, ax=axes[0])\naxes[0].set_title('English sentence word count')\nsns.histplot(df['gloss_token_count'], bins=30, ax=axes[1])\naxes[1].set_title('ASL gloss token count')\nplt.tight_layout()\nplt.show()\n\n# %%\n# Most common words in English sentences\nall_words = ' '.join(df['sentence'].astype(str)).split()\nword_counts = Counter(all_words)\ncommon_words = word_counts.most_common(20)\nplt.figure(figsize=(10,6))\nsns.barplot(x=[w[0] for w in common_words], y=[w[1] for w in common_words])\nplt.xticks(rotation=45)\nplt.title('Top 20 words in English sentences')\nplt.show()\n\n# %%\n# Most common gloss tokens\nall_gloss = ' '.join(df['gloss'].astype(str).str.replace('//', '').str.split()).split()\ngloss_counts = Counter(all_gloss)\ncommon_gloss = gloss_counts.most_common(20)\nplt.figure(figsize=(10,6))\nsns.barplot(x=[g[0] for g in common_gloss], y=[g[1] for g in common_gloss])\nplt.xticks(rotation=45)\nplt.title('Top 20 gloss tokens')\nplt.show()\n\n# %%\n# Length correlation\nplt.figure(figsize=(6,6))\nsns.scatterplot(x=df['sentence_word_count'], y=df['gloss_token_count'])\nplt.xlabel('English length')\nplt.ylabel('Gloss length')\nplt.title('English vs Gloss length')\nplt.show()\n\n# %% [markdown]\n# ## 4. Preprocessing\n\n# %%\n# Clean gloss: remove trailing '//' and extra spaces, split into tokens\ndef clean_gloss(gloss_str):\n    gloss_str = str(gloss_str).replace('//', '').strip()\n    return gloss_str.split()\n\ndf['gloss_tokens'] = df['gloss'].apply(clean_gloss)\ndf['sentence_tokens'] = df['sentence'].astype(str).apply(lambda x: x.split())\n\n# %%\n# Build vocabularies\n# Special tokens\nPAD_TOKEN = '<PAD>'\nSOS_TOKEN = '<SOS>'\nEOS_TOKEN = '<EOS>'\nUNK_TOKEN = '<UNK>'\n\ndef build_vocab(token_lists, max_size=None, min_freq=1):\n    counter = Counter()\n    for tokens in token_lists:\n        counter.update(tokens)\n    if max_size:\n        most_common = counter.most_common(max_size)\n    else:\n        most_common = counter.items()\n    # Keep tokens with frequency >= min_freq\n    vocab = [token for token, freq in most_common if freq >= min_freq]\n    # Add special tokens\n    specials = [PAD_TOKEN, SOS_TOKEN, EOS_TOKEN, UNK_TOKEN]\n    for tok in specials:\n        if tok not in vocab:\n            vocab.insert(0, tok)   # PAD will be first\n    # Reorder to ensure PAD is 0\n    vocab = [PAD_TOKEN] + [t for t in vocab if t != PAD_TOKEN]\n    stoi = {tok:i for i,tok in enumerate(vocab)}\n    itos = {i:tok for i,tok in enumerate(vocab)}\n    return stoi, itos, vocab\n\n# Build English vocab (max 5000, min_freq=2)\nsrc_stoi, src_itos, src_vocab = build_vocab(df['sentence_tokens'], max_size=5000, min_freq=2)\n# Build gloss vocab (max 2000, min_freq=2)\ntgt_stoi, tgt_itos, tgt_vocab = build_vocab(df['gloss_tokens'], max_size=2000, min_freq=2)\n\nprint(f\"English vocab size: {len(src_vocab)}\")\nprint(f\"Gloss vocab size: {len(tgt_vocab)}\")\n\n# %%\n# Convert tokens to indices\ndef tokens_to_indices(tokens, stoi):\n    return [stoi.get(tok, stoi[UNK_TOKEN]) for tok in tokens]\n\ndf['src_indices'] = df['sentence_tokens'].apply(lambda x: tokens_to_indices(x, src_stoi))\ndf['tgt_indices'] = df['gloss_tokens'].apply(lambda x: tokens_to_indices(x, tgt_stoi))\n\n# %%\n# Train/validation split\ntrain_df, val_df = train_test_split(df, test_size=0.1, random_state=42)\nprint(f\"Train samples: {len(train_df)}, Val samples: {len(val_df)}\")\n\n# %%\n# Create PyTorch Dataset\nclass ASLDataset(Dataset):\n    def __init__(self, df, src_stoi, tgt_stoi, src_max_len=None, tgt_max_len=None):\n        self.df = df\n        self.src_stoi = src_stoi\n        self.tgt_stoi = tgt_stoi\n        # Determine max lengths if not given\n        if src_max_len is None:\n            self.src_max_len = max(df['src_indices'].apply(len))\n        else:\n            self.src_max_len = src_max_len\n        if tgt_max_len is None:\n            self.tgt_max_len = max(df['tgt_indices'].apply(len))\n        else:\n            self.tgt_max_len = tgt_max_len\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        src = self.df.iloc[idx]['src_indices']\n        tgt = self.df.iloc[idx]['tgt_indices']\n        # Add SOS and EOS to target\n        tgt_in = [self.tgt_stoi[SOS_TOKEN]] + tgt\n        tgt_out = tgt + [self.tgt_stoi[EOS_TOKEN]]\n        # Pad sequences\n        src_pad = src + [self.src_stoi[PAD_TOKEN]] * (self.src_max_len - len(src))\n        tgt_in_pad = tgt_in + [self.tgt_stoi[PAD_TOKEN]] * (self.tgt_max_len + 1 - len(tgt_in))\n        tgt_out_pad = tgt_out + [self.tgt_stoi[PAD_TOKEN]] * (self.tgt_max_len + 1 - len(tgt_out))\n        return {\n            'src': torch.tensor(src_pad, dtype=torch.long),\n            'tgt_in': torch.tensor(tgt_in_pad, dtype=torch.long),\n            'tgt_out': torch.tensor(tgt_out_pad, dtype=torch.long),\n            'src_len': len(src),\n            'tgt_len': len(tgt)\n        }\n\n# Compute max lengths from training set\nsrc_max_len_train = max(train_df['src_indices'].apply(len))\ntgt_max_len_train = max(train_df['tgt_indices'].apply(len))\nprint(f\"Max source length: {src_max_len_train}, Max target length: {tgt_max_len_train}\")\n\ntrain_dataset = ASLDataset(train_df, src_stoi, tgt_stoi, src_max_len_train, tgt_max_len_train)\nval_dataset = ASLDataset(val_df, src_stoi, tgt_stoi, src_max_len_train, tgt_max_len_train)\n\nbatch_size = 64   # adjust based on GPU memory\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, collate_fn=lambda x: x)\nval_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, collate_fn=lambda x: x)\n\n# %% [markdown]\n# ## 5. Model Definition (Encoder-Decoder with Attention)\n\n# %%\nclass Encoder(nn.Module):\n    def __init__(self, vocab_size, embed_size, hidden_size, num_layers=1, dropout=0.2):\n        super().__init__()\n        self.embedding = nn.Embedding(vocab_size, embed_size, padding_idx=0)\n        self.lstm = nn.LSTM(embed_size, hidden_size, num_layers, batch_first=True, dropout=dropout)\n        self.dropout = nn.Dropout(dropout)\n\n    def forward(self, x, lengths):\n        embedded = self.dropout(self.embedding(x))\n        packed = nn.utils.rnn.pack_padded_sequence(embedded, lengths.cpu(), batch_first=True, enforce_sorted=False)\n        outputs, (hidden, cell) = self.lstm(packed)\n        outputs, _ = nn.utils.rnn.pad_packed_sequence(outputs, batch_first=True)\n        return outputs, hidden, cell\n\nclass Decoder(nn.Module):\n    def __init__(self, vocab_size, embed_size, hidden_size, num_layers=1, dropout=0.2):\n        super().__init__()\n        self.embedding = nn.Embedding(vocab_size, embed_size, padding_idx=0)\n        self.lstm = nn.LSTM(embed_size + hidden_size, hidden_size, num_layers, batch_first=True, dropout=dropout)\n        self.fc = nn.Linear(hidden_size, vocab_size)\n        self.dropout = nn.Dropout(dropout)\n\n    def forward(self, x, hidden, cell, encoder_outputs):\n        # x shape: (batch_size, 1)  (previous token)\n        embedded = self.dropout(self.embedding(x))\n        # Simple attention: use last encoder hidden as context (can be improved)\n        context = hidden[-1].unsqueeze(1)   # (batch_size, 1, hidden_size)\n        rnn_input = torch.cat((embedded, context), dim=2)\n        output, (hidden, cell) = self.lstm(rnn_input, (hidden, cell))\n        prediction = self.fc(output.squeeze(1))\n        return prediction, hidden, cell\n\nclass Seq2Seq(nn.Module):\n    def __init__(self, encoder, decoder):\n        super().__init__()\n        self.encoder = encoder\n        self.decoder = decoder\n\n    def forward(self, src, tgt_in, src_lengths, teacher_forcing_ratio=0.5):\n        batch_size = src.size(0)\n        tgt_len = tgt_in.size(1)\n        tgt_vocab_size = self.decoder.fc.out_features\n\n        encoder_outputs, hidden, cell = self.encoder(src, src_lengths)\n\n        decoder_input = tgt_in[:, 0].unsqueeze(1)  # start with <SOS>\n        outputs = torch.zeros(batch_size, tgt_len, tgt_vocab_size).to(device)\n\n        for t in range(1, tgt_len):\n            output, hidden, cell = self.decoder(decoder_input, hidden, cell, encoder_outputs)\n            outputs[:, t, :] = output\n            teacher_force = random.random() < teacher_forcing_ratio\n            top1 = output.argmax(1)\n            decoder_input = tgt_in[:, t].unsqueeze(1) if teacher_force else top1.unsqueeze(1)\n        return outputs\n\n# Hyperparameters\nembed_size = 256\nhidden_size = 512\nnum_layers = 2\ndropout = 0.3\n\nencoder = Encoder(len(src_vocab), embed_size, hidden_size, num_layers, dropout).to(device)\ndecoder = Decoder(len(tgt_vocab), embed_size, hidden_size, num_layers, dropout).to(device)\nmodel = Seq2Seq(encoder, decoder).to(device)\n\nprint(model)\n\n# %% [markdown]\n# ## 6. Training Setup\n\n# %%\ncriterion = nn.CrossEntropyLoss(ignore_index=tgt_stoi[PAD_TOKEN])\noptimizer = optim.Adam(model.parameters(), lr=0.001)\nscheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=3)\n\n# %%\ndef collate_batch(batch):\n    src = torch.stack([item['src'] for item in batch])\n    tgt_in = torch.stack([item['tgt_in'] for item in batch])\n    tgt_out = torch.stack([item['tgt_out'] for item in batch])\n    src_lengths = torch.tensor([item['src_len'] for item in batch], dtype=torch.long)\n    tgt_lengths = torch.tensor([item['tgt_len'] for item in batch], dtype=torch.long)\n    return src, tgt_in, tgt_out, src_lengths, tgt_lengths\n\n# %% [markdown]\n# ## 7. Training Loop\n\n# %%\nnum_epochs = 40   # can adjust based on convergence\ntrain_losses = []\nval_losses = []\n\nfor epoch in range(num_epochs):\n    model.train()\n    total_loss = 0\n    for batch in train_loader:\n        src, tgt_in, tgt_out, src_lengths, _ = collate_batch(batch)\n        src, tgt_in, tgt_out = src.to(device), tgt_in.to(device), tgt_out.to(device)\n        src_lengths = src_lengths.to('cpu')   # LSTM pack expects CPU lengths\n\n        optimizer.zero_grad()\n        output = model(src, tgt_in, src_lengths)   # (batch, tgt_len, vocab)\n        loss = criterion(output.reshape(-1, output.shape[-1]), tgt_out.reshape(-1))\n        loss.backward()\n        torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1)\n        optimizer.step()\n        total_loss += loss.item()\n\n    avg_train_loss = total_loss / len(train_loader)\n    train_losses.append(avg_train_loss)\n\n    # Validation\n    model.eval()\n    val_loss = 0\n    with torch.no_grad():\n        for batch in val_loader:\n            src, tgt_in, tgt_out, src_lengths, _ = collate_batch(batch)\n            src, tgt_in, tgt_out = src.to(device), tgt_in.to(device), tgt_out.to(device)\n            src_lengths = src_lengths.to('cpu')\n            output = model(src, tgt_in, src_lengths, teacher_forcing_ratio=0)   # greedy\n            loss = criterion(output.reshape(-1, output.shape[-1]), tgt_out.reshape(-1))\n            val_loss += loss.item()\n    avg_val_loss = val_loss / len(val_loader)\n    val_losses.append(avg_val_loss)\n\n    scheduler.step(avg_val_loss)\n\n    print(f\"Epoch {epoch+1}/{num_epochs} - Train Loss: {avg_train_loss:.4f} - Val Loss: {avg_val_loss:.4f}\")\n\n# %%\n# Plot losses\nplt.plot(train_losses, label='Train')\nplt.plot(val_losses, label='Validation')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.legend()\nplt.title('Training History')\nplt.show()\n\n# %% [markdown]\n# ## 8. Evaluation with BLEU Score\n\n# %%\ndef translate_sentence(model, sentence, src_stoi, tgt_itos, max_len=20):\n    model.eval()\n    tokens = sentence.split()\n    indices = [src_stoi.get(t, src_stoi[UNK_TOKEN]) for t in tokens]\n    src_tensor = torch.tensor([indices]).to(device)\n    src_len = torch.tensor([len(indices)])\n\n    encoder_outputs, hidden, cell = model.encoder(src_tensor, src_len)\n\n    decoder_input = torch.tensor([[tgt_stoi[SOS_TOKEN]]]).to(device)\n    decoded_tokens = []\n    for _ in range(max_len):\n        output, hidden, cell = model.decoder(decoder_input, hidden, cell, encoder_outputs)\n        top1 = output.argmax(1).item()\n        if top1 == tgt_stoi[EOS_TOKEN]:\n            break\n        decoded_tokens.append(tgt_itos[top1])\n        decoder_input = torch.tensor([[top1]]).to(device)\n    return ' '.join(decoded_tokens)\n\n# Compute BLEU on validation set\nsmooth_fn = SmoothingFunction().method1\nbleu_scores = []\nfor i in range(len(val_df)):\n    src_sent = val_df.iloc[i]['sentence']\n    true_gloss = val_df.iloc[i]['gloss_tokens']\n    pred_gloss = translate_sentence(model, src_sent, src_stoi, tgt_itos).split()\n    if len(pred_gloss) == 0:\n        pred_gloss = ['<UNK>']  # avoid empty\n    bleu = sentence_bleu([true_gloss], pred_gloss, smoothing_function=smooth_fn)\n    bleu_scores.append(bleu)\n\nprint(f\"Average BLEU score on validation set: {np.mean(bleu_scores):.4f}\")\n\n# %%\n# Show some examples\nfor i in range(5):\n    idx = random.randint(0, len(val_df)-1)\n    src = val_df.iloc[idx]['sentence']\n    true = ' '.join(val_df.iloc[idx]['gloss_tokens'])\n    pred = translate_sentence(model, src, src_stoi, tgt_itos)\n    print(f\"English: {src}\")\n    print(f\"True Gloss: {true}\")\n    print(f\"Pred Gloss: {pred}\")\n    print(\"-\" * 50)\n\n# %% [markdown]\n# ## 9. Save Model and Vocab\n\n# %%\n# Save model\ntorch.save(model.state_dict(), 'asl_translation_model.pt')\n# Save vocabularies\nimport pickle\nwith open('src_stoi.pkl', 'wb') as f:\n    pickle.dump(src_stoi, f)\nwith open('tgt_itos.pkl', 'wb') as f:\n    pickle.dump(tgt_itos, f)\nprint(\"Model and vocab saved.\")\n\n# %% [markdown]\n# ## 10. Conclusion\n# \n# This notebook demonstrates a complete pipeline for translating English sentences to ASL gloss using a sequence‑to‑sequence model with attention. The model achieves a reasonable BLEU score and can be further improved by:\n# - Incorporating motion tokens for full sign generation.\n# - Using transformer architectures.\n# - Applying data augmentation and pretrained embeddings.\n# \n# The saved model can be used for inference on new English sentences.","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}