{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":19018,"databundleVersionId":2703900,"sourceType":"competition"}],"dockerImageVersionId":30840,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q transformers[torch] datasets nlpaug\nimport os\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\nfrom transformers import (\n    AutoTokenizer,\n    AutoModelForSequenceClassification,\n    Trainer,\n    TrainingArguments\n)\nimport nlpaug.augmenter.word as naw\n\n# Load data\ntrain_df = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv')\ntrain_df = train_df[['comment_text', 'toxic']].dropna()\n\n# Class distribution analysis\ntoxic_df = train_df[train_df['toxic'] == 1]\nnon_toxic_df = train_df[train_df['toxic'] == 0]\n\nprint(f\"Original counts - Toxic: {len(toxic_df)}, Non-toxic: {len(non_toxic_df)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T15:29:12.828561Z","iopub.execute_input":"2025-02-01T15:29:12.828832Z","execution_failed":"2025-02-01T15:29:17.500Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Original counts\ntoxic_count = len(toxic_df)\nnon_toxic_count = len(non_toxic_df)\nrequired_toxic = non_toxic_count - toxic_count\n\naug = naw.ContextualWordEmbsAug(\n    model_path='bert-base-uncased',\n    action=\"substitute\",\n    aug_max=10,\n    aug_p=0.6,\n    batch_size=256,\n    device='cuda'\n)\n\n# Batch processing function\ndef batch_augment(texts, aug, num_variants):\n    \"\"\"Generate multiple variants for a batch of texts\"\"\"\n    return [aug.augment(text) for text in texts for _ in range(num_variants)]\n\n# Calculate needed variants per sample\nremaining = required_toxic\naugmented_toxic = []\nbatch_size = 512\n\nfor i in range(0, len(toxic_df), batch_size):\n    batch_texts = toxic_df['comment_text'].iloc[i:i+batch_size].tolist()\n    \n    # Calculate variants needed from this batch\n    variants_needed = min(remaining // (len(toxic_df) // batch_size), 8)\n    variants_needed = max(variants_needed, 1)\n    \n    # Augment batch\n    try:\n        augmented_batch = batch_augment(batch_texts, aug, variants_needed)\n        augmented_toxic.extend(augmented_batch)\n        remaining -= len(augmented_batch)\n        \n        print(f\"Generated {len(augmented_batch)} samples | Remaining: {remaining}\")\n        \n        if remaining <= 0:\n            break\n            \n    except Exception as e:\n        print(f\"Error in batch {i}: {str(e)}\")\n        continue\n\n# Final balanced dataset\nbalanced_toxic = pd.DataFrame({\n    'comment_text': toxic_df['comment_text'].tolist() + augmented_toxic[:required_toxic],\n    'toxic': 1\n})\n\nbalanced_df = pd.concat([balanced_toxic, non_toxic_df]).sample(frac=1, random_state=42)\nbalanced_df['comment_text'] = balanced_df['comment_text'].astype(str)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"balanced_df.to_csv('/kaggle/working/balanced_dataset.csv', index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\ntrain_df = balanced_df\n\nval_df = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\nval_df = val_df[['comment_text', 'toxic']]\n\ntest_df = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')\ntest_df_labels = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test_labels.csv')\n\ntest_df['toxic'] = test_df_labels['toxic']\ntest_df['comment_text'] = test_df['content']\ntest_df = test_df[['comment_text', 'toxic']]\n\nprint(f\"Train size: {len(train_df)}, Val size: {len(val_df)}, Test size: {len(test_df)}\")\n'''","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nfrom transformers import XLMRobertaTokenizer, XLMRobertaForSequenceClassification, Trainer, TrainingArguments\nfrom datasets import Dataset\nimport torch\nfrom sklearn.metrics import roc_auc_score, accuracy_score\n\n# Initialize tokenizer\ntokenizer = XLMRobertaTokenizer.from_pretrained('xlm-roberta-base')\n\n# Tokenization function\ndef tokenize_function(batch):\n    tokenized = tokenizer(\n        batch['comment_text'],\n        padding='max_length',\n        truncation=True,\n        max_length=128\n    )\n\n    tokenized['labels'] = batch['toxic']\n    return tokenized\n\n# Convert to HuggingFace datasets\ntrain_dataset = Dataset.from_pandas(train_df[['comment_text', 'toxic']])\nval_dataset = Dataset.from_pandas(val_df[['comment_text', 'toxic']])\ntest_dataset = Dataset.from_pandas(test_df[['comment_text', 'toxic']])\n\ntrain_dataset = train_dataset.map(tokenize_function, batched=True, batch_size=1024)\nval_dataset = val_dataset.map(tokenize_function, batched=True, batch_size=1024)\ntest_dataset = test_dataset.map(tokenize_function, batched=True, batch_size=1024)\n\n# Metric computation\ndef compute_metrics(eval_pred):\n    logits, labels = eval_pred\n    probs = torch.softmax(torch.tensor(logits), dim=1)[:, 1].numpy()\n    preds = torch.argmax(torch.tensor(logits), dim=1).numpy()\n    roc_auc = roc_auc_score(labels, probs)\n    accuracy = accuracy_score(labels, preds)\n    \n    return {\n        'roc_auc': roc_auc,\n        'accuracy': accuracy\n    }\n\n# Model configuration\nmodel = XLMRobertaForSequenceClassification.from_pretrained(\n    'xlm-roberta-base',\n    num_labels=2\n).to('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Training arguments\ntraining_args = TrainingArguments(\n    output_dir='./results',\n    eval_strategy='epoch',\n    learning_rate=2e-5,\n    per_device_train_batch_size=16,\n    per_device_eval_batch_size=16,\n    num_train_epochs=3,\n    weight_decay=0.01,\n    logging_dir='./logs',\n    save_strategy='no',\n    load_best_model_at_end=False,\n    metric_for_best_model='roc_auc',\n    greater_is_better=True,\n    fp16=True,\n    report_to=\"none\"\n)\n\n# Initialize Trainer\ntrainer = Trainer(\n    model=model,\n    args=training_args,\n    train_dataset=train_dataset,\n    eval_dataset=val_dataset,\n    compute_metrics=compute_metrics\n)\n\n# Start training\ntrainer.train()\n'''","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\n# Evaluate on test dataset\ntest_results = trainer.evaluate(eval_dataset=test_dataset)\nprint(f\"Test ROC-AUC: {test_results['eval_roc_auc']:.4f}\")\nprint(f\"Test Accuracy: {test_results['eval_accuracy']:.4f}\")\n'''","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}