{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":7634,"databundleVersionId":46676}],"dockerImageVersionId":31329,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!mkdir -p /kaggle/working/dataset/train\n\n!7z x /kaggle/input/competitions/tensorflow-speech-recognition-challenge/train.7z -o/kaggle/working/dataset/train -y -bd\n\n!pip install -q transformers evaluate soundfile accelerate matplotlib","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================== #\n#                    EXPERIMENT 2: WAV2VEC 2.0 FULL FREEZING                     #\n# ============================================================================== #\n\n# ------------------------------------------------------------------------------ #\n# [1] IMPORTS & SETUP\n# ------------------------------------------------------------------------------ #\nimport os\nimport random\nimport torch\nimport numpy as np\nimport soundfile as sf\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.metrics import confusion_matrix\nfrom torch.utils.data import Dataset\nfrom transformers import (\n    Wav2Vec2FeatureExtractor, \n    Wav2Vec2ForSequenceClassification,\n    Trainer, \n    TrainingArguments,\n    TrainerCallback\n)\nimport evaluate\n\n# Set random seeds for strict reproducibility\ntorch.manual_seed(42)\nrandom.seed(42)\nnp.random.seed(42)\n\n# ------------------------------------------------------------------------------ #\n# [2] CONFIGURATION & LABEL MAPPING \n# ------------------------------------------------------------------------------ #\nimport glob\n\nfound_paths = glob.glob('/kaggle/working/**/validation_list.txt', recursive=True)\n\nif not found_paths:\n    raise FileNotFoundError(\"Text files unavaliable.\")\n\nBASE_TRAIN_DIR = os.path.dirname(found_paths[0])\nAUDIO_DIR = os.path.join(BASE_TRAIN_DIR, \"audio\")\nVAL_LIST_PATH = os.path.join(BASE_TRAIN_DIR, \"validation_list.txt\")\nTEST_LIST_PATH = os.path.join(BASE_TRAIN_DIR, \"testing_list.txt\")\n\nCORE_CLASSES = ['yes', 'no', 'up', 'down', 'left', 'right', 'on', 'off', 'stop', 'go']\nALL_CLASSES = CORE_CLASSES + ['silence', 'unknown']\n\nlabel2id = {label: i for i, label in enumerate(ALL_CLASSES)}\nid2label = {i: label for i, label in enumerate(ALL_CLASSES)}\n\n\n# ------------------------------------------------------------------------------ #\n# [3] HELPER: READ SPLIT LISTS\n# ------------------------------------------------------------------------------ #\ndef load_split_list(file_path):\n    if not os.path.exists(file_path):\n        print(f\"[WARNING] Split file not found: {file_path}. Returning empty set.\")\n        return set()\n    with open(file_path, 'r') as f:\n        return set(line.strip() for line in f if line.strip())\n\nval_files = load_split_list(VAL_LIST_PATH)\ntest_files = load_split_list(TEST_LIST_PATH)\n\n# ------------------------------------------------------------------------------ #\n# [4] CUSTOM PYTORCH DATASET WITH DETERMINISTIC SPLITS\n# ------------------------------------------------------------------------------ #\nclass SpeechCommandsDataset(Dataset):\n    def __init__(self, audio_dir, feature_extractor, split='train', max_samples=None):\n        self.audio_dir = audio_dir\n        self.feature_extractor = feature_extractor\n        self.split = split\n        self.samples = []\n        \n        self.background_noises = []\n        noise_dir = os.path.join(audio_dir, \"_background_noise_\")\n        if os.path.exists(noise_dir):\n            for f in os.listdir(noise_dir):\n                if f.endswith('.wav') and f != 'README.md':\n                    noise_array, _ = sf.read(os.path.join(noise_dir, f))\n                    self.background_noises.append(noise_array)\n        \n        unknown_files = []\n        class_counts = {label: 0 for label in ALL_CLASSES}\n        \n        if os.path.exists(audio_dir):\n            for folder_name in os.listdir(audio_dir):\n                folder_path = os.path.join(audio_dir, folder_name)\n                \n                if not os.path.isdir(folder_path) or folder_name == '_background_noise_':\n                    continue\n                    \n                for f in os.listdir(folder_path):\n                    if not f.endswith('.wav'):\n                        continue\n                        \n                    rel_path = f\"{folder_name}/{f}\"\n                    is_val = rel_path in val_files\n                    is_test = rel_path in test_files\n                    \n                    if split == 'val' and not is_val: continue\n                    if split == 'test' and not is_test: continue\n                    if split == 'train' and (is_val or is_test): continue\n                    \n                    full_path = os.path.join(folder_path, f)\n                    \n                    if folder_name in CORE_CLASSES:\n                        if max_samples is None or class_counts[folder_name] < max_samples:\n                            self.samples.append({\"path\": full_path, \"label\": label2id[folder_name]})\n                            class_counts[folder_name] += 1\n                    else:\n                        unknown_files.append(full_path)\n        \n        random.shuffle(unknown_files)\n        core_counts = [count for label, count in class_counts.items() if label in CORE_CLASSES and count > 0]\n        \n        target_unknown_count = max_samples if max_samples else (min(core_counts) if core_counts else len(unknown_files))\n        if target_unknown_count == 0: target_unknown_count = len(unknown_files)\n        \n        for f in unknown_files[:target_unknown_count]:\n            self.samples.append({\"path\": f, \"label\": label2id['unknown']})\n            \n        for _ in range(target_unknown_count):\n            self.samples.append({\"path\": \"REAL_NOISE\", \"label\": label2id['silence']})\n\n        print(f\"[INFO] {split.upper()} Dataset loaded with {len(self.samples)} samples.\")\n\n    def __len__(self):\n        return len(self.samples)\n\n    def __getitem__(self, idx):\n        item = self.samples[idx]\n        target_length = 16000\n        \n        if item[\"path\"] == \"REAL_NOISE\":\n            if self.background_noises:\n                noise_audio = random.choice(self.background_noises)\n                start_idx = random.randint(0, len(noise_audio) - target_length)\n                speech = noise_audio[start_idx : start_idx + target_length]\n                speech = speech * random.uniform(0.0, 0.8)\n            else:\n                speech = np.random.normal(0, 0.001, target_length)\n        else:\n            speech, _ = sf.read(item[\"path\"])\n            \n        if len(speech) < target_length:\n            speech = np.pad(speech, (0, target_length - len(speech)), 'constant')\n        else:\n            speech = speech[:target_length]\n            \n        inputs = self.feature_extractor(\n            speech, \n            sampling_rate=16000, \n            return_tensors=\"pt\", \n            padding=True\n        )\n        \n        return {\n            \"input_values\": inputs.input_values.squeeze(0),\n            \"labels\": torch.tensor(item[\"label\"], dtype=torch.long)\n        }\n  \n# ------------------------------------------------------------------------------ #\n# [5] DATASET INSTANTIATION\n# ------------------------------------------------------------------------------ #\nprint(\"[INFO] Loading Feature Extractor...\")\nfeature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(\"facebook/wav2vec2-base\")\n\ntrain_dataset = SpeechCommandsDataset(AUDIO_DIR, feature_extractor, split='train')\neval_dataset = SpeechCommandsDataset(AUDIO_DIR, feature_extractor, split='val')\ntest_dataset = SpeechCommandsDataset(AUDIO_DIR, feature_extractor, split='test')\n\naccuracy_metric = evaluate.load(\"accuracy\")\n\ndef compute_metrics(eval_pred):\n    predictions = np.argmax(eval_pred.predictions, axis=1)\n    return accuracy_metric.compute(predictions=predictions, references=eval_pred.label_ids)\n\nclass LiveLoggingCallback(TrainerCallback):\n    def __init__(self):\n        super().__init__()\n        self.last_train_loss = None \n\n    def on_log(self, args, state, control, logs=None, **kwargs):\n        if logs is not None:\n            if 'loss' in logs:\n                self.last_train_loss = logs['loss']\n            \n            if 'eval_accuracy' in logs and 'eval_loss' in logs:\n                step = state.global_step\n                epoch = int(round(logs.get('epoch', 0), 0))\n                \n                acc = logs['eval_accuracy']\n                val_loss = logs['eval_loss']\n                train_loss_str = f\"{self.last_train_loss:.4f}\" if self.last_train_loss is not None else \"N/A\"\n                \n                print(f\"Epoch {epoch} | (Step: {step}) | Train Loss: {train_loss_str} | Val Loss: {val_loss:.4f} | Val Accuracy: {acc:.4f}\")\n\n# ------------------------------------------------------------------------------ #\n# [6] MULTIPLE RUNS SETUP\n# ------------------------------------------------------------------------------ #\nNUM_RUNS = 5\n\n# Storage for plotting\nall_train_losses = []\nall_eval_losses = []\nall_eval_accs = []\n\n# Storage for the best model's predictions (for the final confusion matrix)\nbest_accuracy = -1.0\nbest_cm_y_true = None\nbest_cm_y_pred = None\nbest_run_idx = -1\n\nfor run in range(NUM_RUNS):\n    print(f\"\\n{'='*60}\")\n    print(f\"[INFO] STARTING RUN {run + 1}/{NUM_RUNS}\")\n    print(f\"{'='*60}\\n\")\n    \n    # Reload model for each run to ensure fresh weights\n    print(\"[INFO] Loading Pre-trained Wav2Vec 2.0 Model...\")\n    model = Wav2Vec2ForSequenceClassification.from_pretrained(\n        \"facebook/wav2vec2-base\",\n        num_labels=len(ALL_CLASSES),\n        label2id=label2id,\n        id2label=id2label\n    )\n\n    # Freeze the feature extractor to save memory\n    model.freeze_feature_encoder()\n    \n    # FULL FREEZING: Freeze the entire Transformer\n    for param in model.wav2vec2.parameters():\n        param.requires_grad = False\n    \n    # Ensure the classification head is trainable\n    for param in model.classifier.parameters():\n        param.requires_grad = True\n        \n    print(\"[INFO] Wav2Vec 2.0 Model Loaded (CNN & First 6 Transformer Layers Frozen)...\")\n\n    # Update output directory for each run to avoid overwriting checkpoints\n    training_args = TrainingArguments(\n        output_dir=f\"/kaggle/tmp/wav2vec2-checkpoints-run{run + 1}\",\n        eval_strategy=\"epoch\",  \n        logging_strategy=\"epoch\",\n        save_strategy=\"epoch\",\n        learning_rate=1e-4,            \n        per_device_train_batch_size=32,\n        per_device_eval_batch_size=32,\n        num_train_epochs=8,            \n        save_total_limit=1,\n        load_best_model_at_end=True,\n        metric_for_best_model=\"accuracy\",\n        fp16=True,\n        report_to=\"none\"\n    )\n\n    trainer = Trainer(\n        model=model,\n        args=training_args,\n        train_dataset=train_dataset,\n        eval_dataset=eval_dataset,\n        compute_metrics=compute_metrics,\n        callbacks=[LiveLoggingCallback()]\n    )\n\n    \n    # ------------------------------------------------------------------------------ #\n    # [7] EXECUTE PIPELINE\n    # ------------------------------------------------------------------------------ #\n    print(f\"\\n[INFO] Evaluating model BEFORE training (Random Classification Head - Run {run + 1})...\")\n    metrics_before = trainer.evaluate(eval_dataset=test_dataset)\n    print(f\"--> Test Accuracy BEFORE training: {metrics_before['eval_accuracy']:.4f}\\n\")\n\n    print(f\"[INFO] Starting training phase for Run {run + 1}...\")\n    trainer.train()\n\n    history = trainer.state.log_history\n    train_loss = [x['loss'] for x in history if 'loss' in x]\n\n    eval_loss = [x['eval_loss'] for x in history if 'eval_loss' in x and x.get('step', 0) > 0]\n    eval_acc = [x['eval_accuracy'] for x in history if 'eval_accuracy' in x and x.get('step', 0) > 0]\n    \n    min_len = min(len(train_loss), len(eval_loss), len(eval_acc))\n    all_train_losses.append(train_loss[:min_len])\n    all_eval_losses.append(eval_loss[:min_len])\n    all_eval_accs.append(eval_acc[:min_len])\n\n    print(f\"\\n[INFO] Training complete. Evaluating BEST model of Run {run + 1} on Test Set...\")\n    metrics_after = trainer.evaluate(eval_dataset=test_dataset)\n    final_test_acc = metrics_after['eval_accuracy']\n    print(f\"--> Final Test Accuracy for Run {run + 1}: {final_test_acc:.4f}\")\n\n    # Check if this run is the best overall\n    if final_test_acc > best_accuracy:\n        best_accuracy = final_test_acc\n        best_run_idx = run + 1\n        \n        # Save predictions for the confusion matrix\n        predictions_output = trainer.predict(test_dataset)\n        best_cm_y_pred = np.argmax(predictions_output.predictions, axis=1)\n        best_cm_y_true = predictions_output.label_ids\n\nprint(f\"\\n[INFO] All runs completed. Best Run was Run {best_run_idx} with Accuracy: {best_accuracy:.4f}\")\n\n# ------------------------------------------------------------------------------ #\n# [8] PLOTTING LEARNING CURVES & CONFUSION MATRIX\n# ------------------------------------------------------------------------------ #\nprint(\"[INFO] Generating learning curve plots...\")\n\n# Convert lists to numpy arrays for easier aggregation (mean, min, max)\narr_train_loss = np.array(all_train_losses)\narr_eval_loss = np.array(all_eval_losses)\narr_eval_acc = np.array(all_eval_accs)\nepochs = range(1, arr_train_loss.shape[1] + 1)\n\nplt.figure(figsize=(14, 5))\n\n# --- Plot 1: Loss ---\nplt.subplot(1, 2, 1)\n# Plot all individual runs\nfor i in range(NUM_RUNS):\n    plt.plot(epochs, arr_train_loss[i], color='blue', alpha=0.2)\n    plt.plot(epochs, arr_eval_loss[i], color='red', alpha=0.2)\n\n# Plot Mean\nplt.plot(epochs, np.mean(arr_train_loss, axis=0), 'b-o', label='Mean Training Loss')\nplt.plot(epochs, np.mean(arr_eval_loss, axis=0), 'r-s', label='Mean Validation Loss')\n\n# Plot Max Deviation (Min-Max Band)\nplt.fill_between(epochs, np.min(arr_train_loss, axis=0), np.max(arr_train_loss, axis=0), color='blue', alpha=0.1, label='Train Min-Max Band')\nplt.fill_between(epochs, np.min(arr_eval_loss, axis=0), np.max(arr_eval_loss, axis=0), color='red', alpha=0.1, label='Val Min-Max Band')\n\nplt.title('Training and Validation Loss (5 Runs)', fontsize=14)\nplt.xlabel('Epochs', fontsize=12)\nplt.ylabel('Loss', fontsize=12)\nplt.legend()\nplt.grid(True, linestyle='--', alpha=0.6)\n\n# --- Plot 2: Accuracy ---\nplt.subplot(1, 2, 2)\n# Plot all individual runs\nfor i in range(NUM_RUNS):\n    plt.plot(epochs, arr_eval_acc[i], color='green', alpha=0.2)\n\n# Plot Mean\nplt.plot(epochs, np.mean(arr_eval_acc, axis=0), 'g-^', label='Mean Validation Accuracy')\n\n# Plot Max Deviation (Min-Max Band)\nplt.fill_between(epochs, np.min(arr_eval_acc, axis=0), np.max(arr_eval_acc, axis=0), color='green', alpha=0.1, label='Val Accuracy Min-Max Band')\n\nplt.title('Validation Accuracy over Epochs (5 Runs)', fontsize=14)\nplt.xlabel('Epochs', fontsize=12)\nplt.ylabel('Accuracy', fontsize=12)\nplt.legend()\nplt.grid(True, linestyle='--', alpha=0.6)\n\nplt.suptitle('Wav2Vec - Full Freezing (Aggregated 5 Runs)', fontsize=16, fontweight='bold')\nplt.tight_layout()\nplt.subplots_adjust(top=0.88)\nplt.savefig(\"wav2vec-full-freezing-5runs.png\", dpi=300)\nplt.show()\n\n# --- Confusion Matrix (Best Run Only) ---\nprint(f\"[INFO] Generating Confusion Matrix for Best Run (Run {best_run_idx})...\")\ncm = confusion_matrix(best_cm_y_true, best_cm_y_pred)\n\nplt.figure(figsize=(10, 8))\nsns.heatmap(\n    cm, \n    annot=True, \n    fmt='d', \n    cmap='Blues', \n    xticklabels=ALL_CLASSES, \n    yticklabels=ALL_CLASSES\n)\n\nplt.title(f'Confusion Matrix - Full Freezing (Best Run: {best_run_idx})', fontsize=14, fontweight='bold')\nplt.ylabel('Actual Label', fontsize=12)\nplt.xlabel('Predicted Label', fontsize=12)\nplt.xticks(rotation=45, ha='right')\nplt.tight_layout()\nplt.savefig(\"wav2vec-full-freezing-conf-matrix-best.png\", dpi=300)\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}