{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Import Libralies","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nimport torch \nimport torch.nn as nn\nimport torchaudio\nimport torchaudio.transforms as tat\nimport numpy as np\nfrom datasets import load_dataset, load_metric, Audio\nimport os\n\nfrom transformers import Trainer,TrainingArguments\nfrom transformers import Wav2Vec2FeatureExtractor,Wav2Vec2Processor, Wav2Vec2CTCTokenizer,Wav2Vec2ForCTC\n\ntorchaudio.set_audio_backend(\"soundfile\")# for mp3 audio file\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2023-10-02T10:55:14.673971Z","iopub.execute_input":"2023-10-02T10:55:14.675096Z","iopub.status.idle":"2023-10-02T10:55:29.711742Z","shell.execute_reply.started":"2023-10-02T10:55:14.675062Z","shell.execute_reply":"2023-10-02T10:55:29.710705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Set hyper-parameters for Finetuning","metadata":{}},{"cell_type":"code","source":"#Parameters\nSR = 16000 \nTRAIN_ALL_WEIGHTS = True\nTRAIN_SIZE = 20000\nNUM_TRAIN_EPOCHS = 13\nPER_DEVICE_TRAIN_BATCH_SIZE = 1 \nTARGET_BATCHSIZE = 16 \nLR = 2e-5 / 16 * TARGET_BATCHSIZE \ntorch.backends.cudnn.benchmark = True ","metadata":{"execution":{"iopub.status.busy":"2023-10-02T10:55:29.713416Z","iopub.execute_input":"2023-10-02T10:55:29.713970Z","iopub.status.idle":"2023-10-02T10:55:29.720523Z","shell.execute_reply.started":"2023-10-02T10:55:29.713942Z","shell.execute_reply":"2023-10-02T10:55:29.719556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(f'/kaggle/input/hf-wav2vec2-0-preprocess-baseline/train_bengali.csv', index_col=0)\nval = pd.read_csv(f'/kaggle/input/hf-wav2vec2-0-preprocess-baseline/val_bengali.csv', index_col=0)\n\n\ntrain = train[:2000]\nval = val[:300]\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-02T10:55:29.721743Z","iopub.execute_input":"2023-10-02T10:55:29.722412Z","iopub.status.idle":"2023-10-02T10:55:36.445449Z","shell.execute_reply.started":"2023-10-02T10:55:29.722373Z","shell.execute_reply":"2023-10-02T10:55:36.444440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_path='/kaggle/input/bengaliai-speech/train_mp3s/'\nfrom IPython.display import Audio\n\naudio_path = train_path+str(train.id[10])+'.mp3'\nprint(audio_path)\nAudio(audio_path)","metadata":{"execution":{"iopub.status.busy":"2023-10-02T10:55:36.450042Z","iopub.execute_input":"2023-10-02T10:55:36.451692Z","iopub.status.idle":"2023-10-02T10:55:36.472693Z","shell.execute_reply.started":"2023-10-02T10:55:36.451647Z","shell.execute_reply":"2023-10-02T10:55:36.471647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Building Processor and Data","metadata":{}},{"cell_type":"code","source":"tokenizer = Wav2Vec2CTCTokenizer(f\"/kaggle/input/hf-wav2vec2-0-preprocess-baseline/vocab_bengali.json\", unk_token=\"[UNK]\", pad_token=\"[PAD]\", word_delimiter_token=\"|\")\nfeature_extractor = Wav2Vec2FeatureExtractor(feature_size=1, sampling_rate=SR, padding_value=0.0, do_normalize=True, return_attention_mask=True)\nprocessor = Wav2Vec2Processor(feature_extractor=feature_extractor, tokenizer=tokenizer)","metadata":{"execution":{"iopub.status.busy":"2023-10-02T10:55:36.474089Z","iopub.execute_input":"2023-10-02T10:55:36.475164Z","iopub.status.idle":"2023-10-02T10:55:36.488303Z","shell.execute_reply.started":"2023-10-02T10:55:36.475124Z","shell.execute_reply":"2023-10-02T10:55:36.487539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class W2v2Dataset(torch.utils.data.Dataset):\n    def __init__(self, df, is_train=False):\n        self.df = df\n        self.pathes = df['id'].values\n        self.sentences = df['sentence'].values\n        self.resampler = tat.Resample(32000, SR)\n        self.is_train = is_train\n\n    def __getitem__(self, idx):\n        if self.is_train:\n            idx = torch.randint(0, len(self.df), (1,))[0].numpy()\n        apath = f'/kaggle/input/bengaliai-speech/train_mp3s/{self.pathes[idx]}.mp3'\n        waveform, sample_rate = torchaudio.load(apath, format=\"mp3\")\n        waveform = self.resampler(waveform)\n        batch = dict()\n        batch[\"input_values\"] = processor(waveform.reshape(-1), sampling_rate=SR).input_values[0]  \n        with processor.as_target_processor():\n            batch[\"labels\"] = processor(self.sentences[idx]).input_ids       \n        \n        return batch\n\n    def __len__(self):\n        return TRAIN_SIZE if self.is_train else len(self.df)\n\ntrain_dataset = W2v2Dataset(train, is_train=True)\nval_dataset = W2v2Dataset(val)","metadata":{"execution":{"iopub.status.busy":"2023-10-02T10:55:36.490079Z","iopub.execute_input":"2023-10-02T10:55:36.490961Z","iopub.status.idle":"2023-10-02T10:55:36.601175Z","shell.execute_reply.started":"2023-10-02T10:55:36.490924Z","shell.execute_reply":"2023-10-02T10:55:36.600407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# From official Documentation\nfrom dataclasses import dataclass, field\nfrom typing import Any, Dict, List, Optional, Union\n\n@dataclass\nclass DataCollatorCTCWithPadding:\n    \"\"\"\n    Data collator that will dynamically pad the inputs received.\n    Args:\n        processor (:class:`~transformers.Wav2Vec2Processor`)\n            The processor used for proccessing the data.\n        padding (:obj:`bool`, :obj:`str` or :class:`~transformers.tokenization_utils_base.PaddingStrategy`, `optional`, defaults to :obj:`True`):\n            Select a strategy to pad the returned sequences (according to the model's padding side and padding index)\n            among:\n            * :obj:`True` or :obj:`'longest'`: Pad to the longest sequence in the batch (or no padding if only a single\n              sequence if provided).\n            * :obj:`'max_length'`: Pad to a maximum length specified with the argument :obj:`max_length` or to the\n              maximum acceptable input length for the model if that argument is not provided.\n            * :obj:`False` or :obj:`'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of\n              different lengths).\n        max_length (:obj:`int`, `optional`):\n            Maximum length of the ``input_values`` of the returned list and optionally padding length (see above).\n        max_length_labels (:obj:`int`, `optional`):\n            Maximum length of the ``labels`` returned list and optionally padding length (see above).\n        pad_to_multiple_of (:obj:`int`, `optional`):\n            If set will pad the sequence to a multiple of the provided value.\n            This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability >=\n            7.5 (Volta).\n    \"\"\"\n\n    processor: Wav2Vec2Processor\n    padding: Union[bool, str] = True\n    max_length: Optional[int] = None\n    max_length_labels: Optional[int] = None\n    pad_to_multiple_of: Optional[int] = None\n    pad_to_multiple_of_labels: Optional[int] = None\n\n    def __call__(self, features: List[Dict[str, Union[List[int], torch.Tensor]]]) -> Dict[str, torch.Tensor]:\n        # split inputs and labels since they have to be of different lenghts and need\n        # different padding methods\n        input_features = [{\"input_values\": feature[\"input_values\"]} for feature in features]\n        label_features = [{\"input_ids\": feature[\"labels\"]} for feature in features]\n\n        batch = self.processor.pad(\n            input_features,\n            padding=self.padding,\n            max_length=self.max_length,\n            pad_to_multiple_of=self.pad_to_multiple_of,\n            return_tensors=\"pt\",\n        )\n        with self.processor.as_target_processor():\n            labels_batch = self.processor.pad(\n                label_features,\n                padding=self.padding,\n                max_length=self.max_length_labels,\n                pad_to_multiple_of=self.pad_to_multiple_of_labels,\n                return_tensors=\"pt\",\n            )\n\n        # replace padding with -100 to ignore loss correctly\n        labels = labels_batch[\"input_ids\"].masked_fill(labels_batch.attention_mask.ne(1), -100)\n\n        batch[\"labels\"] = labels\n\n        return batch","metadata":{"execution":{"iopub.status.busy":"2023-10-02T10:55:36.602538Z","iopub.execute_input":"2023-10-02T10:55:36.603178Z","iopub.status.idle":"2023-10-02T10:55:36.616139Z","shell.execute_reply.started":"2023-10-02T10:55:36.603140Z","shell.execute_reply":"2023-10-02T10:55:36.614823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_collator = DataCollatorCTCWithPadding(processor=processor, padding=True)","metadata":{"execution":{"iopub.status.busy":"2023-10-02T10:55:36.617784Z","iopub.execute_input":"2023-10-02T10:55:36.618466Z","iopub.status.idle":"2023-10-02T10:55:36.635431Z","shell.execute_reply.started":"2023-10-02T10:55:36.618429Z","shell.execute_reply":"2023-10-02T10:55:36.633553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"def get_wav2vec2_model():\n    from transformers import Wav2Vec2ForCTC\n\n    model = Wav2Vec2ForCTC.from_pretrained(\n        'facebook/wav2vec2-large-xlsr-53',\n        ctc_loss_reduction=\"mean\", \n        ctc_zero_infinity=True, \n        pad_token_id=processor.tokenizer.pad_token_id,\n        diversity_loss_weight=100 \n    )\n    model.lm_head = nn.Linear(1024, 112)\n    model.config.vocab_size = 112\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-10-02T10:57:16.206945Z","iopub.execute_input":"2023-10-02T10:57:16.207360Z","iopub.status.idle":"2023-10-02T10:57:16.213648Z","shell.execute_reply.started":"2023-10-02T10:57:16.207307Z","shell.execute_reply":"2023-10-02T10:57:16.212293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\nmodel = get_wav2vec2_model()\nmodel.lm_head = nn.Linear(1024, len(processor.tokenizer))\nmodel.config.vocab_size = len(processor.tokenizer)","metadata":{"execution":{"iopub.status.busy":"2023-10-02T10:57:20.159910Z","iopub.execute_input":"2023-10-02T10:57:20.160375Z","iopub.status.idle":"2023-10-02T10:57:44.734993Z","shell.execute_reply.started":"2023-10-02T10:57:20.160325Z","shell.execute_reply":"2023-10-02T10:57:44.734089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.freeze_feature_extractor()","metadata":{"execution":{"iopub.status.busy":"2023-10-02T10:57:44.736955Z","iopub.execute_input":"2023-10-02T10:57:44.737515Z","iopub.status.idle":"2023-10-02T10:57:44.742867Z","shell.execute_reply.started":"2023-10-02T10:57:44.737476Z","shell.execute_reply":"2023-10-02T10:57:44.741899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train settings","metadata":{}},{"cell_type":"code","source":"# fp16_full_eval=False,\n# fp16=False,\n# fp16_backend=False,\n# half_precision_backend=False,\n# logging_steps=10,\n# learning_rate=LR,\n# warmup_steps=warmup_steps,\n# save_total_limit=3,\n# weight_decay=1e-5,\n# dataloader_num_workers=os.cpu_count()*20,\n# prediction_loss_only=False,\n# lr_scheduler_type='linear',\n# report_to='none',\n# auto_find_batch_size=True,\n\n#         attention_dropout=0.2,\n#         hidden_dropout=0.2,\n#         feat_proj_dropout=0.2,\n#         mask_time_prob=0.1,\n#         layerdrop=0.2,","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import TrainingArguments\ntraining_args = TrainingArguments(\n  output_dir=f\"./ASR\",\n  group_by_length=False,\n  per_device_train_batch_size=PER_DEVICE_TRAIN_BATCH_SIZE,\n  gradient_accumulation_steps=TARGET_BATCHSIZE // PER_DEVICE_TRAIN_BATCH_SIZE,\n  per_device_eval_batch_size=PER_DEVICE_TRAIN_BATCH_SIZE,\n  evaluation_strategy=\"epoch\",\n  num_train_epochs=NUM_TRAIN_EPOCHS,\n  save_strategy='epoch')","metadata":{"execution":{"iopub.status.busy":"2023-08-10T05:17:26.594296Z","iopub.execute_input":"2023-08-10T05:17:26.59472Z","iopub.status.idle":"2023-08-10T05:17:26.664181Z","shell.execute_reply.started":"2023-08-10T05:17:26.594689Z","shell.execute_reply":"2023-08-10T05:17:26.663013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer = Trainer(\n    model=model,\n    data_collator=data_collator,\n    args=training_args,\n    train_dataset=train_dataset,\n    eval_dataset=val_dataset,\n    tokenizer=processor.feature_extractor\n)\ntrainer.train()","metadata":{"execution":{"iopub.status.busy":"2023-08-10T05:17:26.667191Z","iopub.execute_input":"2023-08-10T05:17:26.667894Z","iopub.status.idle":"2023-08-10T05:17:33.260068Z","shell.execute_reply.started":"2023-08-10T05:17:26.667852Z","shell.execute_reply":"2023-08-10T05:17:33.25898Z"},"trusted":true},"execution_count":null,"outputs":[]}]}