{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport os\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        os.path.join(dirname, filename)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-09-03T05:46:39.765901Z","iopub.execute_input":"2023-09-03T05:46:39.766444Z","iopub.status.idle":"2023-09-03T05:50:04.242593Z","shell.execute_reply.started":"2023-09-03T05:46:39.766402Z","shell.execute_reply":"2023-09-03T05:50:04.240759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=pd.read_csv(\"../input/dlsprint/train.csv\")\ndfv=pd.read_csv(\"../input/dlsprint/validation.csv\")\n\ndf = df.iloc[:50]\ndfv= dfv.iloc[:20]\n\ndf['path'] = '/kaggle/input/dlsprint/train_files/' + df['path']\ndfv['path'] = '/kaggle/input/dlsprint/validation_files/' + dfv['path']\n\n\nprint(df.columns)\ndf.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:50:04.245000Z","iopub.execute_input":"2023-09-03T05:50:04.245490Z","iopub.status.idle":"2023-09-03T05:50:07.086756Z","shell.execute_reply.started":"2023-09-03T05:50:04.245444Z","shell.execute_reply":"2023-09-03T05:50:07.085781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport librosa\nimport torch\nimport torchaudio\nimport numpy as np\n\nfrom transformers import WhisperTokenizer\nfrom transformers import WhisperProcessor\nfrom transformers import WhisperFeatureExtractor\nfrom transformers import WhisperForConditionalGeneration","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:50:07.088161Z","iopub.execute_input":"2023-09-03T05:50:07.089451Z","iopub.status.idle":"2023-09-03T05:50:23.266565Z","shell.execute_reply.started":"2023-09-03T05:50:07.089411Z","shell.execute_reply":"2023-09-03T05:50:23.264971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nmodel_path = \"bangla-speech-processing/BanglaASR\"","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:50:23.270179Z","iopub.execute_input":"2023-09-03T05:50:23.271180Z","iopub.status.idle":"2023-09-03T05:50:23.276228Z","shell.execute_reply.started":"2023-09-03T05:50:23.271135Z","shell.execute_reply":"2023-09-03T05:50:23.275373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_extractor = WhisperFeatureExtractor.from_pretrained(model_path)\ntokenizer = WhisperTokenizer.from_pretrained(model_path)\nprocessor = WhisperProcessor.from_pretrained(model_path)\nmodel = WhisperForConditionalGeneration.from_pretrained(model_path).to(device)","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:50:23.278374Z","iopub.execute_input":"2023-09-03T05:50:23.278729Z","iopub.status.idle":"2023-09-03T05:50:34.901286Z","shell.execute_reply.started":"2023-09-03T05:50:23.278700Z","shell.execute_reply":"2023-09-03T05:50:34.899170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install jiwer\nfrom jiwer import wer","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:50:34.904595Z","iopub.execute_input":"2023-09-03T05:50:34.906243Z","iopub.status.idle":"2023-09-03T05:50:54.227445Z","shell.execute_reply.started":"2023-09-03T05:50:34.906194Z","shell.execute_reply":"2023-09-03T05:50:54.225851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install SoundFile\nimport soundfile as sf ","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:50:54.229803Z","iopub.execute_input":"2023-09-03T05:50:54.230324Z","iopub.status.idle":"2023-09-03T05:51:08.042540Z","shell.execute_reply.started":"2023-09-03T05:50:54.230277Z","shell.execute_reply":"2023-09-03T05:51:08.040736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wer_score = []\nreference_transcripts = []\npredicted_transcripts = []\n\nfor i, path in enumerate(df['path'].iloc[:20]):\n    speech_array, sampling_rate = sf.read(\n        path\n    )\n\n    speech_array = librosa.resample(np.asarray(speech_array), orig_sr=sampling_rate, target_sr=16000)\n    input_features = feature_extractor(speech_array, sampling_rate=16000, return_tensors=\"pt\").input_features\n\n    predicted_ids = model.generate(inputs=input_features.to(device))[0]\n\n    transcription = processor.decode(predicted_ids, skip_special_tokens=True)\n    \n    reference_transcripts.append(df['sentence'].iloc[i])\n    predicted_transcripts.append(transcription)\n    print('🟩', df['sentence'].iloc[i], '🟨', transcription)","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:51:08.044887Z","iopub.execute_input":"2023-09-03T05:51:08.045436Z","iopub.status.idle":"2023-09-03T05:55:24.678059Z","shell.execute_reply.started":"2023-09-03T05:51:08.045393Z","shell.execute_reply":"2023-09-03T05:55:24.676994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wer_scores = [wer(ref, hyp) for ref, hyp in zip(reference_transcripts, predicted_transcripts)]\naverage_wer = sum(wer_scores) / len(wer_scores)\nprint(f\"Average Word Error Rate: {average_wer:.2f}\")","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:55:24.680058Z","iopub.execute_input":"2023-09-03T05:55:24.683262Z","iopub.status.idle":"2023-09-03T05:55:24.697974Z","shell.execute_reply.started":"2023-09-03T05:55:24.683186Z","shell.execute_reply":"2023-09-03T05:55:24.696651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"from datasets import DatasetDict, Dataset","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:55:24.702874Z","iopub.execute_input":"2023-09-03T05:55:24.703315Z","iopub.status.idle":"2023-09-03T05:55:25.284140Z","shell.execute_reply.started":"2023-09-03T05:55:24.703279Z","shell.execute_reply":"2023-09-03T05:55:25.282719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = Dataset.from_pandas(df)\nvalidation_dataset = Dataset.from_pandas(dfv)\n\ndataset = DatasetDict({\n    \"train\": train_dataset,\n    \"validation\": validation_dataset\n})","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:55:25.285648Z","iopub.execute_input":"2023-09-03T05:55:25.288504Z","iopub.status.idle":"2023-09-03T05:55:25.341387Z","shell.execute_reply.started":"2023-09-03T05:55:25.288458Z","shell.execute_reply":"2023-09-03T05:55:25.339912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:55:25.343129Z","iopub.execute_input":"2023-09-03T05:55:25.343519Z","iopub.status.idle":"2023-09-03T05:55:25.354929Z","shell.execute_reply.started":"2023-09-03T05:55:25.343486Z","shell.execute_reply":"2023-09-03T05:55:25.352088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"common_voice_train=dataset[\"train\"]\ncommon_voice_test=dataset[\"validation\"]","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:55:25.356606Z","iopub.execute_input":"2023-09-03T05:55:25.357106Z","iopub.status.idle":"2023-09-03T05:55:25.365699Z","shell.execute_reply.started":"2023-09-03T05:55:25.357055Z","shell.execute_reply":"2023-09-03T05:55:25.364161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"common_voice_train = common_voice_train.remove_columns([\"accents\", \"age\", \"client_id\", \"down_votes\", \"gender\", \"locale\", \"up_votes\"])\ncommon_voice_test = common_voice_test.remove_columns([\"accents\", \"age\", \"client_id\", \"down_votes\", \"gender\", \"locale\", \"up_votes\"])","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:55:25.367828Z","iopub.execute_input":"2023-09-03T05:55:25.368262Z","iopub.status.idle":"2023-09-03T05:55:25.384495Z","shell.execute_reply.started":"2023-09-03T05:55:25.368230Z","shell.execute_reply":"2023-09-03T05:55:25.383107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"common_voice_train","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:55:25.386903Z","iopub.execute_input":"2023-09-03T05:55:25.387460Z","iopub.status.idle":"2023-09-03T05:55:25.400591Z","shell.execute_reply.started":"2023-09-03T05:55:25.387405Z","shell.execute_reply":"2023-09-03T05:55:25.399199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datasets import ClassLabel\nimport random\nimport pandas as pd\nfrom IPython.display import display, HTML\n\ndef show_random_elements(dataset, num_examples=10):\n    assert num_examples <= len(dataset), \"Can't pick more elements than there are in the dataset.\"\n    picks = []\n    for _ in range(num_examples):\n        pick = random.randint(0, len(dataset)-1)\n        while pick in picks:\n            pick = random.randint(0, len(dataset)-1)\n        picks.append(pick)\n    \n    df = pd.DataFrame(dataset[picks])\n    display(HTML(df.to_html()))","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:55:25.401875Z","iopub.execute_input":"2023-09-03T05:55:25.402264Z","iopub.status.idle":"2023-09-03T05:55:25.413723Z","shell.execute_reply.started":"2023-09-03T05:55:25.402234Z","shell.execute_reply":"2023-09-03T05:55:25.412578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_random_elements(common_voice_train.remove_columns([\"path\"]), num_examples=10)","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:55:25.415699Z","iopub.execute_input":"2023-09-03T05:55:25.416193Z","iopub.status.idle":"2023-09-03T05:55:25.438964Z","shell.execute_reply.started":"2023-09-03T05:55:25.416153Z","shell.execute_reply":"2023-09-03T05:55:25.438121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\nchars_to_remove_regex = '[\\,\\?\\.\\!\\-\\;\\:\\\"\\“\\%\\‘\\”\\�\\']'\n\ndef remove_special_characters(batch):\n    batch[\"sentence\"] = re.sub(chars_to_remove_regex, '', batch[\"sentence\"]).lower()\n    return batch","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:55:25.440663Z","iopub.execute_input":"2023-09-03T05:55:25.441434Z","iopub.status.idle":"2023-09-03T05:55:25.447851Z","shell.execute_reply.started":"2023-09-03T05:55:25.441378Z","shell.execute_reply":"2023-09-03T05:55:25.446694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"common_voice_train = common_voice_train.map(remove_special_characters)\ncommon_voice_test = common_voice_test.map(remove_special_characters)","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:55:25.449497Z","iopub.execute_input":"2023-09-03T05:55:25.450254Z","iopub.status.idle":"2023-09-03T05:55:25.581365Z","shell.execute_reply.started":"2023-09-03T05:55:25.450210Z","shell.execute_reply":"2023-09-03T05:55:25.579655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def extract_all_chars(batch):\n    all_text = \" \".join(batch[\"sentence\"])\n    vocab = list(set(all_text))\n    return {\"vocab\": [vocab], \"all_text\": [all_text]}","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:55:25.583291Z","iopub.execute_input":"2023-09-03T05:55:25.583674Z","iopub.status.idle":"2023-09-03T05:55:25.589721Z","shell.execute_reply.started":"2023-09-03T05:55:25.583644Z","shell.execute_reply":"2023-09-03T05:55:25.588175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vocab_train = common_voice_train.map(extract_all_chars, batched=True, batch_size=-1, keep_in_memory=False, remove_columns=common_voice_train.column_names)\nvocab_test = common_voice_test.map(extract_all_chars, batched=True, batch_size=-1, keep_in_memory=False, remove_columns=common_voice_test.column_names)","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:55:25.591778Z","iopub.execute_input":"2023-09-03T05:55:25.592225Z","iopub.status.idle":"2023-09-03T05:55:25.654802Z","shell.execute_reply.started":"2023-09-03T05:55:25.592193Z","shell.execute_reply":"2023-09-03T05:55:25.653408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vocab_list = list(set(vocab_train[\"vocab\"][0]) | set(vocab_test[\"vocab\"][0]))","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:55:25.656397Z","iopub.execute_input":"2023-09-03T05:55:25.656816Z","iopub.status.idle":"2023-09-03T05:55:25.666292Z","shell.execute_reply.started":"2023-09-03T05:55:25.656785Z","shell.execute_reply":"2023-09-03T05:55:25.664732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vocab_dict = {v: k for k, v in enumerate(sorted(vocab_list))}\nprint(vocab_dict)","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:55:25.668239Z","iopub.execute_input":"2023-09-03T05:55:25.668677Z","iopub.status.idle":"2023-09-03T05:55:25.683387Z","shell.execute_reply.started":"2023-09-03T05:55:25.668638Z","shell.execute_reply":"2023-09-03T05:55:25.681845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vocab_dict[\"|\"] = vocab_dict[\" \"]\ndel vocab_dict[\" \"]","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:55:25.685867Z","iopub.execute_input":"2023-09-03T05:55:25.686405Z","iopub.status.idle":"2023-09-03T05:55:25.695085Z","shell.execute_reply.started":"2023-09-03T05:55:25.686358Z","shell.execute_reply":"2023-09-03T05:55:25.693408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vocab_dict[\"[UNK]\"] = len(vocab_dict)\nvocab_dict[\"[PAD]\"] = len(vocab_dict)\nlen(vocab_dict)","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:55:25.697524Z","iopub.execute_input":"2023-09-03T05:55:25.698053Z","iopub.status.idle":"2023-09-03T05:55:25.711503Z","shell.execute_reply.started":"2023-09-03T05:55:25.698006Z","shell.execute_reply":"2023-09-03T05:55:25.710175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\nwith open('vocab.json', 'w') as vocab_file:\n    json.dump(vocab_dict, vocab_file)","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:55:25.712914Z","iopub.execute_input":"2023-09-03T05:55:25.713405Z","iopub.status.idle":"2023-09-03T05:55:25.726190Z","shell.execute_reply.started":"2023-09-03T05:55:25.713358Z","shell.execute_reply":"2023-09-03T05:55:25.724281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import WhisperTokenizer\nfrom transformers import WhisperProcessor\nfrom transformers import WhisperFeatureExtractor\nfrom transformers import WhisperForConditionalGeneration","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:55:25.727733Z","iopub.execute_input":"2023-09-03T05:55:25.728235Z","iopub.status.idle":"2023-09-03T05:55:25.739567Z","shell.execute_reply.started":"2023-09-03T05:55:25.728189Z","shell.execute_reply":"2023-09-03T05:55:25.737853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = WhisperTokenizer.from_pretrained(model_path, language=\"Bengali\", task=\"transcribe\")","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:55:25.748637Z","iopub.execute_input":"2023-09-03T05:55:25.749070Z","iopub.status.idle":"2023-09-03T05:55:26.067733Z","shell.execute_reply.started":"2023-09-03T05:55:25.749033Z","shell.execute_reply":"2023-09-03T05:55:26.066170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import WhisperFeatureExtractor\n\nfeature_extractor = WhisperFeatureExtractor.from_pretrained(model_path)","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:55:26.069728Z","iopub.execute_input":"2023-09-03T05:55:26.070143Z","iopub.status.idle":"2023-09-03T05:55:26.160106Z","shell.execute_reply.started":"2023-09-03T05:55:26.070111Z","shell.execute_reply":"2023-09-03T05:55:26.158290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import WhisperProcessor\n\nprocessor = WhisperProcessor.from_pretrained(model_path, language=\"Bengali\", task=\"transcribe\")","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:55:26.162307Z","iopub.execute_input":"2023-09-03T05:55:26.162763Z","iopub.status.idle":"2023-09-03T05:55:26.469676Z","shell.execute_reply.started":"2023-09-03T05:55:26.162728Z","shell.execute_reply":"2023-09-03T05:55:26.468161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import io\n\ndef audio_bytes_generator(row):\n    with open(row['path'], 'rb') as mp3_file:\n        mp3_bytes = mp3_file.read()\n\n    row['audio'] = {\n        'bytes': mp3_bytes,\n        'path': row['path']\n    }\n    return row","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:55:26.472841Z","iopub.execute_input":"2023-09-03T05:55:26.473362Z","iopub.status.idle":"2023-09-03T05:55:26.480110Z","shell.execute_reply.started":"2023-09-03T05:55:26.473301Z","shell.execute_reply":"2023-09-03T05:55:26.478785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"common_voice_train = common_voice_train.map(audio_bytes_generator)\ncommon_voice_test = common_voice_test.map(audio_bytes_generator)","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:55:26.482402Z","iopub.execute_input":"2023-09-03T05:55:26.483313Z","iopub.status.idle":"2023-09-03T05:55:27.025928Z","shell.execute_reply.started":"2023-09-03T05:55:26.483256Z","shell.execute_reply":"2023-09-03T05:55:27.024979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(common_voice_train.to_dict()).head()","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:55:27.027303Z","iopub.execute_input":"2023-09-03T05:55:27.027924Z","iopub.status.idle":"2023-09-03T05:55:27.055977Z","shell.execute_reply.started":"2023-09-03T05:55:27.027888Z","shell.execute_reply":"2023-09-03T05:55:27.054680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datasets import load_dataset, load_metric, Audio\n\ncommon_voice_train = common_voice_train.cast_column(\"audio\", Audio(sampling_rate=16_000))\ncommon_voice_test = common_voice_test.cast_column(\"audio\", Audio(sampling_rate=16_000))","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:55:27.057980Z","iopub.execute_input":"2023-09-03T05:55:27.059243Z","iopub.status.idle":"2023-09-03T05:55:27.069533Z","shell.execute_reply.started":"2023-09-03T05:55:27.059136Z","shell.execute_reply":"2023-09-03T05:55:27.068357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"common_voice_train[0]","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:55:27.071658Z","iopub.execute_input":"2023-09-03T05:55:27.072723Z","iopub.status.idle":"2023-09-03T05:55:27.191066Z","shell.execute_reply.started":"2023-09-03T05:55:27.072669Z","shell.execute_reply":"2023-09-03T05:55:27.190143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import IPython.display as ipd\nimport numpy as np\nimport random\n\nrand_int = random.randint(0, len(common_voice_train)-1)\n\nprint(common_voice_train[rand_int][\"sentence\"])\nipd.Audio(data=common_voice_train[rand_int][\"audio\"][\"array\"], autoplay=False, rate=16000)","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:58:17.423512Z","iopub.execute_input":"2023-09-03T05:58:17.423951Z","iopub.status.idle":"2023-09-03T05:58:17.456681Z","shell.execute_reply.started":"2023-09-03T05:58:17.423920Z","shell.execute_reply":"2023-09-03T05:58:17.455670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rand_int = random.randint(0, len(common_voice_train)-1)\n\nprint(\"Target text:\", common_voice_train[rand_int][\"sentence\"])\nprint(\"Input array shape:\", common_voice_train[rand_int][\"audio\"][\"array\"].shape)\nprint(\"Sampling rate:\", common_voice_train[rand_int][\"audio\"][\"sampling_rate\"])","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:58:30.027493Z","iopub.execute_input":"2023-09-03T05:58:30.027954Z","iopub.status.idle":"2023-09-03T05:58:30.059911Z","shell.execute_reply.started":"2023-09-03T05:58:30.027921Z","shell.execute_reply":"2023-09-03T05:58:30.058541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_dataset(batch):\n    audio = batch[\"audio\"]\n    \n    batch[\"input_features\"] = feature_extractor(audio[\"array\"], sampling_rate=audio[\"sampling_rate\"]).input_features[0]\n    \n    batch[\"labels\"] = tokenizer(batch[\"sentence\"]).input_ids\n    return batch","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:58:40.466120Z","iopub.execute_input":"2023-09-03T05:58:40.467563Z","iopub.status.idle":"2023-09-03T05:58:40.476136Z","shell.execute_reply.started":"2023-09-03T05:58:40.467505Z","shell.execute_reply":"2023-09-03T05:58:40.474404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"common_voice_train=common_voice_train.select(range(50))\ncommon_voice_test=common_voice_test.select(range(20))","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:58:42.138416Z","iopub.execute_input":"2023-09-03T05:58:42.138924Z","iopub.status.idle":"2023-09-03T05:58:42.153503Z","shell.execute_reply.started":"2023-09-03T05:58:42.138887Z","shell.execute_reply":"2023-09-03T05:58:42.152190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"common_voice_train = common_voice_train.map(prepare_dataset, remove_columns=common_voice_train.column_names)\ncommon_voice_test = common_voice_test.map(prepare_dataset, remove_columns=common_voice_test.column_names)","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:58:51.744416Z","iopub.execute_input":"2023-09-03T05:58:51.744904Z","iopub.status.idle":"2023-09-03T05:59:01.923274Z","shell.execute_reply.started":"2023-09-03T05:58:51.744871Z","shell.execute_reply":"2023-09-03T05:59:01.922359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"common_voice_test","metadata":{"execution":{"iopub.status.busy":"2023-09-03T05:59:51.888119Z","iopub.execute_input":"2023-09-03T05:59:51.889515Z","iopub.status.idle":"2023-09-03T05:59:51.897273Z","shell.execute_reply.started":"2023-09-03T05:59:51.889460Z","shell.execute_reply":"2023-09-03T05:59:51.895843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\n\nfrom dataclasses import dataclass, field\nfrom typing import Any, Dict, List, Optional, Union\n\n@dataclass\nclass DataCollatorSpeechSeq2SeqWithPadding:\n    processor: Any\n\n    def __call__(self, features: List[Dict[str, Union[List[int], torch.Tensor]]]) -> Dict[str, torch.Tensor]:\n       \n        input_features = [{\"input_features\": feature[\"input_features\"]} for feature in features]\n        batch = self.processor.feature_extractor.pad(input_features, return_tensors=\"pt\")\n\n        label_features = [{\"input_ids\": feature[\"labels\"]} for feature in features]\n        labels_batch = self.processor.tokenizer.pad(label_features, return_tensors=\"pt\")\n\n        labels = labels_batch[\"input_ids\"].masked_fill(labels_batch.attention_mask.ne(1), -100)\n\n        if (labels[:, 0] == self.processor.tokenizer.bos_token_id).all().cpu().item():\n            labels = labels[:, 1:]\n\n        batch[\"labels\"] = labels\n\n        return batch","metadata":{"execution":{"iopub.status.busy":"2023-09-03T06:01:07.375784Z","iopub.execute_input":"2023-09-03T06:01:07.376297Z","iopub.status.idle":"2023-09-03T06:01:07.394452Z","shell.execute_reply.started":"2023-09-03T06:01:07.376264Z","shell.execute_reply":"2023-09-03T06:01:07.393326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_collator = DataCollatorSpeechSeq2SeqWithPadding(processor=processor)","metadata":{"execution":{"iopub.status.busy":"2023-09-03T06:01:23.207579Z","iopub.execute_input":"2023-09-03T06:01:23.209028Z","iopub.status.idle":"2023-09-03T06:01:23.216680Z","shell.execute_reply.started":"2023-09-03T06:01:23.208968Z","shell.execute_reply":"2023-09-03T06:01:23.214747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wer_metric = load_metric(\"wer\")","metadata":{"execution":{"iopub.status.busy":"2023-09-03T06:01:24.747924Z","iopub.execute_input":"2023-09-03T06:01:24.748399Z","iopub.status.idle":"2023-09-03T06:01:25.357883Z","shell.execute_reply.started":"2023-09-03T06:01:24.748364Z","shell.execute_reply":"2023-09-03T06:01:25.356873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compute_metrics(pred):\n    pred_logits = pred.predictions\n    pred_ids = np.argmax(pred_logits, axis=-1)\n\n    pred.label_ids[pred.label_ids == -100] = processor.tokenizer.pad_token_id\n\n    pred_str = processor.batch_decode(pred_ids)\n    # we do not want to group tokens when computing the metrics\n    label_str = processor.batch_decode(pred.label_ids, group_tokens=False)\n\n    wer = wer_metric.compute(predictions=pred_str, references=label_str)\n\n    return {\"wer\": wer}","metadata":{"execution":{"iopub.status.busy":"2023-09-03T06:01:28.709276Z","iopub.execute_input":"2023-09-03T06:01:28.709745Z","iopub.status.idle":"2023-09-03T06:01:28.717198Z","shell.execute_reply.started":"2023-09-03T06:01:28.709713Z","shell.execute_reply":"2023-09-03T06:01:28.716218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import WhisperForConditionalGeneration\n\nmodel = WhisperForConditionalGeneration.from_pretrained(model_path)","metadata":{"execution":{"iopub.status.busy":"2023-09-03T06:01:30.813124Z","iopub.execute_input":"2023-09-03T06:01:30.813672Z","iopub.status.idle":"2023-09-03T06:01:36.297250Z","shell.execute_reply.started":"2023-09-03T06:01:30.813634Z","shell.execute_reply":"2023-09-03T06:01:36.295694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import Seq2SeqTrainingArguments\n\nstep_n = 5 \n\ntraining_args = Seq2SeqTrainingArguments(\n  output_dir=\"./output_model\",\n  group_by_length=True,\n  per_device_train_batch_size=32,\n  gradient_accumulation_steps=2,\n  evaluation_strategy=\"steps\",\n  num_train_epochs=1,\n  gradient_checkpointing=True,\n  save_steps=step_n,\n  eval_steps=step_n,\n  logging_steps=step_n,\n  learning_rate=3e-4,\n  warmup_steps=10,\n  save_total_limit=2,\n  push_to_hub=False,\n)","metadata":{"execution":{"iopub.status.busy":"2023-09-03T06:01:41.568478Z","iopub.execute_input":"2023-09-03T06:01:41.568956Z","iopub.status.idle":"2023-09-03T06:01:41.645320Z","shell.execute_reply.started":"2023-09-03T06:01:41.568922Z","shell.execute_reply":"2023-09-03T06:01:41.643735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import Seq2SeqTrainer\n\ntrainer = Seq2SeqTrainer(\n    model=model,\n    data_collator=data_collator,\n    args=training_args,\n    compute_metrics=compute_metrics,\n    train_dataset=common_voice_train,\n    eval_dataset=common_voice_test,\n    tokenizer=processor.feature_extractor,\n)","metadata":{"execution":{"iopub.status.busy":"2023-09-03T06:01:43.692069Z","iopub.execute_input":"2023-09-03T06:01:43.692618Z","iopub.status.idle":"2023-09-03T06:01:43.987747Z","shell.execute_reply.started":"2023-09-03T06:01:43.692579Z","shell.execute_reply":"2023-09-03T06:01:43.986160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nos.environ[\"WANDB_DISABLED\"] = \"true\"","metadata":{"execution":{"iopub.status.busy":"2023-09-03T06:01:49.030534Z","iopub.execute_input":"2023-09-03T06:01:49.031049Z","iopub.status.idle":"2023-09-03T06:01:49.036701Z","shell.execute_reply.started":"2023-09-03T06:01:49.031012Z","shell.execute_reply":"2023-09-03T06:01:49.035534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer.train()","metadata":{"execution":{"iopub.status.busy":"2023-09-03T06:02:22.601621Z","iopub.execute_input":"2023-09-03T06:02:22.602085Z","iopub.status.idle":"2023-09-03T06:27:52.271220Z","shell.execute_reply.started":"2023-09-03T06:02:22.602053Z","shell.execute_reply":"2023-09-03T06:27:52.269165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls","metadata":{"execution":{"iopub.status.busy":"2023-09-03T06:27:52.275787Z","iopub.execute_input":"2023-09-03T06:27:52.279487Z","iopub.status.idle":"2023-09-03T06:27:53.579096Z","shell.execute_reply.started":"2023-09-03T06:27:52.279436Z","shell.execute_reply":"2023-09-03T06:27:53.577303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}