{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# STARTER [Pytorch Version]","metadata":{}},{"cell_type":"markdown","source":"## SOURCE\n - [Automatic speech recognition](https://huggingface.co/docs/transformers/main/tasks/asr)\n - [int8 training for automatic speech recognition](https://huggingface.co/docs/peft/task_guides/int8-asr#int8-training-for-automatic-speech-recognition)\n - [Detailed EDA, Normalizer and WER](https://www.kaggle.com/code/mbmmurad/detailed-eda-normalizer-and-wer) ","metadata":{}},{"cell_type":"markdown","source":"# Install","metadata":{}},{"cell_type":"code","source":"%%capture\n%pip install wandb pyctcdecode colorama kenlm\n%pip install git+https://github.com/huggingface/peft.git\n%pip install -q peft transformers datasets accelerate evaluate jiwer bitsandbytes","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:32:06.665355Z","iopub.execute_input":"2023-07-30T16:32:06.666133Z","iopub.status.idle":"2023-07-30T16:33:51.978440Z","shell.execute_reply.started":"2023-07-30T16:32:06.666089Z","shell.execute_reply":"2023-07-30T16:33:51.976931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"import gc\nimport os\nimport time\nimport copy\nimport random\n\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\n# import seaborn as sns\n\n# Pytorch\nimport torch \nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import DataLoader, Dataset\n\nimport librosa\nimport librosa.display\nimport IPython.display as ipd\n\n# huggingface - transformers\nfrom transformers import AutoProcessor, AutoModelForCTC, TrainingArguments, Trainer, Wav2Vec2ProcessorWithLM, AdamW\nfrom transformers import TrainerCallback, TrainerState, TrainerControl\nfrom transformers.trainer_utils import PREFIX_CHECKPOINT_DIR\n\n# evaluate : WER\nimport evaluate\n\n# HuggingFace peft \nfrom peft import LoraConfig, PeftModel, LoraModel, LoraConfig, get_peft_model,  prepare_model_for_int8_training, TaskType\n\n# StratifiedKFold\n# from sklearn.model_selection import StratifiedKFold\n\n# Utils\nfrom tqdm.auto import tqdm, trange\n\n# Suppress warnings\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# For colored terminal text\nfrom colorama import Fore, Back, Style\nb_ = Fore.BLUE\ny_ = Fore.YELLOW\nsr_ = Style.RESET_ALL\n\n# For descriptive error messages\n# os.environ['CUDA_LAUNCH_BLOCKING'] = \"1\"\n\n# Tokenizer Warnings\n# os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:33:51.980757Z","iopub.execute_input":"2023-07-30T16:33:51.981360Z","iopub.status.idle":"2023-07-30T16:34:08.699159Z","shell.execute_reply.started":"2023-07-30T16:33:51.981323Z","shell.execute_reply":"2023-07-30T16:34:08.698022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"config = {\"seed\": 2023, \n          \"model_name\": \"arijitx/wav2vec2-xls-r-300m-bengali\", #\"openai/whisper-base\",\n          \"competition\": \"Bengali.AI Spech Recognition 2023\",\n          \"sample_rate\": 16000\n          }","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:36:18.327769Z","iopub.execute_input":"2023-07-30T16:36:18.328972Z","iopub.status.idle":"2023-07-30T16:36:18.338036Z","shell.execute_reply.started":"2023-07-30T16:36:18.328917Z","shell.execute_reply":"2023-07-30T16:36:18.336507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Data","metadata":{}},{"cell_type":"code","source":"train_data_path = '/kaggle/input/bengali-ai-audio-visualization-on-wandb/train_new.csv'\n\ntrain = pd.read_csv(train_data_path)\nprint(train.shape)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:34:08.710473Z","iopub.execute_input":"2023-07-30T16:34:08.710848Z","iopub.status.idle":"2023-07-30T16:34:16.148753Z","shell.execute_reply.started":"2023-07-30T16:34:08.710818Z","shell.execute_reply":"2023-07-30T16:34:16.147597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# processor","metadata":{}},{"cell_type":"code","source":"### This cell costs time\n# from transformers import AutoProcessor\n\nprint(f\"PRETRAINED: {config['model_name']}\")\nprocessor = AutoProcessor.from_pretrained(config['model_name'])\nprint(\"AutoProcessor Downloaded\")","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:34:16.150519Z","iopub.execute_input":"2023-07-30T16:34:16.151615Z","iopub.status.idle":"2023-07-30T16:35:35.207370Z","shell.execute_reply.started":"2023-07-30T16:34:16.151573Z","shell.execute_reply":"2023-07-30T16:35:35.206180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Some Samples","metadata":{}},{"cell_type":"code","source":"start_index = 10000\nend_index = start_index + 8\n\norig_audios, new_audios = [], []\ninput_list = []\npaths = []\nfor idx in range(start_index, end_index):\n    # Path\n    path = train.path.values[idx]\n    paths.append(path)\n    \n    # Audio\n    # load orignial audio\n    audio, sr = librosa.load(path, sr = None) \n    # resample\n    audio = librosa.resample(audio, orig_sr=sr, target_sr = config['sample_rate']) \n    orig_audios.append(audio)\n    \n    # normalize\n    audio = librosa.util.normalize(audio)\n    new_audios.append(audio)\n    \n    # processor\n    inputs = processor(audio = audio, sampling_rate = config['sample_rate'], return_tensors = 'pt')\n    input_list.append(inputs)\n    ","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:36:22.458559Z","iopub.execute_input":"2023-07-30T16:36:22.458960Z","iopub.status.idle":"2023-07-30T16:36:22.634785Z","shell.execute_reply.started":"2023-07-30T16:36:22.458927Z","shell.execute_reply":"2023-07-30T16:36:22.633678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ORIGINAL\nfor org in orig_audios:\n    print(f\"SHAPE: {org.shape} | MAX: {np.max(org)} | MIN: {np.mean(org)} | MEAN: {np.mean(org)}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:36:24.704644Z","iopub.execute_input":"2023-07-30T16:36:24.705065Z","iopub.status.idle":"2023-07-30T16:36:24.713262Z","shell.execute_reply.started":"2023-07-30T16:36:24.705029Z","shell.execute_reply":"2023-07-30T16:36:24.712133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# NEW\nfor org in new_audios:\n    print(f\"SHAPE: {org.shape} | MAX: {np.max(org)} | MIN: {np.mean(org)} | MEAN: {np.mean(org)}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:36:24.913717Z","iopub.execute_input":"2023-07-30T16:36:24.914935Z","iopub.status.idle":"2023-07-30T16:36:24.923454Z","shell.execute_reply.started":"2023-07-30T16:36:24.914886Z","shell.execute_reply":"2023-07-30T16:36:24.922270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_list[0].input_values.shape, input_list[0].input_values, input_list[0].keys()","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:36:26.474475Z","iopub.execute_input":"2023-07-30T16:36:26.475306Z","iopub.status.idle":"2023-07-30T16:36:26.581737Z","shell.execute_reply.started":"2023-07-30T16:36:26.475260Z","shell.execute_reply":"2023-07-30T16:36:26.580678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# After Single Processor\nfor pr in input_list:\n    print(f\"SHAPE: {pr.input_values.shape} | MAX: {torch.max(pr.input_values)}  | MIN: {torch.min(pr.input_values)} | MEAN: {torch.mean(pr.input_values)}\")\n    print(\"Only 10 floats: \", pr.input_values[0][:10])\n    print()","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:36:27.480077Z","iopub.execute_input":"2023-07-30T16:36:27.481047Z","iopub.status.idle":"2023-07-30T16:36:27.499444Z","shell.execute_reply.started":"2023-07-30T16:36:27.481009Z","shell.execute_reply":"2023-07-30T16:36:27.497888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Before Padding\ninput_list[:3]","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:36:43.102420Z","iopub.execute_input":"2023-07-30T16:36:43.102948Z","iopub.status.idle":"2023-07-30T16:36:43.116683Z","shell.execute_reply.started":"2023-07-30T16:36:43.102905Z","shell.execute_reply":"2023-07-30T16:36:43.115449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## After Padding\ninput_lists = [{'input_values': inputs['input_values'].squeeze(0)} for inputs in input_list]\nbatch = processor.pad(input_lists, padding= \"longest\", return_tensors=\"pt\") \nprint(\"SHAPE: \",batch['input_values'].shape)\nbatch['input_values']","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:36:43.967802Z","iopub.execute_input":"2023-07-30T16:36:43.968676Z","iopub.status.idle":"2023-07-30T16:36:44.013405Z","shell.execute_reply.started":"2023-07-30T16:36:43.968627Z","shell.execute_reply":"2023-07-30T16:36:44.012505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Sentence\n\nsentences = []\nfor idx in range(start_index, end_index):\n    # Path\n    s = train.sentence.values[idx]\n    print(\"Sentence: \", s)\n    sentences.append(s)","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:36:47.567537Z","iopub.execute_input":"2023-07-30T16:36:47.567961Z","iopub.status.idle":"2023-07-30T16:36:47.576466Z","shell.execute_reply.started":"2023-07-30T16:36:47.567928Z","shell.execute_reply":"2023-07-30T16:36:47.575008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = []\nfor s in sentences:\n    inputs = processor(text= s, return_tensors = 'pt').input_ids\n    print(\"SHAPE: \", inputs.shape)\n    print(\"INNPUT IDS: \", inputs[0][:10])\n    labels.append(inputs.squeeze(0))\n    print()","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:36:48.871110Z","iopub.execute_input":"2023-07-30T16:36:48.871588Z","iopub.status.idle":"2023-07-30T16:36:48.885765Z","shell.execute_reply.started":"2023-07-30T16:36:48.871539Z","shell.execute_reply":"2023-07-30T16:36:48.884558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label = [{'input_ids': inputs} for inputs in labels]\nlabels_batch = processor.pad(labels=label, padding=\"longest\", return_tensors=\"pt\")\nlabels_batch['input_ids'].shape","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:36:50.255973Z","iopub.execute_input":"2023-07-30T16:36:50.256999Z","iopub.status.idle":"2023-07-30T16:36:50.265016Z","shell.execute_reply.started":"2023-07-30T16:36:50.256962Z","shell.execute_reply":"2023-07-30T16:36:50.264242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(processor.tokenizer.pad_token_id)\nlabels_batch['input_ids'][0]","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:36:51.279642Z","iopub.execute_input":"2023-07-30T16:36:51.280268Z","iopub.status.idle":"2023-07-30T16:36:51.289436Z","shell.execute_reply.started":"2023-07-30T16:36:51.280234Z","shell.execute_reply":"2023-07-30T16:36:51.288230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = labels_batch[\"input_ids\"].masked_fill(labels_batch.attention_mask.ne(1), -100)\nlabels[0]","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:36:51.596796Z","iopub.execute_input":"2023-07-30T16:36:51.597982Z","iopub.status.idle":"2023-07-30T16:36:51.608656Z","shell.execute_reply.started":"2023-07-30T16:36:51.597941Z","shell.execute_reply":"2023-07-30T16:36:51.607796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels[labels == -100] = processor.tokenizer.pad_token_id\nlabels[0]","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:36:51.795426Z","iopub.execute_input":"2023-07-30T16:36:51.796385Z","iopub.status.idle":"2023-07-30T16:36:51.806900Z","shell.execute_reply.started":"2023-07-30T16:36:51.796348Z","shell.execute_reply":"2023-07-30T16:36:51.805850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sentences","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:36:53.505812Z","iopub.execute_input":"2023-07-30T16:36:53.506254Z","iopub.status.idle":"2023-07-30T16:36:53.513934Z","shell.execute_reply.started":"2023-07-30T16:36:53.506219Z","shell.execute_reply":"2023-07-30T16:36:53.512793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"processor.tokenizer.batch_decode(labels.numpy())","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:36:53.727953Z","iopub.execute_input":"2023-07-30T16:36:53.728400Z","iopub.status.idle":"2023-07-30T16:36:53.738869Z","shell.execute_reply.started":"2023-07-30T16:36:53.728361Z","shell.execute_reply":"2023-07-30T16:36:53.737605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# We needs:\n - Text Preprocessing\n - [Detailed EDA, Normalizer and WER](https://www.kaggle.com/code/mbmmurad/detailed-eda-normalizer-and-wer) \n","metadata":{}},{"cell_type":"code","source":"%%capture\n%pip install git+https://github.com/csebuetnlp/normalizer","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:36:55.620727Z","iopub.execute_input":"2023-07-30T16:36:55.621458Z","iopub.status.idle":"2023-07-30T16:37:18.297652Z","shell.execute_reply.started":"2023-07-30T16:36:55.621410Z","shell.execute_reply":"2023-07-30T16:37:18.296059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We can these two words are the same, but python says different\nprint(\"নিয়ে\" == \"নিয়ে\")\nprint(\"হয়ে\" == \"হয়ে\")\nprint(\"চিড়ে\"==\"চিড়ে\")","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:37:18.300136Z","iopub.execute_input":"2023-07-30T16:37:18.300599Z","iopub.status.idle":"2023-07-30T16:37:18.308358Z","shell.execute_reply.started":"2023-07-30T16:37:18.300546Z","shell.execute_reply":"2023-07-30T16:37:18.306942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Not one Word, I guess\nprint(list(\"নিয়ে\"))\nprint(list(\"নিয়ে\"))","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:37:18.309952Z","iopub.execute_input":"2023-07-30T16:37:18.310332Z","iopub.status.idle":"2023-07-30T16:37:18.323935Z","shell.execute_reply.started":"2023-07-30T16:37:18.310303Z","shell.execute_reply":"2023-07-30T16:37:18.322981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\nfrom normalizer import normalize\n\nsentence = \"এমনকি উকুন ঘরবাড়ি ও খাদ্য-সম্ভারের উপর ছড়িয়ে পড়তে লাগল।\"\n\nnormalized = normalize(sentence)\nprint(sentence)\nprint(normalized)\nprint(normalize==sentence) ## not working in sentence level?","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:37:18.326722Z","iopub.execute_input":"2023-07-30T16:37:18.327290Z","iopub.status.idle":"2023-07-30T16:37:18.548285Z","shell.execute_reply.started":"2023-07-30T16:37:18.327246Z","shell.execute_reply":"2023-07-30T16:37:18.547005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Word Level?\nnormalize(\"নিয়ে\")==normalize(\"নিয়ে\")","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:37:18.550578Z","iopub.execute_input":"2023-07-30T16:37:18.551056Z","iopub.status.idle":"2023-07-30T16:37:18.558782Z","shell.execute_reply.started":"2023-07-30T16:37:18.551009Z","shell.execute_reply":"2023-07-30T16:37:18.557574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_sentence = [normalize(token.strip()) for token in sentence.split()]\nprint(sentence.split())\nprint(new_sentence)\n\nfor word1, word2 in zip(new_sentence, sentence.split()):\n    print(word1 == word2.strip())","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:37:18.560571Z","iopub.execute_input":"2023-07-30T16:37:18.561010Z","iopub.status.idle":"2023-07-30T16:37:18.573056Z","shell.execute_reply.started":"2023-07-30T16:37:18.560971Z","shell.execute_reply":"2023-07-30T16:37:18.571634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## function\ndef clean_text(sentence):\n    sentence = [normalize(s.strip()) for s in sentence.split()]\n    sentence = \" \".join(sentence)\n    text_cleaned = re.sub('[-=+,#/\\?:^.@*\\\"※~ㆍ!』‘|\\(\\)\\[\\]`\\'…》\\”\\“\\’·]', '', sentence)\n    return text_cleaned","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:37:18.574948Z","iopub.execute_input":"2023-07-30T16:37:18.575498Z","iopub.status.idle":"2023-07-30T16:37:18.584196Z","shell.execute_reply.started":"2023-07-30T16:37:18.575455Z","shell.execute_reply":"2023-07-30T16:37:18.583038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sentences","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:37:18.586427Z","iopub.execute_input":"2023-07-30T16:37:18.586907Z","iopub.status.idle":"2023-07-30T16:37:18.598742Z","shell.execute_reply.started":"2023-07-30T16:37:18.586843Z","shell.execute_reply":"2023-07-30T16:37:18.597540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cleaned = [clean_text(sentence) for sentence in sentences]\ncleaned","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:37:18.600723Z","iopub.execute_input":"2023-07-30T16:37:18.601180Z","iopub.status.idle":"2023-07-30T16:37:18.622326Z","shell.execute_reply.started":"2023-07-30T16:37:18.601140Z","shell.execute_reply":"2023-07-30T16:37:18.621078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Cleaned Version\nlabels = [processor(text= s, return_tensors = 'pt').input_ids.squeeze(0) for s in cleaned]\nlabel = [{'input_ids': inputs} for inputs in labels]\nlabels_batch = processor.pad(labels=label, padding=\"longest\", return_tensors=\"pt\")\nlabels = labels_batch[\"input_ids\"].masked_fill(labels_batch.attention_mask.ne(1), -100)\nlabels[labels == -100] = processor.tokenizer.pad_token_id\nprocessor.tokenizer.batch_decode(labels.numpy())","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:37:18.625380Z","iopub.execute_input":"2023-07-30T16:37:18.625712Z","iopub.status.idle":"2023-07-30T16:37:18.645343Z","shell.execute_reply.started":"2023-07-30T16:37:18.625682Z","shell.execute_reply":"2023-07-30T16:37:18.644155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted = processor.tokenizer.batch_decode(labels.numpy())\npredicted ","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:56:39.777732Z","iopub.execute_input":"2023-07-30T16:56:39.778160Z","iopub.status.idle":"2023-07-30T16:56:39.788262Z","shell.execute_reply.started":"2023-07-30T16:56:39.778129Z","shell.execute_reply":"2023-07-30T16:56:39.787187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Compare raw sentences without anything\n\n# ['আপনার সম্পর্কে আমার খুব জানতে ইচ্ছে করে।',\n#  'আগে তুই এটা ঠিক কর তোর রাগটা কীসের ওপর[UNK]',\n#  'প্রশাসনের সাথে লড়াইয়ের পর[UNK] তিনি বিশ্বিদ্যালয়ের সাহিত্য ও মানব বিজ্ঞান বিভাগের ডিন হিসেবে নিযুক্ত হন।',\n#  'পরবর্তীতে তিনি অন্যান্য জ্বালানি যেমন উদ্ভিজ্জ তেল নিয়েও পরীক্ষা চালান।',\n#  'যুক্তরাষ্ট্র ও ইউরোপে এর হার উচ্চতর।',\n#  'পূর্বে যদি আপনি এই মসজিদের সমালোচনা করেন তাহলে আপনি নাৎসি ছিলেন।',\n#  'এবার পরিবেশন পাত্রে বেড়ে ওপরে বেরেস্তা ছিটিয়ে গরম গরম পরিবেশন করুন।',\n#  'শুধু তা[UNK]ই নয়[UNK] অভিনয় থেকেও খুব শিগিরই অবসর নিতে চান তিনি।']","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# WER from evaluate ","metadata":{}},{"cell_type":"code","source":"import evaluate\n\n# evaluate : WER\nwer = evaluate.load(\"wer\")\n\nwer_score = wer.compute(predictions=predicted, references=sentences)\nwer_score","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:56:42.221693Z","iopub.execute_input":"2023-07-30T16:56:42.222085Z","iopub.status.idle":"2023-07-30T16:56:42.779251Z","shell.execute_reply.started":"2023-07-30T16:56:42.222054Z","shell.execute_reply":"2023-07-30T16:56:42.778166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wer_score1 = wer.compute(predictions=predicted, references=cleaned)\nwer_score1","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:56:58.661102Z","iopub.execute_input":"2023-07-30T16:56:58.661561Z","iopub.status.idle":"2023-07-30T16:56:58.678370Z","shell.execute_reply.started":"2023-07-30T16:56:58.661529Z","shell.execute_reply":"2023-07-30T16:56:58.677257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Clean, Normalize Text and Save","metadata":{}},{"cell_type":"code","source":"## Clean + Normalize \ntrain['new_sentence'] = train.sentence.apply(lambda x: clean_text(x))\nprint(train.shape)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:37:27.332687Z","iopub.execute_input":"2023-07-30T16:37:27.333100Z","iopub.status.idle":"2023-07-30T16:45:03.939724Z","shell.execute_reply.started":"2023-07-30T16:37:27.333064Z","shell.execute_reply":"2023-07-30T16:45:03.938561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.drop(['id'], axis = 1, inplace = True)\nprint(train.shape)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:45:49.366537Z","iopub.execute_input":"2023-07-30T16:45:49.367503Z","iopub.status.idle":"2023-07-30T16:45:49.574393Z","shell.execute_reply.started":"2023-07-30T16:45:49.367458Z","shell.execute_reply":"2023-07-30T16:45:49.573267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 'new_sentence': Cleaned + Normalized Column\ntrain.to_csv(\"train_cleaned_normalized.csv\", index = False)","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:45:55.324892Z","iopub.execute_input":"2023-07-30T16:45:55.325314Z","iopub.status.idle":"2023-07-30T16:46:08.052736Z","shell.execute_reply.started":"2023-07-30T16:45:55.325282Z","shell.execute_reply":"2023-07-30T16:46:08.051180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Normalized Column function\ndef normalize_text(sentence):\n    sentence = [normalize(s.strip()) for s in sentence.split()]\n    sentence = \" \".join(sentence)\n    return sentence","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:46:10.824171Z","iopub.execute_input":"2023-07-30T16:46:10.824625Z","iopub.status.idle":"2023-07-30T16:46:10.831191Z","shell.execute_reply.started":"2023-07-30T16:46:10.824588Z","shell.execute_reply":"2023-07-30T16:46:10.830034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['normal_sentence'] = train.sentence.apply(lambda x: normalize_text(x))\nprint(train.shape)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:46:23.538103Z","iopub.execute_input":"2023-07-30T16:46:23.538602Z","iopub.status.idle":"2023-07-30T16:53:54.394719Z","shell.execute_reply.started":"2023-07-30T16:46:23.538570Z","shell.execute_reply":"2023-07-30T16:53:54.393626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 'normal_sentence': Normalized Column\ntrain.to_csv(\"train_cleaned_normalized.csv\", index = False)","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:53:54.396705Z","iopub.execute_input":"2023-07-30T16:53:54.397016Z","iopub.status.idle":"2023-07-30T16:54:11.047614Z","shell.execute_reply.started":"2023-07-30T16:53:54.396989Z","shell.execute_reply":"2023-07-30T16:54:11.046671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Only 'normal_sentence' column\ntrain = train.drop(['new_sentence'], axis = 1,)\nprint(train.shape)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:57:39.792800Z","iopub.execute_input":"2023-07-30T16:57:39.793239Z","iopub.status.idle":"2023-07-30T16:57:39.937920Z","shell.execute_reply.started":"2023-07-30T16:57:39.793179Z","shell.execute_reply":"2023-07-30T16:57:39.936832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.to_csv(\"train_normalized.csv\", index = False)","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:59:41.016771Z","iopub.execute_input":"2023-07-30T16:59:41.017244Z","iopub.status.idle":"2023-07-30T16:59:55.578902Z","shell.execute_reply.started":"2023-07-30T16:59:41.017182Z","shell.execute_reply":"2023-07-30T16:59:55.577476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%ls","metadata":{"execution":{"iopub.status.busy":"2023-07-30T17:00:16.947316Z","iopub.execute_input":"2023-07-30T17:00:16.947805Z","iopub.status.idle":"2023-07-30T17:00:18.101098Z","shell.execute_reply.started":"2023-07-30T17:00:16.947770Z","shell.execute_reply":"2023-07-30T17:00:18.099552Z"},"trusted":true},"execution_count":null,"outputs":[]}]}