{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## About\n\nThis notebook calculates valid WER score for models used in the submission: [Bengali-SR: Public Wav2vec2.0 w/ LM Baseline](https://www.kaggle.com/code/ttahara/bengali-sr-public-wav2vec2-0-w-lm-baseline). ","metadata":{}},{"cell_type":"markdown","source":"## Import","metadata":{}},{"cell_type":"code","source":"!pip install jiwer\n!pip install bnunicodenormalizer\n!pip install pyctcdecode\n!pip install kenlm","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-08-19T02:49:57.109419Z","iopub.execute_input":"2023-08-19T02:49:57.109787Z","iopub.status.idle":"2023-08-19T02:51:13.847601Z","shell.execute_reply.started":"2023-08-19T02:49:57.109756Z","shell.execute_reply":"2023-08-19T02:51:13.846431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import typing as tp\nfrom pathlib import Path\nfrom functools import partial\nfrom dataclasses import dataclass, field\n\nimport pandas as pd\nimport pyctcdecode\nimport numpy as np\nfrom tqdm.notebook import tqdm\n\nimport librosa\n\nimport jiwer\nimport pyctcdecode\nimport kenlm\nimport torch\nfrom transformers import Wav2Vec2Processor, Wav2Vec2ProcessorWithLM, Wav2Vec2ForCTC\nfrom bnunicodenormalizer import Normalizer","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-08-19T02:54:08.977089Z","iopub.execute_input":"2023-08-19T02:54:08.977448Z","iopub.status.idle":"2023-08-19T02:54:08.983979Z","shell.execute_reply.started":"2023-08-19T02:54:08.977419Z","shell.execute_reply":"2023-08-19T02:54:08.982550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT = Path.cwd().parent\nINPUT = ROOT / \"input\"\nDATA = INPUT / \"bengaliai-speech\"\nTRAIN = DATA / \"train_mp3s\"\nTEST = DATA / \"test_mp3s\"\n\nSAMPLING_RATE = 16_000\nMODEL_PATH = INPUT / \"bengali-sr-download-public-trained-models/indicwav2vec_v1_bengali/\"\nLM_PATH = INPUT / \"bengali-sr-download-public-trained-models/wav2vec2-xls-r-300m-bengali/language_model/\"","metadata":{"execution":{"iopub.status.busy":"2023-08-19T02:54:11.563276Z","iopub.execute_input":"2023-08-19T02:54:11.563642Z","iopub.status.idle":"2023-08-19T02:54:11.569827Z","shell.execute_reply.started":"2023-08-19T02:54:11.563611Z","shell.execute_reply":"2023-08-19T02:54:11.568718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### load model, processor, decoder","metadata":{}},{"cell_type":"code","source":"model = Wav2Vec2ForCTC.from_pretrained(MODEL_PATH)\nprocessor = Wav2Vec2Processor.from_pretrained(MODEL_PATH)","metadata":{"execution":{"iopub.status.busy":"2023-08-19T02:54:12.340013Z","iopub.execute_input":"2023-08-19T02:54:12.340370Z","iopub.status.idle":"2023-08-19T02:54:29.151546Z","shell.execute_reply.started":"2023-08-19T02:54:12.340341Z","shell.execute_reply":"2023-08-19T02:54:29.150559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vocab_dict = processor.tokenizer.get_vocab()\nsorted_vocab_dict = {k: v for k, v in sorted(vocab_dict.items(), key=lambda item: item[1])}\n\ndecoder = pyctcdecode.build_ctcdecoder(\n    list(sorted_vocab_dict.keys()),\n    str(LM_PATH / \"5gram.bin\"),\n)","metadata":{"execution":{"iopub.status.busy":"2023-08-19T02:54:29.153612Z","iopub.execute_input":"2023-08-19T02:54:29.154008Z","iopub.status.idle":"2023-08-19T02:55:09.007803Z","shell.execute_reply.started":"2023-08-19T02:54:29.153973Z","shell.execute_reply":"2023-08-19T02:55:09.006660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"processor_with_lm = Wav2Vec2ProcessorWithLM(\n    feature_extractor=processor.feature_extractor,\n    tokenizer=processor.tokenizer,\n    decoder=decoder\n)","metadata":{"execution":{"iopub.status.busy":"2023-08-19T02:55:09.010764Z","iopub.execute_input":"2023-08-19T02:55:09.011115Z","iopub.status.idle":"2023-08-19T02:55:09.022891Z","shell.execute_reply.started":"2023-08-19T02:55:09.011088Z","shell.execute_reply":"2023-08-19T02:55:09.021866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## prepare dataloader","metadata":{}},{"cell_type":"code","source":"class BengaliSRTestDataset(torch.utils.data.Dataset):\n    \n    def __init__(\n        self,\n        audio_paths: list[str],\n        sampling_rate: int\n    ):\n        self.audio_paths = audio_paths\n        self.sampling_rate = sampling_rate\n        \n    def __len__(self,):\n        return len(self.audio_paths)\n    \n    def __getitem__(self, index: int):\n        audio_path = self.audio_paths[index]\n        sr = self.sampling_rate\n        w = librosa.load(audio_path, sr=sr, mono=False)[0]\n        \n        return w","metadata":{"execution":{"iopub.status.busy":"2023-08-19T02:55:09.025822Z","iopub.execute_input":"2023-08-19T02:55:09.026271Z","iopub.status.idle":"2023-08-19T02:55:09.035655Z","shell.execute_reply.started":"2023-08-19T02:55:09.026240Z","shell.execute_reply":"2023-08-19T02:55:09.034687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid = pd.read_csv(DATA / \"train.csv\", dtype={\"id\": str}).query(\"split == 'valid'\").reset_index(drop=True)\nprint(valid.head())","metadata":{"execution":{"iopub.status.busy":"2023-08-19T02:57:37.363788Z","iopub.execute_input":"2023-08-19T02:57:37.364310Z","iopub.status.idle":"2023-08-19T02:57:41.000756Z","shell.execute_reply.started":"2023-08-19T02:57:37.364270Z","shell.execute_reply":"2023-08-19T02:57:40.998822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_audio_paths = [str(TRAIN / f\"{aid}.mp3\") for aid in valid[\"id\"].values]","metadata":{"execution":{"iopub.status.busy":"2023-08-19T02:57:41.003338Z","iopub.execute_input":"2023-08-19T02:57:41.004269Z","iopub.status.idle":"2023-08-19T02:57:41.244729Z","shell.execute_reply.started":"2023-08-19T02:57:41.004230Z","shell.execute_reply":"2023-08-19T02:57:41.243773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_dataset = BengaliSRTestDataset(\n    valid_audio_paths, SAMPLING_RATE\n)\n\ncollate_func = partial(\n    processor_with_lm.feature_extractor,\n    return_tensors=\"pt\", sampling_rate=SAMPLING_RATE,\n    padding=True,\n)\n\nvalid_loader = torch.utils.data.DataLoader(\n    valid_dataset, batch_size=8, shuffle=False,\n    num_workers=2, collate_fn=collate_func, drop_last=False,\n    pin_memory=True,\n)","metadata":{"execution":{"iopub.status.busy":"2023-08-19T02:57:41.246255Z","iopub.execute_input":"2023-08-19T02:57:41.246591Z","iopub.status.idle":"2023-08-19T02:57:41.252741Z","shell.execute_reply.started":"2023-08-19T02:57:41.246559Z","shell.execute_reply":"2023-08-19T02:57:41.251837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inference","metadata":{}},{"cell_type":"code","source":"if not torch.cuda.is_available():\n    device = torch.device(\"cpu\")\nelse:\n    device = torch.device(\"cuda\")\nprint(device)","metadata":{"execution":{"iopub.status.busy":"2023-08-19T02:57:41.255155Z","iopub.execute_input":"2023-08-19T02:57:41.255793Z","iopub.status.idle":"2023-08-19T02:57:41.264137Z","shell.execute_reply.started":"2023-08-19T02:57:41.255759Z","shell.execute_reply":"2023-08-19T02:57:41.263204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = model.to(device)\nmodel = model.eval()\nmodel = model.half()","metadata":{"execution":{"iopub.status.busy":"2023-08-19T02:57:41.265426Z","iopub.execute_input":"2023-08-19T02:57:41.266002Z","iopub.status.idle":"2023-08-19T02:57:41.288955Z","shell.execute_reply.started":"2023-08-19T02:57:41.265966Z","shell.execute_reply":"2023-08-19T02:57:41.288187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_sentence_list = []\n\nwith torch.no_grad():\n    for batch in tqdm(valid_loader):\n        x = batch[\"input_values\"]\n        x = x.to(device, non_blocking=True)\n        with torch.cuda.amp.autocast(True):\n            y = model(x).logits\n        y = y.detach().cpu().numpy()\n        \n        for l in y:  \n            sentence = processor_with_lm.decode(l, beam_width=512).text\n            pred_sentence_list.append(sentence)","metadata":{"execution":{"iopub.status.busy":"2023-08-19T02:57:41.290152Z","iopub.execute_input":"2023-08-19T02:57:41.290541Z","iopub.status.idle":"2023-08-19T02:58:22.721561Z","shell.execute_reply.started":"2023-08-19T02:57:41.290509Z","shell.execute_reply":"2023-08-19T02:58:22.719485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Check Valid WER score","metadata":{}},{"cell_type":"code","source":"bnorm = Normalizer()\n\ndef postprocess(sentence):\n    period_set = set([\".\", \"?\", \"!\", \"।\"])\n    _words = [bnorm(word)['normalized']  for word in sentence.split()]\n    sentence = \" \".join([word for word in _words if word is not None])\n    try:\n        if sentence[-1] not in period_set:\n            sentence+=\"।\"\n    except:\n        # print(sentence)\n        sentence = \"।\"\n    return sentence","metadata":{"execution":{"iopub.status.busy":"2023-08-19T02:58:22.723099Z","iopub.status.idle":"2023-08-19T02:58:22.723592Z","shell.execute_reply.started":"2023-08-19T02:58:22.723344Z","shell.execute_reply":"2023-08-19T02:58:22.723376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pp_pred_sentence_list = [\n    postprocess(s) for s in tqdm(pred_sentence_list)]","metadata":{"execution":{"iopub.status.busy":"2023-08-19T02:58:22.725278Z","iopub.status.idle":"2023-08-19T02:58:22.725753Z","shell.execute_reply.started":"2023-08-19T02:58:22.725513Z","shell.execute_reply":"2023-08-19T02:58:22.725537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid[\"pred_sentence\"] = pp_pred_sentence_list\nvalid[\"wer\"] = [\n    jiwer.wer(s, p_s)\n    for s, p_s in tqdm(valid[[\"sentence\", \"pred_sentence\"]].values)\n]\n\nprint(valid.head())","metadata":{"execution":{"iopub.status.busy":"2023-08-19T02:58:22.727219Z","iopub.status.idle":"2023-08-19T02:58:22.728260Z","shell.execute_reply.started":"2023-08-19T02:58:22.728004Z","shell.execute_reply":"2023-08-19T02:58:22.728034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(valid[\"wer\"].mean())","metadata":{"execution":{"iopub.status.busy":"2023-08-19T02:58:22.729647Z","iopub.status.idle":"2023-08-19T02:58:22.730435Z","shell.execute_reply.started":"2023-08-19T02:58:22.730177Z","shell.execute_reply":"2023-08-19T02:58:22.730201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## EOF","metadata":{}}]}