{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":52324,"databundleVersionId":6229904,"sourceType":"competition"},{"sourceId":4142276,"sourceType":"datasetVersion","datasetId":2446557},{"sourceId":6465877,"sourceType":"datasetVersion","datasetId":3729212},{"sourceId":6517607,"sourceType":"datasetVersion","datasetId":3767700},{"sourceId":6723801,"sourceType":"datasetVersion","datasetId":3873607},{"sourceId":6639409,"sourceType":"datasetVersion","datasetId":2433942,"isSourceIdPinned":true}],"dockerImageVersionId":30214,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Team AudioAlchemists - Inference Notebook - Wav2Vec2\n\n- Team Members: `Syed Mostofa Monsur Dipto` | `Sakib Chowdhury` | `Md Boktiar Mahbub Murad` |","metadata":{}},{"cell_type":"markdown","source":"# Weights","metadata":{}},{"cell_type":"markdown","source":"1. [Model weights](https://www.kaggle.com/datasets/mbmmurad/checkpoint-450000ind2-aug)\n2. [Language Model](https://www.kaggle.com/datasets/mbmmurad/final-lm-benai/)\n3. [Dependencies](https://www.kaggle.com/datasets/shahruk10/csefest2022dlsprintdeps/versions/9)","metadata":{}},{"cell_type":"markdown","source":"# Install dependencies","metadata":{}},{"cell_type":"code","source":"!ls ","metadata":{"execution":{"iopub.status.busy":"2023-11-16T09:34:50.823186Z","iopub.execute_input":"2023-11-16T09:34:50.823589Z","iopub.status.idle":"2023-11-16T09:34:51.800294Z","shell.execute_reply.started":"2023-11-16T09:34:50.823508Z","shell.execute_reply":"2023-11-16T09:34:51.798815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp -r ../input/csefest2022dlsprintdeps ./deps","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2023-11-16T09:36:37.069457Z","iopub.execute_input":"2023-11-16T09:36:37.070181Z","iopub.status.idle":"2023-11-16T09:36:38.215212Z","shell.execute_reply.started":"2023-11-16T09:36:37.070144Z","shell.execute_reply":"2023-11-16T09:36:38.213897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install ./deps/pygtrie-2.5.0/pygtrie-2.5.0\n!pip install ./deps/exceptiongroup-1.0.0rc8-py3-none-any.whl\n!pip install ./deps/hypothesis-6.54.4-py3-none-any.whl\n!pip install ./deps/pyctcdecode-0.4.0-py2.py3-none-any.whl\n!pip install ./deps/pypi-kenlm-0.1.20220713/pypi-kenlm-0.1.20220713\n!pip install ./deps/bnunicodenormalizer-0.0.23/bnunicodenormalizer-0.0.23\n!pip install ./deps/python-Levenshtein-0.12.2/python-Levenshtein-0.12.2\n!pip install ./deps/jiwer-2.3.0-py3-none-any.whl\n\n!chmod +x ./deps/kenlm/kenlm/bin/lmplz","metadata":{"execution":{"iopub.status.busy":"2023-11-16T09:36:54.988764Z","iopub.execute_input":"2023-11-16T09:36:54.989159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"from typing import Dict, List, Tuple, Any, Union\n\nimport os\nimport re\nimport json\nimport glob\nfrom tqdm.notebook import tqdm\n\nimport numpy as np\nimport pandas as pd\n\nfrom transformers import Wav2Vec2ForCTC, Wav2Vec2CTCTokenizer, Wav2Vec2FeatureExtractor, Wav2Vec2Processor\nfrom pyctcdecode import build_ctcdecoder\n\nimport torch\nimport torchaudio\nimport torchaudio.functional as F\nimport torchaudio.transforms as T\nfrom torch.utils.data import Dataset, DataLoader, IterableDataset\n\nimport pyctcdecode\n\nfrom bnunicodenormalizer import Normalizer \nfrom datasets import load_metric\n\nbnorm = Normalizer()\nwer = load_metric(\"../input/csefest2022dlsprintdeps/metrics/metrics/wer.py\")\ncer = load_metric(\"../input/csefest2022dlsprintdeps/metrics/metrics/cer.py\")","metadata":{"execution":{"iopub.status.busy":"2023-11-16T09:36:11.800532Z","iopub.status.idle":"2023-11-16T09:36:11.801291Z","shell.execute_reply.started":"2023-11-16T09:36:11.800920Z","shell.execute_reply":"2023-11-16T09:36:11.800954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vocabPath = '/kaggle/input/ckpt-270k/check_de_model/vocab.json'\nlmPath = \"/kaggle/input/final-lm-benai/5gram.arpa\"\nckptPath = \"/kaggle/input/checkpoint-450000ind2-aug\"\n\nsampleSubmissionPath = '/kaggle/input/bengaliai-speech/sample_submission.csv'\n\ntestDataDir = '../input/dlsprint/test_files'\n\nctcdecoderAlpha = 0.60\nctcdecoderBeta = 0.25\n\nrunName = f\"run_004_ckpt_40000_kenlm_5gram_with_discount_fallback_with_dev_alpha{ctcdecoderAlpha}_beta{ctcdecoderBeta}\"","metadata":{"execution":{"iopub.status.busy":"2023-11-16T09:36:11.803607Z","iopub.status.idle":"2023-11-16T09:36:11.804324Z","shell.execute_reply.started":"2023-11-16T09:36:11.803951Z","shell.execute_reply":"2023-11-16T09:36:11.803982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Model","metadata":{}},{"cell_type":"code","source":"processor = Wav2Vec2Processor.from_pretrained(\"/kaggle/input/ckpt-270k/check_de_model\")","metadata":{"execution":{"iopub.status.busy":"2023-11-16T09:36:11.806033Z","iopub.status.idle":"2023-11-16T09:36:11.806773Z","shell.execute_reply.started":"2023-11-16T09:36:11.806374Z","shell.execute_reply":"2023-11-16T09:36:11.806408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vocab_dict = processor.tokenizer.get_vocab()\nsorted_vocab_dict = {k: v for k, v in sorted(vocab_dict.items(), key=lambda item: item[1])}\n\ndecoder = pyctcdecode.build_ctcdecoder(\n    list(sorted_vocab_dict.keys()),\n    lmPath,\n)","metadata":{"execution":{"iopub.status.busy":"2023-11-16T09:36:11.809295Z","iopub.status.idle":"2023-11-16T09:36:11.810065Z","shell.execute_reply.started":"2023-11-16T09:36:11.809677Z","shell.execute_reply":"2023-11-16T09:36:11.809711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(vocabPath, 'r', encoding=\"utf-8\") as vocabFile:\n    vocabc2n = json.load(vocabFile)\n\nvocabn2c = { v:k for k,v in vocabc2n.items() }\nvocab = list(vocabc2n.keys())\n\ntokenizer = processor.tokenizer\n\nfeature_extractor = processor.feature_extractor\n\n\nmodel = Wav2Vec2ForCTC.from_pretrained(ckptPath)\nmodel.cuda()\nmodel.eval()","metadata":{"execution":{"iopub.status.busy":"2023-11-16T09:36:11.812062Z","iopub.status.idle":"2023-11-16T09:36:11.812783Z","shell.execute_reply.started":"2023-11-16T09:36:11.812408Z","shell.execute_reply":"2023-11-16T09:36:11.812441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv(sampleSubmissionPath)\ntest_df['id'] = [ os.path.join(testDataDir, x) for x in test_df['id'] ]","metadata":{"execution":{"iopub.status.busy":"2023-11-16T09:36:11.814721Z","iopub.status.idle":"2023-11-16T09:36:11.815423Z","shell.execute_reply.started":"2023-11-16T09:36:11.815053Z","shell.execute_reply":"2023-11-16T09:36:11.815081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utilities","metadata":{}},{"cell_type":"code","source":"class AudioConverter:\n    \"\"\"\n    AudioConverter offers methods to load, transcode and augment\n    audio data in various ways.\n    \"\"\"\n\n    # Configurations for parameters used in torchaudio's resampling kernel.\n    resampleFilterParams = {\n        \"fast\": {  # Fast and less accurate but still MSE = ~2e-5 compared to librosa.\n            \"lowpass_filter_width\": 16,\n            \"rolloff\": 0.85,\n            \"resampling_method\": \"kaiser_window\",\n            \"beta\": 8.555504641634386,\n        },\n        \"best\": { # Twice as slow, and a little bit more accurate.\n            \"lowpass_filter_width\": 64,\n            \"rolloff\": 0.9475937167399596,\n            \"resampling_method\": \"kaiser_window\",\n            \"beta\": 14.769656459379492,       \n        },\n    }\n\n    def __init__(\n        self,\n        sampleRate: int,\n    ):\n        \"\"\"\n        Initializes AudioConverter.\n\n        Parameters\n        ----------\n        sampleRate: int\n            Sampling rate to convert audio to, if required.\n        \"\"\"\n        self.sampleRate = sampleRate\n\n    @classmethod\n    def loadAudio(\n        cls, audioPath: str, sampleRate: int = None, returnTensor: bool = True, resampleType: str = \"fast\",\n    ) -> Union[torch.Tensor, np.ndarray]:\n        \"\"\"\n        Uses torchaudio to load and resample (if necessary) audio files and returns\n        audio samples as either a numpy.float32 array or a torch.Tensor.\n        \n        Parameters\n        ----------\n        audioPath: str\n            Path to audio file file (wav / mp3 / flac).\n        \n        sampleRate: int, optional\n            Sampling rate to convert audio to. If None,\n            audio is not resampled.\n        \n        returnTensor: bool, optional\n            If True, the audio samples are returned as a torch.Tensor.\n            Otherwise, the samples are returned as a numpy.float32 array.\n            \n        resampleType: str, optional\n            Either \"fast\" or \"best\" - sets the quality of resampling.\n            \"best\" is twice as slow as \"fast\" but more accurate. \"fast\"\n            is still comparable to librosa's resampled output though,\n            in terms of MSE.\n\n        Returns\n        -------\n        Union[torch.Tensor, np.ndarray]\n            Audio waveform scaled between +/- 1.0 as either a numpy.float32 array,\n            or torch.Tensor, with shape (channels, numSamples)\n        \"\"\"\n        x, sr = torchaudio.load(audioPath)\n        if sampleRate is not None or sr != sampleRate:\n            x = F.resample(x, sr, sampleRate)\n        \n        if returnTensor:\n            return x\n        \n        return x.numpy()","metadata":{"execution":{"iopub.status.busy":"2023-11-16T09:36:11.817707Z","iopub.status.idle":"2023-11-16T09:36:11.818232Z","shell.execute_reply.started":"2023-11-16T09:36:11.817956Z","shell.execute_reply":"2023-11-16T09:36:11.817980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def normalize(sen):\n    _words = [ bnorm(word)['normalized']  for word in sen.split() ]\n    sen = \" \".join([word for word in _words if word is not None]) \n    sen = sen.replace(\"\\u2047\", \"-\")\n\n    return sen","metadata":{"execution":{"iopub.status.busy":"2023-11-16T09:36:11.819608Z","iopub.status.idle":"2023-11-16T09:36:11.820099Z","shell.execute_reply.started":"2023-11-16T09:36:11.819858Z","shell.execute_reply":"2023-11-16T09:36:11.819882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Infer on single data","metadata":{}},{"cell_type":"code","source":"def infer(audioPath):\n    wav = AudioConverter.loadAudio(audioPath, sampleRate=16000, returnTensor=False)[0]\n    inputs = processor(wav, sampling_rate=16000, return_tensors=\"pt\", padding=True)\n\n    with torch.no_grad():\n        logits = model(inputs.input_values.float().cuda()).logits\n   \n    items = logits.squeeze(0).cpu().numpy()\n    preds = decoder.decode_beams(items)[0][0]\n\n    return normalize(preds) + \"।\"\n\n\ninfer('/kaggle/input/bengaliai-speech/train_mp3s/0001565ed181.mp3')","metadata":{"execution":{"iopub.status.busy":"2023-11-16T09:36:11.821415Z","iopub.status.idle":"2023-11-16T09:36:11.821908Z","shell.execute_reply.started":"2023-11-16T09:36:11.821654Z","shell.execute_reply":"2023-11-16T09:36:11.821679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Infer on a batch of data","metadata":{}},{"cell_type":"code","source":"def batch_infer(audio_paths, batch_size):\n    preds = [ infer(x) for x in tqdm(audio_paths) ]\n    return preds","metadata":{"execution":{"iopub.status.busy":"2023-11-16T09:36:11.822991Z","iopub.status.idle":"2023-11-16T09:36:11.823504Z","shell.execute_reply.started":"2023-11-16T09:36:11.823228Z","shell.execute_reply":"2023-11-16T09:36:11.823251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Infer on a directory","metadata":{}},{"cell_type":"code","source":"def directory_infer(audio_dir):\n    audio_paths = sorted(glob.glob(audio_dir+'/*'))\n    preds = batch_infer(audio_paths, 10)\n    base_paths=[ os.path.basename(p) for p in audio_paths ]\n    \n    return pd.DataFrame({'id': base_paths ,'sentence': preds})","metadata":{"execution":{"iopub.status.busy":"2023-11-16T09:36:11.825253Z","iopub.status.idle":"2023-11-16T09:36:11.825761Z","shell.execute_reply.started":"2023-11-16T09:36:11.825499Z","shell.execute_reply":"2023-11-16T09:36:11.825523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"test_dir = \"/kaggle/input/bengaliai-speech/test_mp3s\"\nsub = directory_infer(test_dir)\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-16T09:36:11.827153Z","iopub.status.idle":"2023-11-16T09:36:11.827673Z","shell.execute_reply.started":"2023-11-16T09:36:11.827401Z","shell.execute_reply":"2023-11-16T09:36:11.827424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub['id'] = sub['id'].apply(lambda x:x.split(\".\")[0])\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-16T09:36:11.828901Z","iopub.status.idle":"2023-11-16T09:36:11.829389Z","shell.execute_reply.started":"2023-11-16T09:36:11.829130Z","shell.execute_reply":"2023-11-16T09:36:11.829155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv(\"submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2023-11-16T09:36:11.830921Z","iopub.status.idle":"2023-11-16T09:36:11.831315Z","shell.execute_reply.started":"2023-11-16T09:36:11.831090Z","shell.execute_reply":"2023-11-16T09:36:11.831105Z"},"trusted":true},"execution_count":null,"outputs":[]}]}