{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This Notebook is a fork of [this notebook](https://www.kaggle.com/code/hengck23/help-why-submission-score-error) which uses the [ai4bharat/indicwav2vec_v1_bengali](https://huggingface.co/ai4bharat/indicwav2vec_v1_bengali) model to infer on the test set.\n> It scored **0.52 on the LB** and **0.40 on the CV**","metadata":{"execution":{"iopub.status.busy":"2023-07-21T12:21:10.958630Z","iopub.execute_input":"2023-07-21T12:21:10.958940Z","iopub.status.idle":"2023-07-21T12:21:12.130537Z","shell.execute_reply.started":"2023-07-21T12:21:10.958911Z","shell.execute_reply":"2023-07-21T12:21:12.129296Z"}}},{"cell_type":"markdown","source":"# Installations","metadata":{}},{"cell_type":"code","source":"!cp -r /kaggle/input/python-packages2 .","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!tar xvfz python-packages2/jiwer.tgz\n!tar xvfz python-packages2/normalizer.tgz\n!tar xvfz python-packages2/pyctcdecode.tgz\n!tar xvfz python-packages2/pypikenlm.tgz","metadata":{"execution":{"iopub.status.busy":"2023-07-21T12:21:49.101319Z","iopub.execute_input":"2023-07-21T12:21:49.101712Z","iopub.status.idle":"2023-07-21T12:21:53.288869Z","shell.execute_reply.started":"2023-07-21T12:21:49.101667Z","shell.execute_reply":"2023-07-21T12:21:53.287279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import jiwer","metadata":{"execution":{"iopub.status.busy":"2023-07-21T12:24:33.405119Z","iopub.execute_input":"2023-07-21T12:24:33.406019Z","iopub.status.idle":"2023-07-21T12:24:33.410543Z","shell.execute_reply.started":"2023-07-21T12:24:33.405989Z","shell.execute_reply":"2023-07-21T12:24:33.409244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import clear_output\nimport sys, os\n\nif 0:\n    !pip download kenlm\n    \nif 0:\n    !pip install kenlm\n    !pip install pyctcdecode\n    !pip install jiwer\n    \ntry:\n    from bnunicodenormalizer import Normalizer\n    import numpy as np\n    import pandas as pd\n    import jiwer\nexcept:     \n    !tar xvfz ./python-packages2/jiwer.tgz\n    !pip install ./jiwer/jiwer-2.3.0-py3-none-any.whl -f ./ --no-index\n    !tar xvfz ./python-packages2/normalizer.tgz\n    !pip install ./normalizer/bnunicodenormalizer-0.0.24.tar.gz -f ./ --no-index\n    !tar xvfz ./python-packages2/pyctcdecode.tgz\n    !pip install ./pyctcdecode/attrs-22.1.0-py2.py3-none-any.whl -f ./ --no-index --no-deps\n    !pip install ./pyctcdecode/exceptiongroup-1.0.0rc9-py3-none-any.whl -f ./ --no-index --no-deps\n    !pip install ./pyctcdecode/hypothesis-6.54.4-py3-none-any.whl -f ./ --no-index --no-deps\n    !pip install ./pyctcdecode/numpy-1.21.6-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl -f ./ --no-index --no-deps\n    !pip install ./pyctcdecode/pygtrie-2.5.0.tar.gz -f ./ --no-index --no-deps\n    !pip install ./pyctcdecode/sortedcontainers-2.4.0-py2.py3-none-any.whl -f ./ --no-index --no-deps\n    !pip install ./pyctcdecode/pyctcdecode-0.4.0-py2.py3-none-any.whl -f ./ --no-index --no-deps\n\n    !tar xvfz ./python-packages2/pypikenlm.tgz\n    !pip install ./pypikenlm/pypi-kenlm-0.1.20220713.tar.gz -f ./ --no-index --no-deps\n    #!pip install /home/pip-packages/* ","metadata":{"execution":{"iopub.status.busy":"2023-07-21T12:24:47.050463Z","iopub.execute_input":"2023-07-21T12:24:47.050836Z","iopub.status.idle":"2023-07-21T12:25:58.673741Z","shell.execute_reply.started":"2023-07-21T12:24:47.050805Z","shell.execute_reply":"2023-07-21T12:25:58.672495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Check imports**","metadata":{}},{"cell_type":"code","source":" #clear_output(wait=True) \nos.chdir('/kaggle/working')\n\nfrom bnunicodenormalizer import Normalizer\nimport numpy as np\nimport pandas as pd\nimport jiwer\n \n\n#-------------------------------------------------\nfrom timeit import default_timer as timer\n#helper\ndef time_to_str(t, mode='min'):\n    if mode=='min':\n        t  = int(t)/60\n        hr = t//60\n        min = t%60\n        return '%2d hr %02d min'%(hr,min)\n\n    elif mode=='sec':\n        t   = int(t)\n        min = t//60\n        sec = t%60\n        return '%2d min %02d sec'%(min,sec)\n\n    else:\n        raise NotImplementedError\n        \n        \nmode='submit'#'submit' #debug\n\n\nprint('mode', mode)\nprint('IMPORT OK !!!!')","metadata":{"execution":{"iopub.status.busy":"2023-07-21T12:25:58.693870Z","iopub.execute_input":"2023-07-21T12:25:58.694601Z","iopub.status.idle":"2023-07-21T12:25:58.706177Z","shell.execute_reply.started":"2023-07-21T12:25:58.694566Z","shell.execute_reply":"2023-07-21T12:25:58.705293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load model","metadata":{}},{"cell_type":"code","source":"!cp -a /kaggle/input/ai4bharat-indicwav2vec-v1-bengali/indicwav2vec_v1_bengali .","metadata":{"execution":{"iopub.status.busy":"2023-07-21T12:26:30.416075Z","iopub.execute_input":"2023-07-21T12:26:30.417075Z","iopub.status.idle":"2023-07-21T12:26:49.888712Z","shell.execute_reply.started":"2023-07-21T12:26:30.417039Z","shell.execute_reply":"2023-07-21T12:26:49.887335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import pipeline\n\nif mode=='debug':\n    device=-1\nif mode=='submit':\n    device=0\n\nlocal_file = '/kaggle/working/indicwav2vec_v1_bengali'\npipe = pipeline(\n    'automatic-speech-recognition',\n    model=local_file,\n    #model='shahruk10/wav2vec2-xls-r-300m-bengali-commonvoice',\n    device=device,\n)\nprint('MODEL OK !!!!')","metadata":{"execution":{"iopub.status.busy":"2023-07-21T12:27:01.602635Z","iopub.execute_input":"2023-07-21T12:27:01.603148Z","iopub.status.idle":"2023-07-21T12:27:27.275609Z","shell.execute_reply.started":"2023-07-21T12:27:01.603100Z","shell.execute_reply":"2023-07-21T12:27:27.274037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if mode=='debug':\n    mp3_dir = f'/kaggle/input/bengaliai-speech/train_mp3s'\n    valid_df = pd.read_csv('/kaggle/input/bengaliai-speech/train.csv')\n    valid_df = valid_df[:10]\nif mode=='submit':\n    mp3_dir = f'/kaggle/input/bengaliai-speech/test_mp3s'\n    valid_df = pd.read_csv('/kaggle/input/bengaliai-speech/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-07-21T12:27:30.631424Z","iopub.execute_input":"2023-07-21T12:27:30.631807Z","iopub.status.idle":"2023-07-21T12:27:30.656632Z","shell.execute_reply.started":"2023-07-21T12:27:30.631776Z","shell.execute_reply":"2023-07-21T12:27:30.655586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Normalizer**","metadata":{}},{"cell_type":"code","source":"#post processor\nbnorm = Normalizer()\ndef normalize(sentence):\n    word = [bnorm(word)['normalized']  for word in sentence.split()]\n    return \" \".join([w for w in word if w is not None])\n\ndef dari(sentence):\n    try:\n        if sentence[-1]!=\"।\":\n            sentence+=\"।\"\n    except:\n        print(sentence)\n    return sentence\n\n    ","metadata":{"execution":{"iopub.status.busy":"2023-07-21T12:27:33.142379Z","iopub.execute_input":"2023-07-21T12:27:33.142770Z","iopub.status.idle":"2023-07-21T12:27:33.149737Z","shell.execute_reply.started":"2023-07-21T12:27:33.142737Z","shell.execute_reply":"2023-07-21T12:27:33.148536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Infer on test data","metadata":{}},{"cell_type":"code","source":"files = sorted(os.listdir(\"/kaggle/input/bengaliai-speech/test_mp3s\"))\nids = []\npreds = []\nfor file in files:\n    ids.append(file.split(\".\")[0])\n    mp3 = os.path.join(mp3_dir,file)\n    text = pipe(mp3)\n    text = text['text']\n    if len(text)==0:\n        text = \",\"\n    preds.append(text)","metadata":{"execution":{"iopub.status.busy":"2023-07-21T12:34:54.242345Z","iopub.execute_input":"2023-07-21T12:34:54.243042Z","iopub.status.idle":"2023-07-21T12:34:54.747899Z","shell.execute_reply.started":"2023-07-21T12:34:54.243008Z","shell.execute_reply":"2023-07-21T12:34:54.746085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit_df = pd.DataFrame({\"id\":ids,\"sentence\":preds})\nsubmit_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-21T12:34:57.965203Z","iopub.execute_input":"2023-07-21T12:34:57.965586Z","iopub.status.idle":"2023-07-21T12:34:57.976809Z","shell.execute_reply.started":"2023-07-21T12:34:57.965557Z","shell.execute_reply":"2023-07-21T12:34:57.975711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def check(sentence):\n    if len(sentence)==0:\n        sentence = \",\"\n    return sentence","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit_df.sentence = submit_df.sentence.apply(lambda x:dari(normalize(x)))\nsubmit_df.sentence = submit_df.sentence.apply(lambda x:check(x))\nsubmit_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-21T12:35:01.589868Z","iopub.execute_input":"2023-07-21T12:35:01.590294Z","iopub.status.idle":"2023-07-21T12:35:01.616173Z","shell.execute_reply.started":"2023-07-21T12:35:01.590251Z","shell.execute_reply":"2023-07-21T12:35:01.614924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit_df.to_csv('submission.csv',index=False)\nprint(submit_df)\nprint('SUBMIT DONE !!!!!!!')","metadata":{"execution":{"iopub.status.busy":"2023-07-21T12:35:06.304424Z","iopub.execute_input":"2023-07-21T12:35:06.304812Z","iopub.status.idle":"2023-07-21T12:35:06.314225Z","shell.execute_reply.started":"2023-07-21T12:35:06.304778Z","shell.execute_reply":"2023-07-21T12:35:06.313198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !rm -rf '/root/.cache/huggingface'\n# print( scan_cache_dir())","metadata":{"execution":{"iopub.status.busy":"2023-07-21T04:07:02.758875Z","iopub.execute_input":"2023-07-21T04:07:02.759532Z","iopub.status.idle":"2023-07-21T04:07:02.763451Z","shell.execute_reply.started":"2023-07-21T04:07:02.759496Z","shell.execute_reply":"2023-07-21T04:07:02.762462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf indicwav2vec_v1_bengali","metadata":{"execution":{"iopub.status.busy":"2023-07-21T12:32:30.610976Z","iopub.execute_input":"2023-07-21T12:32:30.611370Z","iopub.status.idle":"2023-07-21T12:32:31.909720Z","shell.execute_reply.started":"2023-07-21T12:32:30.611338Z","shell.execute_reply":"2023-07-21T12:32:31.908034Z"},"trusted":true},"execution_count":null,"outputs":[]}]}