{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"!cp -r ../input/python-packages2 ./\n\n!tar xvfz ./python-packages2/jiwer.tgz\n!pip install ./jiwer/jiwer-2.3.0-py3-none-any.whl -f ./ --no-index\n!tar xvfz ./python-packages2/normalizer.tgz\n!pip install ./normalizer/bnunicodenormalizer-0.0.24.tar.gz -f ./ --no-index\n!tar xvfz ./python-packages2/pyctcdecode.tgz\n!pip install ./pyctcdecode/attrs-22.1.0-py2.py3-none-any.whl -f ./ --no-index --no-deps\n!pip install ./pyctcdecode/exceptiongroup-1.0.0rc9-py3-none-any.whl -f ./ --no-index --no-deps\n!pip install ./pyctcdecode/hypothesis-6.54.4-py3-none-any.whl -f ./ --no-index --no-deps\n!pip install ./pyctcdecode/numpy-1.21.6-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl -f ./ --no-index --no-deps\n!pip install ./pyctcdecode/pygtrie-2.5.0.tar.gz -f ./ --no-index --no-deps\n!pip install ./pyctcdecode/sortedcontainers-2.4.0-py2.py3-none-any.whl -f ./ --no-index --no-deps\n!pip install ./pyctcdecode/pyctcdecode-0.4.0-py2.py3-none-any.whl -f ./ --no-index --no-deps\n\n!tar xvfz ./python-packages2/pypikenlm.tgz\n!pip install ./pypikenlm/pypi-kenlm-0.1.20220713.tar.gz -f ./ --no-index --no-deps\n\nimport os\n!python -m pip install --no-index --find-links=../input/bengaliai-pip-wheels-demucs demucs","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rm -r python-packages2 jiwer normalizer pyctcdecode pypikenlm","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir ./generated_audio","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Demucs","metadata":{}},{"cell_type":"code","source":"%%python\nimport glob\nimport subprocess\nimport os\n\nimport typing as tp\nfrom pathlib import Path\nfrom functools import partial\nfrom dataclasses import dataclass, field\n\nimport pandas as pd\nimport pyctcdecode\nimport numpy as np\nfrom tqdm.notebook import tqdm\n\nimport librosa\n\nimport pyctcdecode\nimport kenlm\nimport torch\nfrom transformers import Wav2Vec2Processor, Wav2Vec2ProcessorWithLM, Wav2Vec2ForCTC\nfrom bnunicodenormalizer import Normalizer\n\nimport cloudpickle as cpkl\n\nROOT = Path.cwd().parent\nINPUT = ROOT / \"input\"\nDATA = INPUT / \"bengaliai-speech\"\nTRAIN = DATA / \"train_mp3s\"\nTEST = DATA / \"test_mp3s\"\n\nSAMPLING_RATE = 16_000\nMODEL_PATH = INPUT / \"bengali-sr-download-public-trained-models/indicwav2vec_v1_bengali/\"\nLM_PATH = INPUT / \"bengali-sr-download-public-trained-models/wav2vec2-xls-r-300m-bengali/language_model/\"\n\n# Customize the following options!\nmodel = \"htdemucs\"\nextensions = [\"mp3\", \"wav\", \"ogg\", \"flac\"]  # we will look for all those file types.\n# two_stems = None   # only separate one stems from the rest, for instance\ntwo_stems = \"vocals\"\n\n# Options for the output audio.\nmp3 = True\nmp3_rate = 48000\nfloat32 = False  # output as float 32 wavs, unsused if 'mp3' is True.\nint24 = False    # output as int24 wavs, unused if 'mp3' is True.\n# You cannot set both `float32 = True` and `int24 = True` !!\n\nin_path = TEST\nout_path = './music_output'\nrepo_path = '/kaggle/input/demucs-model'\n\n#@title Useful functions, don't forget to execute\nimport io\nfrom pathlib import Path\nimport select\nfrom shutil import rmtree\nimport subprocess as sp\nimport sys\nfrom typing import Dict, Tuple, Optional, IO\n\ndef find_files(in_path):\n    out = []\n    for file in Path(in_path).iterdir():\n        if file.suffix.lower().lstrip(\".\") in extensions:\n            out.append(file)\n    return out\n\ndef copy_process_streams(process: sp.Popen):\n    def raw(stream: Optional[IO[bytes]]) -> IO[bytes]:\n        assert stream is not None\n        if isinstance(stream, io.BufferedIOBase):\n            stream = stream.raw\n        return stream\n\n    p_stdout, p_stderr = raw(process.stdout), raw(process.stderr)\n    stream_by_fd: Dict[int, Tuple[IO[bytes], io.StringIO, IO[str]]] = {\n        p_stdout.fileno(): (p_stdout, sys.stdout),\n        p_stderr.fileno(): (p_stderr, sys.stderr),\n    }\n    fds = list(stream_by_fd.keys())\n\n    while fds:\n        # `select` syscall will wait until one of the file descriptors has content.\n        ready, _, _ = select.select(fds, [], [])\n        for fd in ready:\n            p_stream, std = stream_by_fd[fd]\n            raw_buf = p_stream.read(2 ** 16)\n            if not raw_buf:\n                fds.remove(fd)\n                continue\n            buf = raw_buf.decode()\n            std.write(buf)\n            std.flush()\n\ndef separate(inp=None, outp=None):\n    inp = inp or in_path\n    outp = outp or out_path\n    cmd = [\"python3\", \"-m\", \"demucs.separate\", \"-o\", str(outp), \"-n\", model, \"--repo\", repo_path]\n    if mp3:\n        cmd += [\"--mp3\", f\"--mp3-bitrate={mp3_rate}\"]\n    if float32:\n        cmd += [\"--float32\"]\n    if int24:\n        cmd += [\"--int24\"]\n    if two_stems is not None:\n        cmd += [f\"--two-stems={two_stems}\"]\n    files = [str(f) for f in find_files(inp)]\n    if not files:\n        print(f\"No valid audio files in {in_path}\")\n        return\n    print(\"Going to separate the files:\")\n    print('\\n'.join(files))\n    print(\"With command: \", \" \".join(cmd))\n    p = sp.Popen(cmd + files, stdout=sp.PIPE, stderr=sp.PIPE)\n    copy_process_streams(p)\n    p.wait()\n    if p.returncode != 0:\n        print(\"Command failed, something went wrong.\")\n\nseparate()\n\ndirlist = glob.glob(f'./{out_path}/htdemucs/*/')\nfor d in dirlist:\n    before = os.path.join(d, 'vocals.mp3')\n    after = os.path.join('./generated_audio', d[:-1].replace(f'./{out_path}/htdemucs/', '')+'.mp3')\n    subprocess.run(f\"mv {before} {after}\",\n                   shell=True, capture_output=True, text=True)\n\nTEST = Path('./generated_audio')\n\n# import soundfile as sf\nfor d in glob.glob(str(TEST/'*')):\n    audio = librosa.load(d, sr=16000)[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference1","metadata":{}},{"cell_type":"code","source":"%%python\nimport typing as tp\nfrom pathlib import Path\nfrom functools import partial\nfrom dataclasses import dataclass, field\n\nimport re\nimport pandas as pd\nimport pyctcdecode\nimport numpy as np\nfrom tqdm.notebook import tqdm\n\nimport librosa\n\nimport pyctcdecode\nimport kenlm\nimport torch\nfrom transformers import Wav2Vec2Processor, Wav2Vec2ProcessorWithLM, Wav2Vec2ForCTC\nfrom bnunicodenormalizer import Normalizer\n\nimport cloudpickle as cpkl\n\nROOT = Path.cwd().parent\nINPUT = ROOT / \"input\"\nDATA = INPUT / \"bengaliai-speech\"\nTRAIN = DATA / \"train_mp3s\"\nTEST = DATA / \"test_mp3s\"\n\nSAMPLING_RATE = 16_000\nMODEL_PATH = INPUT / \"bengali-sr-download-public-trained-models/indicwav2vec_v1_bengali/\"\nLM_PATH = INPUT / \"bengali-sr-download-public-trained-models/wav2vec2-xls-r-300m-bengali/language_model/\"\n\n\nif not torch.cuda.is_available():\n    device = torch.device(\"cpu\")\nelse:\n    device = torch.device(\"cuda\")\n\nmodel = Wav2Vec2ForCTC.from_pretrained(MODEL_PATH)\nprocessor = Wav2Vec2Processor.from_pretrained(MODEL_PATH)\n\nmodel.load_state_dict(torch.load('/kaggle/input/baisr-ver5-3/model_ver5_3_WER0.32805.pth', map_location=device)['model'])\n\nvocab_dict = processor.tokenizer.get_vocab()\nsorted_vocab_dict = {k: v for k, v in sorted(vocab_dict.items(), key=lambda item: item[1])}\n\ndecoder = pyctcdecode.build_ctcdecoder(\n    list(sorted_vocab_dict.keys()),\n    str(\"/kaggle/input/5gramlm4/5gram_normalized_bn_train_commonvoice_fleurs_openslr_openslr37_oscar_without_punc_prune2.arpa\"),\n)\n\nprocessor_with_lm = Wav2Vec2ProcessorWithLM(\n    feature_extractor=processor.feature_extractor,\n    tokenizer=processor.tokenizer,\n    decoder=decoder\n)\n\nclass BengaliSRTestDataset(torch.utils.data.Dataset):\n    \n    def __init__(\n        self,\n        audio_paths: list[str],\n        sampling_rate: int\n    ):\n        self.audio_paths = audio_paths\n        self.sampling_rate = sampling_rate\n        \n    def __len__(self,):\n        return len(self.audio_paths)\n    \n    def __getitem__(self, index: int):\n        audio_path = self.audio_paths[index]\n        sr = self.sampling_rate\n        w = librosa.load(audio_path, sr=sr)[0]\n\n        return w\n\ntest = pd.read_csv(DATA / \"sample_submission.csv\", dtype={\"id\": str})\ntest['original_index'] = test.index\ntest['path'] = [str(TEST / f\"{aid}.mp3\") for aid in test[\"id\"].values]\n\ndef read_audio(mp3_path, target_sr=16000):\n    audio, sr = librosa.load(mp3_path, sr=32000)\n    return audio\n\ndef get_audio_length(row):\n    audio = read_audio(row['path'])\n    row['audio_length'] = len(audio)/32000 \n    return row\n\ntest = test.apply(get_audio_length, axis=1)\n\ntest = test.sort_values('audio_length').reset_index(drop=True)\n\ntest_audio_paths = [str(TEST / f\"{aid}.mp3\") for aid in test[\"id\"].values]\n\ntest_dataset = BengaliSRTestDataset(\n    test_audio_paths, SAMPLING_RATE\n)\n\ncollate_func = partial(\n    processor_with_lm.feature_extractor,\n    return_tensors=\"pt\", sampling_rate=SAMPLING_RATE,\n    padding=True,\n)\n\ntest_loader = torch.utils.data.DataLoader(\n    test_dataset, batch_size=8, shuffle=False,\n    num_workers=2, collate_fn=collate_func, drop_last=False,\n    pin_memory=True,\n)\n\nmodel = model.to(device)\nmodel = model.eval()\nmodel = model.half()\n\npred_sentence_list = []\n\nwith torch.no_grad():\n    for batch in tqdm(test_loader):\n        x = batch[\"input_values\"]\n        x = x.to(device, non_blocking=True)\n        with torch.cuda.amp.autocast(True):\n            y = model(x).logits\n        y = y.detach().cpu().numpy()\n        \n        for l in y:  \n            sentence = processor_with_lm.decode(l, beam_width=10).text\n            pred_sentence_list.append(sentence)\n\nbnorm = Normalizer()\n\ndef postprocess(sentence):\n    _words = [bnorm(word)['normalized']  for word in sentence.split()]\n    sentence = \" \".join([word for word in _words if word is not None])\n    try:\n        sentence = \" \".join(re.sub('[\\,\\।\\?\\!\\-]', \" \", sentence).split())\n        if len(sentence) == 0:\n            sentence = \"।\"\n    except:\n        sentence = \"।\"\n    return sentence\n\npp_pred_sentence_list = [\n    postprocess(s) for s in tqdm(pred_sentence_list)]\n\ntest[\"sentence\"] = pp_pred_sentence_list\ntest = test.sort_values('original_index').reset_index(drop=True).drop(['original_index', 'audio_length', 'path'], axis=1)\ntest.to_csv(\"pre_submission_no_demucs.csv\", index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%python\nimport typing as tp\nfrom pathlib import Path\nfrom functools import partial\nfrom dataclasses import dataclass, field\n\nimport re\nimport pandas as pd\nimport pyctcdecode\nimport numpy as np\nfrom tqdm.notebook import tqdm\n\nimport librosa\n\nimport pyctcdecode\nimport kenlm\nimport torch\nfrom transformers import Wav2Vec2Processor, Wav2Vec2ProcessorWithLM, Wav2Vec2ForCTC\nfrom bnunicodenormalizer import Normalizer\n\nimport cloudpickle as cpkl\n\nROOT = Path.cwd().parent\nINPUT = ROOT / \"input\"\nDATA = INPUT / \"bengaliai-speech\"\nTRAIN = DATA / \"train_mp3s\"\nTEST = Path('./generated_audio')\n\nSAMPLING_RATE = 16_000\nMODEL_PATH = INPUT / \"bengali-sr-download-public-trained-models/indicwav2vec_v1_bengali/\"\nLM_PATH = INPUT / \"bengali-sr-download-public-trained-models/wav2vec2-xls-r-300m-bengali/language_model/\"\n\n\nif not torch.cuda.is_available():\n    device = torch.device(\"cpu\")\nelse:\n    device = torch.device(\"cuda\")\n\nmodel = Wav2Vec2ForCTC.from_pretrained(MODEL_PATH)\nprocessor = Wav2Vec2Processor.from_pretrained(MODEL_PATH)\n\nmodel.load_state_dict(torch.load('/kaggle/input/baisr-ver5-3/model_ver5_3_WER0.32805.pth', map_location=device)['model'])\n\nvocab_dict = processor.tokenizer.get_vocab()\nsorted_vocab_dict = {k: v for k, v in sorted(vocab_dict.items(), key=lambda item: item[1])}\n\ndecoder = pyctcdecode.build_ctcdecoder(\n    list(sorted_vocab_dict.keys()),\n    str(\"/kaggle/input/5gramlm4/5gram_normalized_bn_train_commonvoice_fleurs_openslr_openslr37_oscar_without_punc_prune2.arpa\"),\n)\n\nprocessor_with_lm = Wav2Vec2ProcessorWithLM(\n    feature_extractor=processor.feature_extractor,\n    tokenizer=processor.tokenizer,\n    decoder=decoder\n)\n\nclass BengaliSRTestDataset(torch.utils.data.Dataset):\n    \n    def __init__(\n        self,\n        audio_paths: list[str],\n        sampling_rate: int\n    ):\n        self.audio_paths = audio_paths\n        self.sampling_rate = sampling_rate\n        \n    def __len__(self,):\n        return len(self.audio_paths)\n    \n    def __getitem__(self, index: int):\n        audio_path = self.audio_paths[index]\n        sr = self.sampling_rate\n        w = librosa.load(audio_path, sr=sr)[0]\n\n        return w\n\ntest = pd.read_csv(DATA / \"sample_submission.csv\", dtype={\"id\": str})\ntest['original_index'] = test.index\ntest['path'] = [str(TEST / f\"{aid}.mp3\") for aid in test[\"id\"].values]\n\ndef read_audio(mp3_path, target_sr=16000):\n    audio, sr = librosa.load(mp3_path, sr=32000)\n    return audio\n\ndef get_audio_length(row):\n    audio = read_audio(row['path'])\n    row['audio_length'] = len(audio)/32000 \n    return row\n\ntest = test.apply(get_audio_length, axis=1)\n\ntest = test.sort_values('audio_length').reset_index(drop=True)\n\ntest_audio_paths = [str(TEST / f\"{aid}.mp3\") for aid in test[\"id\"].values]\n\ntest_dataset = BengaliSRTestDataset(\n    test_audio_paths, SAMPLING_RATE\n)\n\ncollate_func = partial(\n    processor_with_lm.feature_extractor,\n    return_tensors=\"pt\", sampling_rate=SAMPLING_RATE,\n    padding=True,\n)\n\ntest_loader = torch.utils.data.DataLoader(\n    test_dataset, batch_size=8, shuffle=False,\n    num_workers=2, collate_fn=collate_func, drop_last=False,\n    pin_memory=True,\n)\n\nmodel = model.to(device)\nmodel = model.eval()\nmodel = model.half()\n\npred_sentence_list = []\n\nwith torch.no_grad():\n    for batch in tqdm(test_loader):\n        x = batch[\"input_values\"]\n        x = x.to(device, non_blocking=True)\n        with torch.cuda.amp.autocast(True):\n            y = model(x).logits\n        y = y.detach().cpu().numpy()\n        \n        for l in y:  \n            sentence = processor_with_lm.decode(l, beam_width=10).text\n            pred_sentence_list.append(sentence)\n\nbnorm = Normalizer()\n\ndef postprocess(sentence):\n    _words = [bnorm(word)['normalized']  for word in sentence.split()]\n    sentence = \" \".join([word for word in _words if word is not None])\n    try:\n        sentence = \" \".join(re.sub('[\\,\\।\\?\\!\\-]', \" \", sentence).split())\n        if len(sentence) == 0:\n            sentence = \"।\"\n    except:\n        sentence = \"।\"\n    return sentence\n\npp_pred_sentence_list = [\n    postprocess(s) for s in tqdm(pred_sentence_list)]\n\ntest[\"sentence\"] = pp_pred_sentence_list\ntest = test.sort_values('original_index').reset_index(drop=True).drop(['original_index', 'audio_length', 'path'], axis=1)\ntest.to_csv(\"pre_submission_demucs.csv\", index=False)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%python\nimport pandas as pd\nimport numpy as np\nimport subprocess\n\ndf_before = pd.read_csv('pre_submission_no_demucs.csv')\ndf_after = pd.read_csv('pre_submission_demucs.csv')\npred = []\nfor id, before, after in zip(df_after[\"id\"].values, df_before[\"sentence\"].values, df_after[\"sentence\"].values):\n    if len(after.split()) < len(before.split()):\n        subprocess.run(f'cp /kaggle/input/bengaliai-speech/test_mp3s/{id}.mp3 ./generated_audio/{id}.mp3', shell=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference2","metadata":{}},{"cell_type":"code","source":"%%python\nimport typing as tp\nfrom pathlib import Path\nfrom functools import partial\nfrom dataclasses import dataclass, field\n\nimport re\nimport pandas as pd\nimport pyctcdecode\nimport numpy as np\nfrom tqdm.notebook import tqdm\n\nimport librosa\n\nimport pyctcdecode\nimport kenlm\nimport torch\nfrom transformers import Wav2Vec2Processor, Wav2Vec2ProcessorWithLM, Wav2Vec2ForCTC\nfrom bnunicodenormalizer import Normalizer\n\nimport cloudpickle as cpkl\n\nROOT = Path.cwd().parent\nINPUT = ROOT / \"input\"\nDATA = INPUT / \"bengaliai-speech\"\nTRAIN = DATA / \"train_mp3s\"\nTEST = Path('./generated_audio')\n\nSAMPLING_RATE = 16_000\nMODEL_PATH = INPUT / \"bengali-sr-download-public-trained-models/indicwav2vec_v1_bengali/\"\nLM_PATH = INPUT / \"bengali-sr-download-public-trained-models/wav2vec2-xls-r-300m-bengali/language_model/\"\n\n\nif not torch.cuda.is_available():\n    device = torch.device(\"cpu\")\nelse:\n    device = torch.device(\"cuda\")\n\nmodel = Wav2Vec2ForCTC.from_pretrained(MODEL_PATH)\nprocessor = Wav2Vec2Processor.from_pretrained(MODEL_PATH)\n\nmodel.load_state_dict(torch.load('/kaggle/input/baisr-ver5-3/model_ver5_3_WER0.32805.pth', map_location=device)['model'])\n\nvocab_dict = processor.tokenizer.get_vocab()\nsorted_vocab_dict = {k: v for k, v in sorted(vocab_dict.items(), key=lambda item: item[1])}\n\ndecoder = pyctcdecode.build_ctcdecoder(\n    list(sorted_vocab_dict.keys()),\n    str(\"/kaggle/input/5gramlm4/5gram_normalized_bn_train_commonvoice_fleurs_openslr_openslr37_oscar_without_punc_prune2.arpa\"),\n)\n\nprocessor_with_lm = Wav2Vec2ProcessorWithLM(\n    feature_extractor=processor.feature_extractor,\n    tokenizer=processor.tokenizer,\n    decoder=decoder\n)\n\nclass BengaliSRTestDataset(torch.utils.data.Dataset):\n    \n    def __init__(\n        self,\n        audio_paths: list[str],\n        sampling_rate: int\n    ):\n        self.audio_paths = audio_paths\n        self.sampling_rate = sampling_rate\n        \n    def __len__(self,):\n        return len(self.audio_paths)\n    \n    def __getitem__(self, index: int):\n        audio_path = self.audio_paths[index]\n        sr = self.sampling_rate\n        w = librosa.load(audio_path, sr=sr)[0]\n\n        return w\n\ntest = pd.read_csv(DATA / \"sample_submission.csv\", dtype={\"id\": str})\ntest['original_index'] = test.index\ntest['path'] = [str(TEST / f\"{aid}.mp3\") for aid in test[\"id\"].values]\n\ndef read_audio(mp3_path, target_sr=16000):\n    audio, sr = librosa.load(mp3_path, sr=32000)\n    return audio\n\ndef get_audio_length(row):\n    audio = read_audio(row['path'])\n    row['audio_length'] = len(audio)/32000 \n    return row\n\ntest = test.apply(get_audio_length, axis=1)\n\ntest = test.sort_values('audio_length').reset_index(drop=True)\n\ntest_audio_paths = [str(TEST / f\"{aid}.mp3\") for aid in test[\"id\"].values]\n\ntest_dataset = BengaliSRTestDataset(\n    test_audio_paths, SAMPLING_RATE\n)\n\ncollate_func = partial(\n    processor_with_lm.feature_extractor,\n    return_tensors=\"pt\", sampling_rate=SAMPLING_RATE,\n    padding=True,\n)\n\ntest_loader = torch.utils.data.DataLoader(\n    test_dataset, batch_size=8, shuffle=False,\n    num_workers=2, collate_fn=collate_func, drop_last=False,\n    pin_memory=True,\n)\n\nmodel = model.to(device)\nmodel = model.eval()\nmodel = model.half()\n\npred_sentence_list = []\n\nwith torch.no_grad():\n    for batch in tqdm(test_loader):\n        x = batch[\"input_values\"]\n        x = x.to(device, non_blocking=True)\n        with torch.cuda.amp.autocast(True):\n            y = model(x).logits\n        y = y.detach().cpu().numpy()\n        \n        for l in y:  \n            sentence = processor_with_lm.decode(l, beam_width=256*3).text\n            pred_sentence_list.append(sentence)\n\nbnorm = Normalizer()\n\ndef postprocess(sentence):\n    _words = [bnorm(word)['normalized']  for word in sentence.split()]\n    sentence = \" \".join([word for word in _words if word is not None])\n    try:\n        sentence = \" \".join(re.sub('[\\,\\।\\?\\!\\-]', \" \", sentence).split())\n        if len(sentence) == 0:\n            sentence = \"।\"\n    except:\n        sentence = \"।\"\n    return sentence\n\npp_pred_sentence_list = [\n    postprocess(s) for s in tqdm(pred_sentence_list)]\n\ntest[\"sentence\"] = pp_pred_sentence_list\ntest = test.sort_values('original_index').reset_index(drop=True).drop(['original_index', 'audio_length', 'path'], axis=1)\ntest.to_csv(\"pre_submission.csv\", index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Punctuation","metadata":{}},{"cell_type":"code","source":"import os\nos.system('python -m pip install --no-index --find-links=../input/vakyansh-models-punctuation-models-bengali indic-nlp-library')\nimport torch\nfrom transformers import AutoTokenizer, AutoModelForTokenClassification\nimport numpy as np\nimport pandas as pd\nimport json\nimport torch.nn as nn\nfrom indicnlp.tokenize import indic_tokenize\n\nCHECKPOINT_PATH = '/kaggle/input/punctuation-models/xlm-roberta-large_exp016_cp175.pt'\nLABEL_ENCODER_PATH = '/kaggle/input/punctuation-models/label_encoder.json'\n\nlabel_encoder_path = LABEL_ENCODER_PATH\npunctuation_dict = {'qm': '? ', 'comma': ', ', 'end': '। ', 'blank': ' ', 'hyp': '-', 'PAD': ' '}\n\nwith open(label_encoder_path) as label_encoder:\n    train_encoder = json.load(label_encoder)\n\ntokenizer = AutoTokenizer.from_pretrained(\n    '/kaggle/input/xlm-roberta-large',\n)\nmodel = AutoModelForTokenClassification.from_pretrained(\n    '/kaggle/input/xlm-roberta-large',\n    num_labels=len(train_encoder),\n    output_attentions=False,\n    output_hidden_states=False,\n)\n\n# model = nn.DataParallel(model)\ncheckpoint = torch.load(CHECKPOINT_PATH)\nmodel.load_state_dict(checkpoint['state_dict'], strict=False)\n\nmodel.eval()\nmodel.cuda()\n\n# Added model\nCHECKPOINT_PATH_1 = '/kaggle/input/punctuation-models/xlm-roberta-base_exp017_cp200.pt'\ntokenizer_1 = AutoTokenizer.from_pretrained(\n    '/kaggle/input/bengaliai-xlm-roberta-base',\n)\nmodel_1 = AutoModelForTokenClassification.from_pretrained(\n    '/kaggle/input/bengaliai-xlm-roberta-base',\n    num_labels=len(train_encoder),\n    output_attentions=False,\n    output_hidden_states=False,\n)\ncheckpoint_1 = torch.load(CHECKPOINT_PATH_1)\nmodel_1.load_state_dict(checkpoint_1['state_dict'], strict=False)\nmodel_1.eval()\nmodel_1.cuda()\n\ndef get_tokens_and_labels_indices_from_text(text):\n    # MODEL 0\n    tokenized_sentence = tokenizer.encode(text)\n    input_ids = torch.tensor([tokenized_sentence]).cuda()\n    with torch.no_grad():\n        output = model(input_ids)\n    label_indices = output[0].to('cpu').numpy()\n    tokens = tokenizer.convert_ids_to_tokens(input_ids.to('cpu').numpy()[0])\n    # MODEL 1\n    tokenized_sentence_1 = tokenizer_1.encode(text)\n    input_ids_1 = torch.tensor([tokenized_sentence_1]).cuda()\n    with torch.no_grad():\n        output_1 = model_1(input_ids_1)\n    label_indices_1 = output_1[0].to('cpu').numpy()\n    tokens_1 = tokenizer_1.convert_ids_to_tokens(input_ids_1.to('cpu').numpy()[0])\n    return tokens, label_indices, tokens_1, label_indices_1\n\ndef map_tokens_and_labels_to_word_and_punctuations(text):\n    if len(text) > 2048:\n        full_text = \" \".join(text.split())\n        if full_text[-1] not in [\"।\", \"!\", \"?\", \"-\"]:\n            full_text += \"।\"\n        return full_text\n    tokens, label_indices, tokens_1, label_indices_1 = get_tokens_and_labels_indices_from_text(text)\n    new_tokens = []\n    new_labels = []\n    for i in range(1, len(tokens) - 1):\n        if tokens[i].startswith(\"▁\"):\n            current_word = tokens[i][1:]\n            new_labels.append(label_indices[0][i])\n            for j in range(i + 1, len(tokens) - 1):\n                if not tokens[j].startswith(\"▁\"):\n                    current_word = current_word + tokens[j]\n                if tokens[j].startswith(\"▁\"):\n                    break\n            new_tokens.append(current_word)\n    full_text = ''\n    tokenized_text = indic_tokenize.trivial_tokenize_indic(text)\n    \n    if len(tokenized_text) == len(new_labels):\n        full_text_tokens = tokenized_text\n    else:\n        full_text_tokens = new_tokens\n    new_tokens_1 = []\n    new_labels_1 = []\n    for i in range(1, len(tokens_1) - 1):\n        if tokens_1[i].startswith(\"▁\"):\n            current_word_1 = tokens_1[i][1:]\n            new_labels_1.append(label_indices_1[0][i])\n            for j in range(i + 1, len(tokens_1) - 1):\n                if not tokens_1[j].startswith(\"▁\"):\n                    current_word_1 = current_word_1 + tokens_1[j]\n                if tokens_1[j].startswith(\"▁\"):\n                    break\n            new_tokens_1.append(current_word_1)\n    if len(new_labels) == len(new_labels_1):\n        for word, punctuation_0, punctuation_1 in zip(full_text_tokens, new_labels, new_labels_1):\n            x_0 = np.exp(punctuation_0 - np.max(punctuation_0))\n            pred_0 = x_0 / np.sum(x_0)\n            x_1 = np.exp(punctuation_1 - np.max(punctuation_1))\n            pred_1 = x_1 / np.sum(x_1)\n            punctuation = pred_0 * 0.95 + pred_1 * 0.05\n            punctuation = np.argmax(punctuation)\n            punctuation = list(train_encoder.keys())[list(train_encoder.values()).index(punctuation)]\n            full_text = full_text + word + punctuation_dict[punctuation]\n    else:\n        for word, punctuation in zip(full_text_tokens, new_labels):\n            punctuation = np.argmax(punctuation)\n            punctuation = list(train_encoder.keys())[list(train_encoder.values()).index(punctuation)]\n            full_text = full_text + word + punctuation_dict[punctuation]\n    \n    full_text = \" \".join(full_text.split())\n    if len(full_text) == 0:\n        full_text = \" \".join(text.split())\n        if len(full_text) == 0:\n            full_text = \"।\"\n    if full_text[-1] in [\",\", \"-\"]:\n        full_text = full_text[:-1] + \"।\"\n    if full_text[-1] not in [\"।\", \"!\", \"?\"]:\n        full_text += \"।\"\n    \n    return full_text\n\nsub = pd.read_csv(\"pre_submission.csv\")\nsub['sentence'] = sub['sentence'].apply(map_tokens_and_labels_to_word_and_punctuations)\n\nfrom bnunicodenormalizer import Normalizer\nbnorm = Normalizer()\ndef normalize(sentence):\n    word = [bnorm(word)['normalized'] for word in sentence.split()]\n    return \" \".join([w for w in word if w is not None])\nsub['sentence'] = sub['sentence'].apply(lambda x: normalize(x))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf ./*","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}