{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":73047,"databundleVersionId":8149390,"sourceType":"competition"},{"sourceId":8032358,"sourceType":"datasetVersion","datasetId":4734632},{"sourceId":8208103,"sourceType":"datasetVersion","datasetId":4766413}],"dockerImageVersionId":30673,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\n\nimport pandas as pd\n\nimport librosa\nimport librosa.display\n\nimport numpy as np\n\nimport IPython.display as ipd\n\nimport matplotlib.pyplot as plt\n\nimport random\n\nfrom collections import Counter\n\nfrom sklearn.model_selection import train_test_split\n\nimport torch\nimport torchaudio\n\nfrom dataclasses import dataclass\nfrom typing import Any, Dict, List, Union\nfrom datasets import DatasetDict\nfrom datasets import Dataset as DS\n\nfrom transformers import (\n    WhisperFeatureExtractor,\n    WhisperTokenizer,\n    WhisperProcessor,\n    WhisperForConditionalGeneration,\n    Seq2SeqTrainingArguments,\n    Seq2SeqTrainer,\n    TrainerCallback,\n    TrainingArguments,\n    TrainerState,\n    TrainerControl,\n    EarlyStoppingCallback,\n    pipeline\n)\n\nfrom torchmetrics.text import WordErrorRate, CharErrorRate","metadata":{"execution":{"iopub.status.busy":"2024-04-23T16:56:44.663079Z","iopub.execute_input":"2024-04-23T16:56:44.663451Z","iopub.status.idle":"2024-04-23T16:56:44.672603Z","shell.execute_reply.started":"2024-04-23T16:56:44.663425Z","shell.execute_reply":"2024-04-23T16:56:44.671649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_DIR = '/kaggle/input/ben10/ben10'\ntrain_data_dir = f\"{BASE_DIR}/16_kHz_train_audio/\"\ntest_data_dir = f\"{BASE_DIR}/16_kHz_valid_audio/\"\ndata_path = f\"{BASE_DIR}/train.csv\"","metadata":{"execution":{"iopub.status.busy":"2024-04-23T16:56:44.674485Z","iopub.execute_input":"2024-04-23T16:56:44.674872Z","iopub.status.idle":"2024-04-23T16:56:44.702716Z","shell.execute_reply.started":"2024-04-23T16:56:44.674846Z","shell.execute_reply":"2024-04-23T16:56:44.701802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pipe = pipeline(\n    \"automatic-speech-recognition\",\n    model='/kaggle/input/bengalie-asr-with-whisper/whisper-reg-ben',#replace this model path with yours which you import as dataset\n    chunk_length_s=30,\n    device=0,\n)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T16:56:44.703646Z","iopub.execute_input":"2024-04-23T16:56:44.703888Z","iopub.status.idle":"2024-04-23T16:56:51.906410Z","shell.execute_reply.started":"2024-04-23T16:56:44.703867Z","shell.execute_reply":"2024-04-23T16:56:51.905632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pretty_sort(filename):\n    name, number_str = filename.split(\" (\")\n    number = int(number_str.split(\")\")[0])\n    return name, number","metadata":{"execution":{"iopub.status.busy":"2024-04-23T16:56:51.908955Z","iopub.execute_input":"2024-04-23T16:56:51.909736Z","iopub.status.idle":"2024-04-23T16:56:51.914690Z","shell.execute_reply.started":"2024-04-23T16:56:51.909700Z","shell.execute_reply":"2024-04-23T16:56:51.913843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from bnunicodenormalizer import Normalizer \n\n\nbnorm = Normalizer()\ndef normalize(sen):\n    _words = [bnorm(word)['normalized']  for word in sen.split()]\n    return \" \".join([word for word in _words if word is not None])\n\ndef dari(sentence):\n    try:\n        if sentence[-1]!=\"।\":\n            sentence+=\"।\"\n    except:\n        print(sentence)\n    return sentence","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids = []\npreds=[]","metadata":{"execution":{"iopub.status.busy":"2024-04-23T16:56:51.915876Z","iopub.execute_input":"2024-04-23T16:56:51.916149Z","iopub.status.idle":"2024-04-23T16:56:51.925471Z","shell.execute_reply.started":"2024-04-23T16:56:51.916126Z","shell.execute_reply":"2024-04-23T16:56:51.924620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for root, dirs, files in os.walk(\"/kaggle/input/ben10/ben10/16_kHz_valid_audio\"):\n    files = sorted(files, key=pretty_sort)\n    \n    print(files.index(\"valid_sandwip (1).wav\"))\n\n    shift = files[1070 : 1202]\n    \n    files = shift + files[:1070] + files[1202:]\n    ids = files.copy()\n    \n    for file in files:\n        composed_path = f\"{test_data_dir}{file}\"\n        audio, sr = librosa.load(composed_path, sr=16_000)\n        text = pipe(audio)[\"text\"]\n        text = normalize(text)\n        text = dari(text)\n        preds.append(text)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T16:56:51.926646Z","iopub.execute_input":"2024-04-23T16:56:51.926938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = pd.DataFrame()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df[\"id\"] = ids\nsub_df[\"sentence\"] = preds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.head(20)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}