{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from datasets import load_dataset, concatenate_datasets\nfrom IPython.display import Audio","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-19T13:47:12.721436Z","iopub.execute_input":"2023-07-19T13:47:12.722078Z","iopub.status.idle":"2023-07-19T13:47:13.485721Z","shell.execute_reply.started":"2023-07-19T13:47:12.722014Z","shell.execute_reply":"2023-07-19T13:47:13.484450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# A List of Bengali ASR Datasets available in the  🤗 Datasets hub\n* [Fleurs Dataset](https://huggingface.co/datasets/google/fleurs/viewer/bn_in/train)\n* [OpenSLR](https://huggingface.co/datasets/openslr/viewer)\n* [Common Voice](https://huggingface.co/datasets/mozilla-foundation/common_voice_13_0) This Dataset requires approval to be accesed\n\n## These datasets are  saved to Kaggle in [bengali-asr-datasets](https://www.kaggle.com/datasets/ksmcg90/bengali-asr-datasets)","metadata":{}},{"cell_type":"markdown","source":"# [Fleurs Dataset](https://huggingface.co/datasets/google/fleurs/viewer/bn_in/train)","metadata":{}},{"cell_type":"code","source":"fleurs_bn_train = load_dataset(\"google/fleurs\", \"bn_in\", split=\"train\")\nfleurs_bn_valid = load_dataset(\"google/fleurs\", \"bn_in\", split=\"validation\")\nfleurs_bn_test = load_dataset(\"google/fleurs\", \"bn_in\", split=\"test\")","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fleurs_bn_train = fleurs_bn_train.add_column(\"split\", [\"train\"] * len(fleurs_bn_train))\nfleurs_bn_valid = fleurs_bn_valid.add_column(\"split\", [\"valid\"] * len(fleurs_bn_valid))\nfleurs_bn_test = fleurs_bn_test.add_column(\"split\", [\"test\"] * len(fleurs_bn_test))","metadata":{"execution":{"iopub.status.busy":"2023-07-19T13:50:56.752057Z","iopub.execute_input":"2023-07-19T13:50:56.752393Z","iopub.status.idle":"2023-07-19T13:50:56.807726Z","shell.execute_reply.started":"2023-07-19T13:50:56.752363Z","shell.execute_reply":"2023-07-19T13:50:56.806487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fleurs_bn = concatenate_datasets([fleurs_bn_train, fleurs_bn_valid, fleurs_bn_test])\nfleurs_bn = fleurs_bn.rename_column(\"transcription\", \"sentence\")\nfleurs_bn","metadata":{"execution":{"iopub.status.busy":"2023-07-19T13:50:56.811954Z","iopub.execute_input":"2023-07-19T13:50:56.812458Z","iopub.status.idle":"2023-07-19T13:50:57.221424Z","shell.execute_reply.started":"2023-07-19T13:50:56.812415Z","shell.execute_reply":"2023-07-19T13:50:57.220350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = fleurs_bn[0]\nprint(sample['sentence'])\nAudio(sample['audio']['array'], rate=sample['audio']['sampling_rate'])","metadata":{"execution":{"iopub.status.busy":"2023-07-19T13:50:57.222885Z","iopub.execute_input":"2023-07-19T13:50:57.223303Z","iopub.status.idle":"2023-07-19T13:51:12.880458Z","shell.execute_reply.started":"2023-07-19T13:50:57.223271Z","shell.execute_reply":"2023-07-19T13:51:12.879510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample","metadata":{"execution":{"iopub.status.busy":"2023-07-19T13:51:12.881977Z","iopub.execute_input":"2023-07-19T13:51:12.882905Z","iopub.status.idle":"2023-07-19T13:51:12.891024Z","shell.execute_reply.started":"2023-07-19T13:51:12.882869Z","shell.execute_reply":"2023-07-19T13:51:12.889992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fleurs_bn.save_to_disk(\"./fleurs_bn\")","metadata":{"execution":{"iopub.status.busy":"2023-07-19T13:51:12.892655Z","iopub.execute_input":"2023-07-19T13:51:12.892996Z","iopub.status.idle":"2023-07-19T13:51:20.283543Z","shell.execute_reply.started":"2023-07-19T13:51:12.892967Z","shell.execute_reply":"2023-07-19T13:51:20.282586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# [OpenSLR](https://huggingface.co/datasets/openslr/viewer)","metadata":{}},{"cell_type":"code","source":"openslr_bn = load_dataset('openslr', 'SLR53', split='train')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = openslr_bn[0]\nprint(sample['sentence'])\nAudio(sample['audio']['array'], rate=sample['audio']['sampling_rate'])","metadata":{"execution":{"iopub.status.busy":"2023-07-19T14:01:52.826600Z","iopub.execute_input":"2023-07-19T14:01:52.826982Z","iopub.status.idle":"2023-07-19T14:01:53.770758Z","shell.execute_reply.started":"2023-07-19T14:01:52.826949Z","shell.execute_reply":"2023-07-19T14:01:53.769540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample","metadata":{"execution":{"iopub.status.busy":"2023-07-19T14:01:53.774541Z","iopub.execute_input":"2023-07-19T14:01:53.775065Z","iopub.status.idle":"2023-07-19T14:01:53.783371Z","shell.execute_reply.started":"2023-07-19T14:01:53.775029Z","shell.execute_reply":"2023-07-19T14:01:53.782270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"openslr_bn.save_to_disk(\"./openslr_bn\")","metadata":{"execution":{"iopub.status.busy":"2023-07-19T14:01:53.785190Z","iopub.execute_input":"2023-07-19T14:01:53.785649Z","iopub.status.idle":"2023-07-19T14:01:54.518148Z","shell.execute_reply.started":"2023-07-19T14:01:53.785602Z","shell.execute_reply":"2023-07-19T14:01:54.516962Z"},"trusted":true},"execution_count":null,"outputs":[]}]}