{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"! pip install -q bnunicodenormalizer\n! pip install -q git+https://github.com/csebuetnlp/normalizer","metadata":{"execution":{"iopub.status.busy":"2023-07-26T00:52:27.076229Z","iopub.execute_input":"2023-07-26T00:52:27.076709Z","iopub.status.idle":"2023-07-26T00:52:56.804548Z","shell.execute_reply.started":"2023-07-26T00:52:27.076673Z","shell.execute_reply":"2023-07-26T00:52:56.803070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport tqdm\nfrom bnunicodenormalizer import Normalizer\nfrom normalizer import normalize","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-26T00:53:10.684065Z","iopub.execute_input":"2023-07-26T00:53:10.684527Z","iopub.status.idle":"2023-07-26T00:53:10.860311Z","shell.execute_reply.started":"2023-07-26T00:53:10.684473Z","shell.execute_reply":"2023-07-26T00:53:10.859130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/bengaliai-speech/train.csv\")\nbnorm = Normalizer()\n\nindices = [795715, 951198]\ncorrected_sentence = \"প্রস্তুতপ্রণালি: কাগজ তৈরির জন্য প্রয়োজনীয় উপকরণগুলো যন্ত্রের সাহায্যে চূর্ণ করে নিতে হয়।\"\ntrain.iloc[indices, 1] = corrected_sentence\n\nnormalized = []\nspecial_cases = []\nfor idx, sentence in enumerate(tqdm.tqdm(train[\"sentence\"])):\n    sentence = normalize(sentence)\n    words = sentence.split()\n    try:\n        normalized.append(\" \".join(bnorm(word)[\"normalized\"] for word in words))\n    except TypeError:\n        normalized.append(sentence)\n        special_cases.append(idx)\n        \ntrain[\"normalized\"] = normalized","metadata":{"execution":{"iopub.status.busy":"2023-07-26T00:54:31.539094Z","iopub.execute_input":"2023-07-26T00:54:31.540222Z","iopub.status.idle":"2023-07-26T01:56:53.619191Z","shell.execute_reply.started":"2023-07-26T00:54:31.540174Z","shell.execute_reply":"2023-07-26T01:56:53.617973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.iloc[special_cases, 1].tolist()","metadata":{"execution":{"iopub.status.busy":"2023-07-26T01:58:05.250679Z","iopub.execute_input":"2023-07-26T01:58:05.251927Z","iopub.status.idle":"2023-07-26T01:58:05.260667Z","shell.execute_reply.started":"2023-07-26T01:58:05.251884Z","shell.execute_reply":"2023-07-26T01:58:05.259544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corrected = [\n    'সেই পাতায় লেখা আছে: শ্রীমতী হেমনলিনীর প্রতি অক্ষয়শ্রদ্ধার উপহার।',\n    'স্ত্রী: নূর মেহের বেগম।',\n    'স্ত্রী: নূর মেহের বেগম।',\n    'স্ত্রী: নূর মেহের বেগম।',\n    'উদাহরণ: অটোমোবাইল দ্বারা ঘোড়ায় টানা রথ প্রতিস্থাপন।',\n    'সেই পাতায় লেখা আছে: শ্রীমতী হেমনলিনীর প্রতি অক্ষয়শ্রদ্ধার উপহার।',\n    'উদাহরণ: অটোমোবাইল দ্বারা ঘোড়ায় টানা রথ প্রতিস্থাপন।'\n]\n\nnormalized = []\nfor sentence in corrected:\n    words = sentence.split()\n    normalized.append(\" \".join(bnorm(word)[\"normalized\"] for word in words))\n\ntrain.iloc[special_cases, 3] = normalized","metadata":{"execution":{"iopub.status.busy":"2023-07-26T01:59:38.997874Z","iopub.execute_input":"2023-07-26T01:59:38.998321Z","iopub.status.idle":"2023-07-26T01:59:39.043134Z","shell.execute_reply.started":"2023-07-26T01:59:38.998287Z","shell.execute_reply":"2023-07-26T01:59:39.042052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from collections import Counter\ninput_str = \"\".join(train.normalized.unique())\n\nCounter(input_str)","metadata":{"execution":{"iopub.status.busy":"2023-07-26T02:01:18.685692Z","iopub.execute_input":"2023-07-26T02:01:18.686902Z","iopub.status.idle":"2023-07-26T02:01:22.539903Z","shell.execute_reply.started":"2023-07-26T02:01:18.686860Z","shell.execute_reply":"2023-07-26T02:01:22.538750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"set(input_str)","metadata":{"execution":{"iopub.status.busy":"2023-07-26T02:01:26.179503Z","iopub.execute_input":"2023-07-26T02:01:26.179884Z","iopub.status.idle":"2023-07-26T02:01:27.594545Z","shell.execute_reply.started":"2023-07-26T02:01:26.179854Z","shell.execute_reply":"2023-07-26T02:01:27.593594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.to_csv(\"normalized.csv\", index=False)","metadata":{},"execution_count":null,"outputs":[]}]}