{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import re\nimport pandas as pd","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-08-24T18:07:12.338896Z","iopub.execute_input":"2023-08-24T18:07:12.339465Z","iopub.status.idle":"2023-08-24T18:07:12.369750Z","shell.execute_reply.started":"2023-08-24T18:07:12.339404Z","shell.execute_reply":"2023-08-24T18:07:12.368694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fix_text(text: str):\n    # remove punctuations\n    chars_to_ignore = '[-,\\.:;\\'\"!\\?।]'\n    text = re.sub(chars_to_ignore, ' ', text)\n    \n    # match multiple spaces and replace them with a single space\n    text = re.sub(r'\\s+', ' ', text).strip()\n    \n    return text","metadata":{"execution":{"iopub.status.busy":"2023-08-24T18:07:12.372814Z","iopub.execute_input":"2023-08-24T18:07:12.373615Z","iopub.status.idle":"2023-08-24T18:07:12.379724Z","shell.execute_reply.started":"2023-08-24T18:07:12.373575Z","shell.execute_reply":"2023-08-24T18:07:12.378912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/macro-normalization/normalized.csv\")\ncleaned = train.normalized.apply(fix_text)\ncleaned.head()","metadata":{"execution":{"iopub.status.busy":"2023-08-24T18:07:12.380792Z","iopub.execute_input":"2023-08-24T18:07:12.381684Z","iopub.status.idle":"2023-08-24T18:07:32.659127Z","shell.execute_reply.started":"2023-08-24T18:07:12.381650Z","shell.execute_reply":"2023-08-24T18:07:32.657896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vocab_list = sorted(set(\"\".join(cleaned)))\nvocab_dict = {v: k for k, v in enumerate(sorted(vocab_list))}\nvocab_dict","metadata":{"execution":{"iopub.status.busy":"2023-08-24T18:07:32.662476Z","iopub.execute_input":"2023-08-24T18:07:32.662954Z","iopub.status.idle":"2023-08-24T18:07:36.369053Z","shell.execute_reply.started":"2023-08-24T18:07:32.662911Z","shell.execute_reply":"2023-08-24T18:07:36.367798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vocab_dict[\"|\"] = vocab_dict[\" \"]\ndel vocab_dict[\" \"]\n\nvocab_dict[\"[UNK]\"] = len(vocab_dict)\nvocab_dict[\"[PAD]\"] = len(vocab_dict)\nlen(vocab_dict)","metadata":{"execution":{"iopub.status.busy":"2023-08-24T18:07:36.370598Z","iopub.execute_input":"2023-08-24T18:07:36.371592Z","iopub.status.idle":"2023-08-24T18:07:36.391696Z","shell.execute_reply.started":"2023-08-24T18:07:36.371550Z","shell.execute_reply":"2023-08-24T18:07:36.390554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\nwith open('vocab.json', 'w') as vocab_file:\n    json.dump(vocab_dict, vocab_file)","metadata":{"execution":{"iopub.status.busy":"2023-08-24T18:07:36.393009Z","iopub.execute_input":"2023-08-24T18:07:36.393358Z","iopub.status.idle":"2023-08-24T18:07:36.402434Z","shell.execute_reply.started":"2023-08-24T18:07:36.393331Z","shell.execute_reply":"2023-08-24T18:07:36.401322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! ls kenlm","metadata":{"execution":{"iopub.status.busy":"2023-08-24T18:07:36.403892Z","iopub.execute_input":"2023-08-24T18:07:36.404605Z","iopub.status.idle":"2023-08-24T18:07:37.522038Z","shell.execute_reply.started":"2023-08-24T18:07:36.404573Z","shell.execute_reply":"2023-08-24T18:07:37.520448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%sh\nsudo apt install -y build-essential cmake libboost-system-dev libboost-thread-dev libboost-program-options-dev libboost-test-dev libeigen3-dev zlib1g-dev libbz2-dev liblzma-dev > /dev/null\nwget -O - https://kheafield.com/code/kenlm.tar.gz | tar xz\nmkdir kenlm/build && cd kenlm/build && cmake .. && make -j2 > /dev/null","metadata":{"execution":{"iopub.status.busy":"2023-08-24T18:09:57.971656Z","iopub.execute_input":"2023-08-24T18:09:57.972108Z","iopub.status.idle":"2023-08-24T18:10:00.410092Z","shell.execute_reply.started":"2023-08-24T18:09:57.972074Z","shell.execute_reply":"2023-08-24T18:10:00.409415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(\"text.txt\", \"w\") as file:\n    file.write(\" \".join(cleaned.unique()))","metadata":{"execution":{"iopub.status.busy":"2023-08-24T18:09:38.323468Z","iopub.status.idle":"2023-08-24T18:09:38.323889Z","shell.execute_reply.started":"2023-08-24T18:09:38.323692Z","shell.execute_reply":"2023-08-24T18:09:38.323712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! kenlm/build/bin/lmplz -o 5 <\"text.txt\" > \"5gram.arpa\"","metadata":{"execution":{"iopub.status.busy":"2023-08-24T18:10:08.619032Z","iopub.execute_input":"2023-08-24T18:10:08.619494Z","iopub.status.idle":"2023-08-24T18:10:09.737929Z","shell.execute_reply.started":"2023-08-24T18:10:08.619461Z","shell.execute_reply":"2023-08-24T18:10:09.736709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! head -20 5gram.arpa","metadata":{"execution":{"iopub.status.busy":"2023-08-24T18:09:38.326967Z","iopub.status.idle":"2023-08-24T18:09:38.327359Z","shell.execute_reply.started":"2023-08-24T18:09:38.327176Z","shell.execute_reply":"2023-08-24T18:09:38.327197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(\"5gram.arpa\", \"r\") as read_file, open(\"5gram_correct.arpa\", \"w\") as write_file:\n    has_added_eos = False\n    for line in read_file:\n        if not has_added_eos and \"ngram 1=\" in line:\n            count=line.strip().split(\"=\")[-1]\n            write_file.write(line.replace(f\"{count}\", f\"{int(count)+1}\"))\n        elif not has_added_eos and \"<s>\" in line:\n            write_file.write(line)\n            write_file.write(line.replace(\"<s>\", \"</s>\"))\n            has_added_eos = True\n        else:\n            write_file.write(line)","metadata":{"execution":{"iopub.status.busy":"2023-08-24T18:09:38.329474Z","iopub.status.idle":"2023-08-24T18:09:38.329868Z","shell.execute_reply.started":"2023-08-24T18:09:38.329681Z","shell.execute_reply":"2023-08-24T18:09:38.329699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! head -20 5gram_correct.arpa","metadata":{"execution":{"iopub.status.busy":"2023-08-24T18:09:38.331291Z","iopub.status.idle":"2023-08-24T18:09:38.331687Z","shell.execute_reply.started":"2023-08-24T18:09:38.331477Z","shell.execute_reply":"2023-08-24T18:09:38.331495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! kenlm/build/bin/build_binary 5gram_correct.arpa 5gram.bin\n! rm -rf kenlm","metadata":{},"execution_count":null,"outputs":[]}]}