{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path_lexicon = '../input/indicword/IndicWord/IndicWord/Bangla/Bangla_lexicon.txt'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(path_lexicon,'r',encoding='utf-16') as f:\n    lexicon = [word.strip() for word in f.readlines()]\n# lexicon","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path_class_map = '../input/bengaliai-cv19/class_map.csv'\ndf_map = pd.read_csv(path_class_map)\ndf_map.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_root = df_map.groupby('component_type').get_group('grapheme_root')\ndf_root.index = df_root['label']\ndf_root = df_root.drop(columns = ['label','component_type'])\ndf_root.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_vd = df_map.groupby('component_type').get_group('vowel_diacritic')\ndf_vd.index = df_vd['label']\ndf_vd = df_vd.drop(columns = ['label','component_type'])\ndf_vd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_cd = df_map.groupby('component_type').get_group('consonant_diacritic')\ndf_cd.index = df_cd['label']\ndf_cd = df_cd.drop(columns = ['label','component_type'])\ndf_cd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"word = 'র্য'\nprint('{} = {}'.format(word, [u for u in word]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for wd in lexicon[:50]:\n    print('word: {}, unicode elements: {}'.format(wd,[uc for uc in wd]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def word2grapheme(word):\n    \n    graphemes = []\n    grapheme = ''\n    i = 0\n    while i < len(word):    \n        grapheme+=(word[i])\n#         print(word[i], grapheme, graphemes)\n        # deciding if the grapheme has ended\n        if word[i] in ['\\u200d','্']:\n            # these denote the grapheme is contnuing\n            pass\n        elif  word[i] == 'ঁ': # 'ঁ' always stays at the end\n            graphemes.append(grapheme)\n            grapheme = ''\n        elif word[i] in list(df_root.values)+ ['়']: \n            # root is generally followed by the diacritics\n            # if there are trailing diacritics, don't end it\n            if i+1 ==len(word):\n                graphemes.append(grapheme)\n            elif word[i+1] not in ['্', '\\u200d', 'ঁ', '়'] + list(df_vd.values):\n                # if there are no trailing diacritics end it\n                graphemes.append(grapheme)\n                grapheme = ''\n\n        elif word[i] in df_vd.values: \n            # if the current character is a vowel diacritic\n            # end it if there's no trailing 'ঁ' + diacritics\n            # Note: vowel diacritics are always placed after consonants \n            if i+1 ==len(word):\n                graphemes.append(grapheme)\n            elif word[i+1] not in ['ঁ'] + list(df_vd.values):\n                graphemes.append(grapheme)\n                grapheme = ''                \n\n        i = i+1\n        # Note: df_cd's are constructed by df_root + '্'\n        # so, df_cd is not used in the code\n\n    return graphemes\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"word_list = ['আর্দ্র','ওরা','হিজড়াদের','শ্য়পূ','আবহাওয়াবিদ্যা','প্রকাশ্যে',\n             'প্রিথ্রীব্রি', 'য়ন্তে', 'য়র্সে','ধ্য়য়নে', 'খ্য়যো', 'মায়াবি', '়য়বা',\n             'ন্দর্যে', 'সৌন্দর্য','উৎকৃষ্টতম্', 'হতাে','ফেক্সােফেনাডিন', 'জুতশীঅশােক',\n            'টাের্মিনাসগুলির', 'র্যােককুন']\n# word_list = [ ]\nfor wd in word_list:\n    print('word: {}, unicode elements: {}, graphemes: {}'.format(\n         wd,[uc for uc in wd], word2grapheme(wd)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for wd in lexicon:\n    if 'ঁ' in [uc for uc in wd]:\n        print('word: {}, unicode elements: {}, graphemes: {}'.format(wd,[uc for uc in wd], word2grapheme(wd)))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.DataFrame(\n    data = {'lexicon': lexicon, 'graphemes': ['--'.join(word2grapheme(wd.replace('-', ''))) for wd in lexicon]},\n    columns = ['lexicon','graphemes']\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head(50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.to_csv('indicword_bengali_lexicon_grapheme.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"word = 'দুুর্জন'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"[u for u in word]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"word2grapheme(word)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}