{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":52324,"databundleVersionId":6229904,"sourceType":"competition"},{"sourceId":4143520,"sourceType":"datasetVersion","datasetId":2447262},{"sourceId":6707460,"sourceType":"datasetVersion","datasetId":3865741}],"dockerImageVersionId":30528,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\nimport torch\n\nfrom transformers import pipeline, AutoModelForTokenClassification, AutoTokenizer\nPUNCT_MODELS = [\n    '/kaggle/input/bengali-ai-asr-submission/punct-model-6layers/',\n    '/kaggle/input/bengali-ai-asr-submission/punct-model-8layers/',\n    '/kaggle/input/bengali-ai-asr-submission/punct-model-11layers/',\n    '/kaggle/input/bengali-ai-asr-submission/punct-model-12layers/'\n]\n\nPUNCT_WEIGHTS = [[1.0, 1.4, 1.0, 0.8]]\n\nmodels = [\n    AutoModelForTokenClassification.from_pretrained(f).eval() for f in PUNCT_MODELS\n]\ntokenizer = AutoTokenizer.from_pretrained(PUNCT_MODELS[0])\ndef punctuate(text):\n    input_ids = tokenizer(text).input_ids\n    with torch.no_grad():\n        model = models[0]\n        logits = torch.nn.functional.softmax(\n            model(input_ids=torch.LongTensor([input_ids])).logits[0, 1:-1],\n            dim=1).cpu()\n        for model in models[1:]:\n            logits += torch.nn.functional.softmax(\n                model(input_ids=torch.LongTensor([input_ids])).logits[0, 1:-1],\n                dim=1).cpu()\n        logits = logits / len(models)\n        logits *= torch.FloatTensor(PUNCT_WEIGHTS)\n        label_ids = torch.argmax(logits, dim=-1)\n\n        tokens = tokenizer(text, add_special_tokens=False).input_ids\n        punct_text = \"\"\n        for index, token in enumerate(tokens):\n            token_str = tokenizer.decode(token)\n            if '##' not in token_str:\n                punct_text += \" \" + token_str\n            else:\n                punct_text += token_str[2:]\n            punct_text += ['', '।', ',', '?'][label_ids[index].item()]\n\n    punct_text = punct_text.strip()\n    return punct_text","metadata":{"execution":{"iopub.status.busy":"2024-04-17T04:42:02.605804Z","iopub.execute_input":"2024-04-17T04:42:02.607159Z","iopub.status.idle":"2024-04-17T04:42:43.884651Z","shell.execute_reply.started":"2024-04-17T04:42:02.607106Z","shell.execute_reply":"2024-04-17T04:42:43.883494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"punctuate(\"আসসালামুয়ালাইকুম ট্রিপল থ্রি থেকে রেজওয়ান বলছি কিভাবে সহযোগিতা করতে পারি স্যার কালকে ফোন দিছিলাম বুঝছেন \")","metadata":{"execution":{"iopub.status.busy":"2024-04-17T04:58:37.870561Z","iopub.execute_input":"2024-04-17T04:58:37.870976Z","iopub.status.idle":"2024-04-17T04:58:38.136910Z","shell.execute_reply.started":"2024-04-17T04:58:37.870940Z","shell.execute_reply":"2024-04-17T04:58:38.135962Z"},"trusted":true},"execution_count":null,"outputs":[]}]}