{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Sentence Piece tokenizer trained on bengaliai-speech competation text data\n\nfor more inf about **Sentence Piece** algorithm and how it work plz refer to official repo can be found [here](https://github.com/google/sentencepiece) and read this [blog](https://colabdoge.medium.com/understanding-sentencepiece-under-standing-sentence-piece-ac8da59f6b08) to get full understanding of how it work \n\n![SentencePiece components (blue) as part of the tokenisation process](attachment:77d4f053-6419-43bc-a77d-493be6fee2f2.png)","metadata":{},"attachments":{"77d4f053-6419-43bc-a77d-493be6fee2f2.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAhUAAAIQCAIAAAC5SwuaAAAgAElEQVR4nO3df3xU1Z3/8U++Hb4rGVgRmDApvxJqKSAMtchSQrZZlsEfmBapNpLAauHb6AIyPBDb+lBKm0j3oVb0a2LVldZUahJKa2ksUJRk+cYmQRYpdYKEopIJwU3IAEbNEPuY6yPfP069vc4kk3DyY27g9Xzwx8w95557Zph733POvXeS0NHRIQAAXKT/Fe8OAAAGJfIDAKCD/AAA6CA/AAA6yA8AgA7yAwCgg/wAAOggPwAAOsgPAIAO8gMAoIP8AADoID8AADrIDwCADvIDAKCD/AAA6CA/AAA6yA8AgA7yAwCgg/wAAOggPwAAOsgPAIAO8gMAoIP8AADoID8AADrIDwCADvIDAKCD/AAA6CA/AAA6yA8AgA7yAwCgg/wAAOggPwAAOsgPAIAO8gMAoIP8AADoID8AADrIDwCADvIDAKCD/AAA6CA/AAA6yA8AgA7yAwCgwxHvDkhCQkK8u4DBpKOjI95dACDC+AMAoCf+4w+FL5XoFkNVwFYYfwAAdJAfAAAd5AcAQAf5AQDQQX4AAHSQHwAAHeQHAEAH+QEA0GGX+wf7WygUCgaDLpfL6XTGuy8iIoFAoKamJi0tTUTs0ysA6LnLZfxRVlaWmppaVlbW3xsKhUIlJSUxKuzZs2fmzJmpqanLli1LTU1NTU29+uqr9+zZMzBbB4C+crnkx4CprKxctmxZV6WhUOjmm2/2+/27d+9uaWlpaWnZvXu3iNx8881PPfVUf28dAPrQZZofgUAgGAyqx9XV1dXV1V2V7tmzJxAIdFVqLlF1AoHAK6+80mkdpaioSESKi4sXLVrkcrlcLteiRYv8fr/b7a6pqbHWNAyjurq6pKQkEAgYhtGTnne1dY2mTMFgUL26UCgUUdRVswAuFx3xNjDdKC4uFpHi4mJzox6Pp76+3uv1qg643e76+nprqd/v93g8qjQ7O9tamp2d3elLsL6xEXWs3fB6vTG6Gg6H8/Ly3G632ZTX621ra+u259Fb125KycvLM1d0u91+v78nPew/NvnEAlDivzfGKz/U0bOqqiocDhcWFopIXl6etdTtdhcXF7e0tGRnZ1vzIEZ+dHR0qMpddSMcDqvDrs/nsx6praydMfvm8/l60vOIrfemKRUe2dnZLS0tZpCoCIndbP8hPwBbif/eGMf8qKqqUk/D4bAKDGup+XW7o6NDfUM3S7Xzo6Ojo6qqyvrNPTs7Wx2+zQoejydigKKGQeoLfuyeR2y9N02paLG+gWbmxW62/5AfgK1cLtfvdmrOnDnqgcPhEJHm5mZrqfUo73K5RERdAdzLjc6bN6+pqam2traysrKsrKy0tLS0tFRE6uvrU1JSQqGQmjezXkaVlJSktm5e5hu750pvmlLnRa655hpzxZycnJycnJ43C+CSd1nnhzpidsUaFZmZmaWlpaFQqPf5ocyYMWPGjBn33HOPYRgbNmwoKCh44IEHSkpKTp48KSJ+vz/2ZVSxe670pimVIpmZmdrNArjkXabXX/WE9YqjXbt2iYj55Tri6qaLajaivsPh2LJli9vtLi0tDQaDw4cPly7OvaekpFzUhnrT1KRJk+TTV92HzQK4lJAfXVJftBUVGGrw4Xa7y8vLzRioq6vrYYOhUCghIWH8+PER18L+9a9/VQ8SExNVRKkZLZPeBbK9acrpdHq93v3795tLamtrc3JySkpK+rCHAAY18qNLy5cvr66uNgwjPz+/vLxcnZoWkVWrVonIs88+q26AWL58uXUtlTGd3kvhdDoLCwubm5tvueWW6urqUChkGEYgEMjNzW1ubvb5fE6n0+VyqQ2tW7cuGAwGg8GcnJzU1NRZs2b1pM/WrfeyqcWLFzc3N+fk5Kiffrn33ntLS0sXL17cy2YBXDr6/pT8RRqYbnR6/VVX3VCPrRdKWe//qK+vN5er+yfU1UeqtK2tLfZ7a+aQlbpM1mwh4u4Kt9ttlsbuecTWe9NU9E0ehYWFPelh/7HJJxaAktDx2fvOBl5CQoJE3XzX58zfK1Rz9OraIXVBkWJdYnbJMIxXX3112rRpETP7hmEcPHjwgw8+WLRokYhUV1c3NDSYrVVXV48dO1ZEujofEAqFKisrW1tb1VOzV9GbaGhomDFjxtSpU82z3LF73unWtZtS67766qsikpGREXFtVVfN9p+B+agA6KHLJT8uig27BOH/BbAZzn8AAHSQHwAAHZf1/YNdqa+vj3cXAMDuOP+BQYOPCmArzF8BAHSQHwAAHeQHAEAH+QEA0EF+AAB0kB8AAB3kBwBAB/kBANBBfgAAdJAfAAAddvn9K/XTFACAwYLxBwBAR/x/PxHR+KFAAPbH+AMAoIP8AADoID8AADrIDwCADvIDAKCD/AAA6CA/AAA6yA8AgA7yAwCgg/wAAOggPwAAOsgPAIAO8gMAoIP8AADoID8AADrIDwCADvIDAKCD/AAA6HDEuwOQQCBQU1MTvbykpMT6NC0tLSUlZaA6BQDd4O+fx18wGExKSuq2WktLi8vlGoD+AEBPMH8Vfy6Xy+PxxK7j8XgIDwC2Qn7YQm5ubi8rAMAAY/7KFrqdwmLyCoDdMP6wBZfL5fV6uyr1er2EBwC7IT/sYv369RpFABAvzF/ZRSgUGjZsWKdFbW1tTqdzgPsDALEx/rALp9PZ6RSW1+slPADYEPlhI53OUzF5BcCemL+yEcMwhgwZErEwHA47HPxMAADbYfxhIw6Hw+fzWZf4fD7CA4A9kR/2kpWVFeMpANgH81f2EjGFxeQVANti/GEv1iksJq8A2Bn5YTvmnBWTVwDsjPkrO0pOThaRpqameHcEALrE9IgdrVq1Kt5dAIBukB92tGTJknh3AQC6Ef/8SEhIiHcXMJgw4wrYBOfPAQA64j/+UCpPt8e7C7C7jHFD490FAH/H+AMAoIP8AADoID8AADrIDwCADvIDAKCD/AAA6CA/AAA6yA8AgA7yo0vtF0JNjQ2t54LRRV0tH2AZ44aat9RljBuav+bO+PYHwGWF/OhS1Su/Xzp3yuKZE6KjYuncKQWb7otLr7qyYHHW9Ou+Gu9eALiM2OX3S+wsf82dj2/fE+9edGPTT1+IdxcAXF4Yf3Rj0tTph6v2H6jYG7uaYRi1h2oOVOw1DMO6vPVcsKmxQUSaGhtUI02NDeaSfTu3v1t31FxF1Ylowax5oGJv+4VQVx2ImFJTk29NjQ2drmIYxoGKvft2blc9iWjEMIymxobaQzWxXzKAyxzjj25sLCi6Lyfz0fv+vaT6raGJzugKhmG8WPhI2bat54Nn1JIVGzbecsddI0a5RKRg030VZTtWbNhYtGWziFSebt/68KaKsh23rlz90vNPq/qz0uc/+uLLLxY+ouqMdI15rGTXF6ZOV41/a/bVZsuqcqeDoaVzpyxYnKVGIb8teubJH9xrLX1+3yHVYOu54O+2Pac2ZN26+kPrqpH6E8dO1h1dsDhrxuy0XrxzAC5xjD+6kThs+Pcee/Z88MzWhzd1WkEd969Ny9h+4PjeE2dVVGx78mFrnbJtWzcWFj2/75C55MiB157fd6jszVOz0ucfrtq/IGW4iOw9cfbhF3aeD57ZXVpkNn4+eGbdQ49Xnm7fe+LsgsVZh6v2v1t3NEaH2y+EnvzBvZOmTt974mzl6fbtB46PdI25LydTDWtUeKzYsLHydHtF4KNbV64+XLX/xcJHzNUrynaIyMbCIl/+Y5pvGYDLA/nRvbkLblywOOul55/u9MCtvstv+ukLyeMnDk10fnv9g6qytc6/+b6/cMlSNQJQNhYUfWHq9BGjXDfdfoeIzEqf/+31Dw5NdM5dcKOI7P/9S6raDbct31hY9I1/yxWRoYnOZfd8V0T+uLcsRm//2n5BRK4a5Rryv/9BRJLHTyypfquo/JAaYZRt26q2JSIOh8OXv2Wka0zZtq3WFu79jycXLlmqxk8A0BXmr3rEl/9YRdmOzb4VW/9wQB2IFXXKYVb6fGvlCVdPFpGmxobk8RPVkn/9xm0RDU784hT1YPp1c0Uk/Yavm0ULFmepQYCIJI+fODQxseHt4yePHxWRuiOHROTUOydidHXEKJdqIfemuTmrN8z+2oIRo1xq5q31XPB88My1aRn7dm7/e/3RrpN1R9svhMzZOaatAPQE+dEjI0a5NhYWbV674uVfbv3milXm8vYLF1SptfLYlC+IyNE3Dpj5Ef1d3hpCIjJ8xFWdbtc8kzErff6IUa4e3nTiy39swtWTy7Zt3bx2hYhMmjpdDXcOvVYhIhVlO8x8MrWeO9vp2R0A6ArzVz01/+u3TZo6/ckf3Gs9iLuSx8qnoxDTe4F3RcQ9bkIvt6jOZIjIuocef3z7nk0/fWHFho09WXHEKNe31z+480hg+4Hjt65cfbLu6GbfCrNLt65cXXm6PeKfGXWTLJNsABAD+dFTDofjP57/jYhYb/N2OBzqnLa1pv9gtYiMdo/t5Rb/p6FeRCZNnW6OeCI21Cl15a56nDx+oi9/y4LFWSfrjjY1NqguHTnwmrV+U2OD9Yrh1MnTetltAJcJ8uMiJI+fqC5Ysi5c+M1sEclfc6e62eIXT/z4cNX+SVOnm9/otX1+YqqInKw7um/ndtWyOldfUbbjt0XPdLXWnw/8cencKb944scqFQzDOFJTOdI1xpU8Nnn8xFnp80/WHf3FEz9WMXPv0kVL50753vJv9LKrAC5D5MfFWb3pkZGuMdYlszO86x56/EhN5dK5U26cPLpoy+ZbV65+oi/uVx+a6Lx15WoR2bx2hWr5qZ0Vagor4vYOq7kLbrx15eqiLZsXpAxfcm3KgpTh54NnVm96WJ1xUZNgRVs23zh59NK5Uw5X7R/pGvPoiy/3vrcALjcJHR0dce5BQoKIVJ5uj283ojU1Nhx940D6DV+POLH8bt3Rk8ePusdNsF6nZBhG3ZH/bvvww9kZXuu58dpDNc2nTy1csrSrJe0XQlWv/H76dXPN8UpEhdZzwUOvVbjHTbj6mplDE52GYQSb3hOR5PET1WVUqua+ndutXWpqbAicqPuL//CXPLO+PPefI16CYRiHKsvbPmy1bje6EbtRPxYZ908sAIX8wKBBfgC2wvwVAEAH+QEA0EF+AAB0kB8AAB3kBwBAB/kBANBBfgAAdJAfAAAd5AcAQAf5AQDQQX4AAHSQHwAAHeQHAEAH+QEA0GGX328Heijun1gACuMPAICO+I8/EE2NyfivAWBnjD8AADrIDwCADvIDAKCD/AAA6CA/AAA6yA8AgA7yAwCgg/wAAOggPwAAOsgPAIAO8gMAoIP8AADoID8AADrIDwCADvIDAKCD/AAA6CA/AAA6yA8AgA5HvDsAqaqqKi0tjV6+Zs0a69Ps7Oz09PSB6hQAdIO/fx5/dXV106ZN67basWPHpk6dOgD9AYCeID9sYejQoR9//HGMCldccUV7e/uA9QcAusX5D1vIzMzsZQUAGGCMP2yh2yksJq8A2A35YRejRo06f/58p0UjR448d+7cAPcHAGJj/soufD6fRhEAxAvjD7toaWkZM2ZMp0VnzpxJSkoa4P4AQGzkh410OoXF5BUAe2L+ykY6nadi8gqAPTH+sJGPP/546NChEQvb29uvuOKKuPQHAGJg/GEjV1xxxcyZM61LZs6cSXgAsCfyw17uvvvuGE8BwD6Yv7KXiCksJq8A2BbjD3uxTmExeQXAzsgP2zHnrJi8AmBnzF/Z0ec+9zkR+eSTT+LdEQDoEn8/yo4yMjLi3QUA6Ab5YUcRf3kQAGwo/vmRkJAQ7y5gMLmEZ1zZF3BR4r4vcP4cAKAj/uMPpfI0f5wV3cgYF/njLpck9gV0yyb7AuMPAIAO8gMAoIP8AADoID8AADrIDwCADvIDAKCD/AAA6CA/AAA67HL/4KDTfiHUeu5sV6XJ4ycOZGfUzUTcd4a4Mwwj2PRejAqu5LEOx0UcdpoaG5bOnbKxsGjhkqW97h36GPmhqeqV329eu6KrUg7luDwFm95bOndKjArbDxy/qG9XQxMTFyzOco+b0Ouuoe+RH72yYsPGsSlfiHcvALsYMWr0xsIi86n6jmVdMmLU6Its0LXppy/0VffQt8iPXrnhtuUxvkw1NTYMTUwcMcolIrWHakRkxuy0iDqt54LtFy6IyIhRo4cmOq1FhmHUHflvEZl67T9FD/mbGhsCJ+pmZ3gdDses9PmHq/ZHr9t8+tT06+ZaZwxUl4ZdeVWw6b0LbR99Yep0vRcOdGpootM60aTyw7rEMIymxgaVIv/TUJ84bLjagwzDaHj7+MnjR93jJlg/8GpCTO0d1sftF0JVr/x+9tcWqP3LShUN+8cRU788yyw1143YLnqD/OhHS+dOGeka8/TLlT/57ip1fB/pGvNYyS7zqP2LJ35ctGWzemwtaj0X/N2258q2bT0fPKOKFt+Re8sdd6mdofVcsGDTfRVlO1RR/nMl1l3IMIwXCx8x1xWRWenzN/30BVVHdenatIyKsh0LFmdd2t/sDMM4ePBgRUXFM88809TUFO/uQOTTCa51Dz3+y4JHzgfPbCwsSh4/8d26oysXzrZWe/iFnXMX3GjWV+c/zMci8nT+/eoTPit9/uPb96i13q07WvzUT9Suoax76PFvrlhltrNiw0a1x6ntDtSLvmRx/VX/Oh8888DK21Zs2FgR+GjdQ4+fD575494yVaTCY8HirLI3T63YsPF88MzKhbPVMKVg031FWzZfm5ax/cDxvSfOXpuWUbRl8++2PadWzF9zZ0XZjltXrt574mz+cyWPP7DOusO8/MutRVs2L74jt/J0u9ro4ar9BZvus3bpSE3lxsKi3PvzB/CdGDihUGjPnj3r1q0bMmRIenr6D3/4w+bm5nh3Cp/xy4JHUr80bWNhUfoNXzcMY7NvhYhsP3C88nT78/sOTZo6/dH7/r2rdTevXVF35NDTL1eWvXlKjbybGhtU0X05mRVlO57aWVF5un37geOz0uc/+YN73607aq5btGXzrStXP/zCzvQbvt7fr/FyEP+/f67+Zs6gO+G8b+f2rs6fm9eKqMuintpZYU5bZYwbOtI1ZueRQMRj1WDdkUPfyvWNGDX6xsmj5bPvyYqFs0WkaN+h1nPBxTMnTJo6vWjfIVWkLlAx69+7dNH754JmqVpyuGr/3hNnhyY6VZfML3eDi+p8V5/YYDC4b9++oqKi8vLy6NK4f857YpDuCzFEXxloflzL3jylxsTmPK25mxRs2vDS80+rHcd6/VXER11EDlTsvf/OJWqQoXYNc8BhbuvWlat9+VvMdSsCH13UBWD2FHtfGDCD/n2Mr1np86OnXyOuFbn6mpnW+moiq/VcUESuTfv73zlfuGSpSh31ZWrB4ixrI6mTp1WU7TBPlqROnmYWuZLHmo/bL4QOV+2fNHX6vp3bI3rVeu6seX7ly3P/+SJfqH3V1tbu3LnzpZde8vv98e4Lesq64zgcDpUTByr2tn3YKiKBE3Ui0nz6VPT5QhG5deVq83HK5Kkicrr+HRE59FqFiHzYej7iw69aUxYszroEwsM+eCt75bs/eabbWVTrWXFzn1ExMOHqyV2tFVE013tTRdmOxpNvj3aPVU/NIuv+oG5JOVl3NMa1xRFdGozUiY0dO3bs2LGDuanByPqtq/1C6MGV31Lfq9TXpvfPBWOsO/Xa2Z0uOVD+BxExTyiarJeWWHcc9B750Y8mTZ1+0jL3aqWuAzn1zokeFql9Y7R77NDERPXUvKal/UIoooUYJ8ZjdGmwyMnJKS0tvdi1+NPitvXnA39Uh3hzRit/zZ0an1K141ini9HfOH/ej6yzTBGGJjpnpc8/UlNpLmlqbMhfc+e+nduji0Sk/sQxETGvBlZPFett8GoXsp5OF5HWc0HDMLrtEhAXhyr3icjGwiJzUBLxAe4hNQqxjjbUtcJ90Ud0jvzolcCJuqbGhuh/0WOCaDfdfsf54Jn8NXe2ngu2ngv+5LurKsp2TL9uroik3/B1VdR+IdR+IfSLJ358su7ogsVZagdbsDjrZN3Rgk0b2i+Eag/VPLDyNrPNoYlONQNQsGlDU2ND67lg/po7F8+ckHvT3H57DwZaSUlJOByuqqry+Xxut7uHa3UMBv36vtnW7IyFIvJ0/v2t54LqW9RI1xhzSc/bUZdUlW3bum/ndsMwag/VfGv21UvnTvlt0TP91HMwf9Ur99+5pNPlPfm5nvlfv+29wLtl27Yunvm38+3rHnpcnU256fY7Pmw9X7Ztq7oQS0RWbNh4yx13qce+/MdE5KXnn37p+adFZFb6fOv9gw88+fMJV08u27ZVlYrISNeYJz69QP7S4HA45s2bN2/evCeffJLz54Pd7AzvSNeY88EzakcY6RpTVH5o/dJFJ+uOFmy6r+dXmQ9NdD6/71DxUz/ZvHaFef7v1pWrzcux0Oe4fldTU2PD0TcOdFU6/bq5yeMn1h6qaT59yhok0UsMwzhUWS4iX577z9H3n6sidZN5px2YNGX6xC9OUbeaRzSrFqoK5urRHRhEuH530FGXQlk/b+rmcPe4CRFnKdQn0/y4qt8nHZqY+A9DE6te+b3aodS66rG1NesS6eL+805rDl42uX6X/MCg0cN9JhQKVVZWvvLKKwUFBebCuH/Oe4J9AT1EfnzaA/YZ9MzF7jOD7vdL2BfQQ+THpz1gn0HP2GSf6T/sC+ghm+wLXH8FANBBfgAAdJAfAAAd5AcAQAf5AQDQQX4AAHSQHwAAHeQHAEAH+QEA0EF+AAB0kB8AAB3kBwBAh11+PxHoobh/YvsP+wIuStz3BcYfAAAd8R9/IJr6Hsp/DQA7Y/wBANBBfgAAdJAfAAAd5AcAQAf5AQDQQX4AAHSQHwAAHeQHAEAH+QEA0EF+AAB0kB8AAB3kBwBAB/kBANBBfgAAdJAfAAAd5AcAQAf5AQDQQX4AAHQ44t0BSCAQqKmpiV5eUlJifZqWlpaSkjJQnQKAbvD3z+MvGAwmJSV1W62lpcXlcg1AfwCgJ5i/ij+Xy+XxeGLX8Xg8hAcAWyE/bCE3N7eXFQBggDF/ZQvdTmExeQXAbhh/2ILL5fJ6vV2Ver1ewgOA3ZAfdrF+/XqNIgCIF+av7CIUCg0bNqzTora2NqfTOcD9AYDYGH/YhdPp7HQKy+v1Eh4AbIj8sJFO56mYvAJgT8xf2YhhGEOGDIlYGA6HHQ5+JgCA7TD+sBGHw+Hz+axLfD4f4QHAnsgPe8nKyorxFADsg/kre4mYwmLyCoBtMf6wF+sUFpNXAOyM/LAdc86KySsAdsb8lR0lJyeLSFNTU7w7AgBdYnrEjlatWhXvLgBAN8gPO1qyZEm8uwAA3Yh/fiQkJMS7CxhMmHEFbILz5wAAHfEffyh8qUS3GKoCtsL4AwCgg/wAAOggPwAAOsgPAIAO8gMAoIP8AADoID8AADrIDwCADrvcP3gpCYVCwWBQRFJSUqJLg8FgKBRyOp0ul0ujcXUPXU9ut+x5TQDQwPij75WVlaWmpqampoZCoYgiwzA8Hk9qauq6devi0jcA6CvkRz8qKiqKWHLw4MHm5ua4dAYA+hb50V+8Xu/WrVsjFv7oRz9yu93RlQ3DqK6urq6uNgwjujQQCOzZs6fTInPdkpKSQCDQVR0A6HPkR39Zv3693+8PBALmklAoVF5e/uCDD1qrBYPB/Pz8IUOGpKenp6enjx8/Pj8/35z4CgaDOTk5qampN99886xZs6qrq63rGoaRn58/fvz49PT0ZcuWpaam3nTTTdGTZgDQLzrizSbd6EPFxcUiEg6HRSQvL89cXlhYKCItLS0ikp2drRZmZ2erp/X19fX19eqpuZbH4xERn8/X1tZWVVWlnppvl2pQVQ6Hw+qpz+dTpZfeG3vpvSJgUIv/3njpHRRUfnR0dPh8PrfbbS73eDwej6ejo8PMD5Ul1jodHR1ut1tVU6Ver9csqq+vt75dHo/HWtrxad60tbV1XIpv7KX3ioBBjet3+9F3vvOdgoKC6urqefPmBYNBv9+vhggmNdc0f/5868L58+eXlpaGQiFVar3M1/o4FAr5/X6Px1NSUmIuTEpKEpFgMOh0OvvnNQHA35Af/WjGjBlut3vHjh3z5s371a9+JSIrVqywVvjoo49EJC0tzbowMzOztLQ0GAyq0szMTLPImgonT54UEb/fv2zZsv58EQDQOc6f968HH3ywoKDAMIytW7dmZ2dHDAsmTZokIjU1NdaFu3btUg/UlVrmU/l0vKIMHz5cLOdRrDq9bxEA+hb50b9uv/12Ebnjjjv8fv+aNWsiSp1Op8fj2b9/v3WhemreoK5uZVfUmMNcV0RKS0ut63IJL4ABQ370L5fL5fF4SktL3W73nDlzoivk5uY2Nzfn5OSoEx75+fnNzc3Z2dkqPLKzs8vLy9etWxcKhaqrq5cvX25tWV2stW7dumAwaF7pO2vWrAF7dQAuawN/yj6CTbrRh8zrr5Tdu3fLZy/kFcu8U1tbW15envV/JC8vr6WlRZWaV/QqXq/X6/Wajat1rTckut1uc91L74299F4RMKgldMT79/UuvZ/5CwQCNTU1OTk56mkoFCorK1u4cKF59VRJScnEiRPnzZtnrmIYxquvvioi119/vcMReVGDanDGjBlTp049ePBgQ0OD2bhaVy1UFczV1XVZ1pqD3aX3UQEGNfIDgwYfFcBWOP8BANBBfgAAdJAfAAAd5AcAQAf5AQDQQX4AAHSQHwAAHeQHAEAH+QEA0EF+AAB0kB8AAB3kBwBAB/kBANBBfgAAdET+qYl4UT/NDQAYLBh/AAB0xP/vRyEafygJgP0x/gAA6CA/AAA6yA8AgA7yAwCgg/wAAOggPwAAOsgPAIAO8gMAoIP8AADoID8AADrIDwCADvIDAKCD/AAA6CA/AAA6yA8AgA7yAwCgg/wAAOggPwAAOhzx7gAkEAjU1NRELy8pKbE+TWtcoLsAABHySURBVEtLS0lJGahOAUA3+Pvn8RcMBpOSkrqt1tLS4nK5BqA/ANATzF/Fn8vl8ng8set4PB7CA4CtkB+2kJub28sKADDAmL+yhW6nsJi8AmA3jD9sweVyeb3erkq9Xi/hAcBuyA+7WL9+vUYRAMQL81d2EQqFhg0b1mlRW1ub0+kc4P4AQGyMP+zC6XR2OoXl9XoJDwA2RH7YSKfzVExeAbAn5q9sxDCMIUOGRCwMh8MOBz8TAMB2GH/YiMPh8Pl81iU+n4/wAGBP5Ie9ZGVlxXgKAPbB/JW9RExhMXkFwLYYf9iLdQqLySsAdkZ+2I45Z8XkFQA7Y/7KjpKTk0Wkqakp3h0BgC4xPWJHq1atincXAKAb5IcdLVmyJN5dAIBuxD8/EhIS4t0FDCbMuAI2wflzAICO+I8/lJePvR/vLsDuvjHtqnh3AcDfMf4AAOggPwAAOsgPAIAO8gMAoIP8AADoID8AADrIDwCADvIDAKCD/NB05r1T1n/x7Uzlrl9/Y9pVlbt+Hd9uALis2OX+80End+HMiCVfW3Tr7H+5If3GJZ/jjz4BuAxwpNM3c27GPfkFItL4zvF33jqyZ/vzr+15qXxn8YOFxVckOuPdOwDoX8xf6bvyqtFjxk4YM3bCdRnXL139/W2v/SXnnvvfPFB5141f+cQwrDU/OH+2ctevj/3p9Q/On41o5BPDiDEJplaMXkut+EblqzGmzjrd6McXQmfeO/WJYXxw/mzgxFsfXwhd3GsGgE8x/uhLS1d///TJt1/b81Ljyb+kTL5GRAIn3vr5Iw++eaDSrLPpmV9dl3G9evzB+bNrb5nXerZFPc1cfvf/+d5mNf31RuWr257ID5x4SxXNnJuRveb+aV/5qnpauevXP390o1ox5577I7oRY6MH/2vPlu/dlXPP/SVPPSwiW/e9yVAJgB7GH31s9r/cICINJ46JyCeG4bslveHtul9Wvf3ysfe37nszZfI1+atuVwMCFR4isuHR535Z9faI0Um7XvzPH919m4icee9U/qrbW88HNzz63I43Tm949LmGt+vuX36TGi4c+9PrW753l4g8/OIfdrxxWkRUGCixN6qUPPVw5vK7Nz3zq9FjPj+Qbw6ASwn50cemXDtHRA79v1dE5C/+N0Qk/2e/vXLkaBEZM3bCv296TET+sP3nIvLnmv2tZ1vu/78vZGR+68qRowt/V525/O45/7pIRMpeeFpEfA8VZmR+64pEZ0bmt3wPFYrIyeO1IlK193eq2Wlf+eoVic6lq7//tUW3mh2IvVFlxOikux54+LqM6znVD0Abh49+5H+9UkReL9+lhiMi8tEH74vI6ZNvi8hf/IdFZNSnI4ArR46+64G/DSM+PH9WRMZfPcVsSj32v1457StfVaVDncPN0nGTvtjDjSpZd2/o0xcK4HJEfvSx40cOisiUa/9JRN46fEA+O7lk1fjucREZM3ZCdNEH758VEevkkqqmMkCVWldMnjDJfBx7o8rwK/lDTAB6i/zoY2rmyj0uRUSuvGq0iGzd92anIXHNrLlvHqg8896p6FJVdPbM/5hF1uusxn9hSsSKTadOmqWxNwoAfYXzH31p+9OPvLbnpZTJ11w7719F5B9HjpZPRyTKxxdC5nnsr3ozRaTxneNm6XP/cf9j933n4wshz1czIorUY9XglzyzRKQ99JFZqsYcSuyNAkBfIT/0ffD+WfPWjV3Fz/luSVezRvc++p/qvPTiO1eLyM8f3XjsT69/YhiVu359141f+bf0Lx770+siMn7Sl0Sk4Adr36h8Vd3MsevF/xw36YtXJDq/5LnOLBKRY396fdsT+WaD6hT9pu9889ifXv/4Qmj7049YL9WNvVEA6CvMX+l780BlxK+YzJybcd9PfqYufBKRMWMnPPziH0p/+vD9y28y6+Tc87fbOD7ncBT8rurnjzyYv+p2VTRidNIt375HFT384h+ezb/PLEqZfM2mZ36lpqRGj/n8hkef+/mjG81mzfs5ut0oAPSVhI6Ojjj3ICFBRF4+9n58u3GxIn6scMq1c0aP+XxXl8N+cP7sn2v2O4df+cUZXzHTxXTmvVPHjxycOHmauuUwumjKtXOiT2Z8YhhHqv9LRL444yv/cMXQg/+1J6JapxuN0aD9fWPaVSIS908sAIX8wKBBfgC2wvkPAIAO8gMAoIP8AADoID8AADrIDwCADvIDAKCD/AAA6CA/AAA6yA8AgA7yAwCgg/wAAOggPwAAOsgPAIAO8gMAoMMuv98O9FDcP7EAFMYfAAAd8R9/IJoak/FfA8DOGH8AAHSQHwAAHeQHAEAH+QEA0EF+AAB0kB8AAB3kBwBAB/kBANBBfgAAdJAfAAAd5AcAQAf5AQDQQX4AAHSQHwAAHeQHAEAH+QEA0EF+AAB0kB8AAB2OeHcAEggEampqopeXlJRYn6alpaWkpAxUpwCgG/z98/gLBoNJSUndVmtpaXG5XAPQHwDoCeav4s/lcnk8nth1PB4P4QHAVsgPW8jNze1lBQAYYMxf2UK3U1hMXgGwG8YftuByubxeb1elXq+X8ABgN+SHXaxfv16jCADihfkruwiFQsOGDeu0qK2tzel0DnB/ACA2xh924XQ6O53C8nq9hAcAGyI/bKTTeSomrwDYE/NXNmIYxpAhQyIWhsNhh4OfCQBgO4w/bMThcPh8PusSn89HeACwJ/LDXrKysmI8BQD7YP7KXiKmsJi8AmBbjD/sxTqFxeQVADsjP2zHnLNi8gqAnTF/ZUfJycki0tTUFO+OAECXmB6xo1WrVsW7CwDQDfLDjpYsWRLvLgBAN+KfHwkJCfHuAgYTZlwBm+D8OQBAR/zHHwpfKtEthqqArTD+AADoID8AADrIDwCADvIDAKCD/AAA6CA/AAA6yA8AgA7yAwCgwy73D8YWDAabm5s//PDDsWPHpqSkxLs7sRiGMWvWrJaWlgMHDsSo5nQ6XS5XX230qaeeWrt2bVtbm4gEg8FOGw8EAn27USUhISE7O7ukpKSX7RiGMX78+KSkpMOHD/NXT4DBoSPeYndj9+7dbrc7os95eXnhcLiH7YfD4eLi4ra2tt53tSdNFRYWisju3buzs7NjvO3Z2dl6fWhraysuLo5Y6PF4PB5PR0dHcXGxiLjd7uhO9majMfRhs1VVVSJSWFgYY1t2+MQCUOK/N8Y4KJiHYI/HU1xcXFVV5fV61RK3293DCKmvrxeR+vr63ne126ZUBXU8raqqKv6UeiHZ2dnmkqqqKr0+qIOsdYkadqjDrsqPTo/p9s+Pjk//x7t6h8kPwFbivzd2dVDw+/0qJ/x+v3V5S0uLOsrk5eWZSyKOOOaSlpaWvLw8NSAwl5iVq6qqIo7jF9VUNK/X2+l3f3VYjx43hMNhFTN+v9+MQ7VFayPhcLi+vr6lpcXsg3pqvgoRUe+S2pB6fyLeN+uB3rq6ucR8UWZpfX19RN/q6+urqqqsya2aDYfDfr+/uLi403fGfJn19fXWddWG1KtT/xFtbW1ut1uNpaKRH4CtxH9v7OqgEBESVipaREQdjFTN6HU7LCMYcyvmN1xzKOP1es2j3kU1FaGlpUVEfD5fdFF0foTD4cLCQuvUnNfrVUdt9eq8Xm9EH/x+v7UPZhiov5eu3gq1ITUiiRiiWVeJHjRYX5Qa8EVsq62tzfzD7NZQV10130xV2Qyntra2vLy8iJdp9kpVNgdnaqEKyIjwi+4kgLiL/97Y1UFBHXQ6PY6Ew2G1ljruxzjod0RNiainbrdbnclQRyu32x29YrdNRVBnPjqdmIrOD7VEHUxVlliPoeqpqr97925rLEX30O12m2GjmjUfWM8lXFR+qAhRgxKPx6OWFBYWtrW1qb6ZuW4mSlVVVVtbm+qe2Vv19qrK4XDY+lSt63a71f+FdVAoXZwFIT8AW4n/3tjVQUEt7+okh/ourA7WGvlhPZSrJWq+qDf5EaM0Oj+8Xq/H47G+OvUVXnUjHA6rpy0tLeoIa9aM6KEa9Jgtm/nR0dGhjvtmfy42P8wVzTkxa2Vziimistqumccej8c6kDJ7pV6OWjcicdXgqdNzKuQHYCs2vVDSMAz1oJ8u5UxLSzMfZ2ZmlpaW/vnPf543b17vW+7J5cWGYZSXl3s8nh07dkQUqatvHQ5HSUlJUlJSUlKSiNTX13f1Puzbt08++3JMZWVlqampubm5qs7FinghmZmZ1qfmFKIybtw48/E111zj9/tDodCFCxf8fr/X642+uvf06dNm+3PmzLEWOZ1Ojd4CGHg2zQ+Hw+F2u5ubm2tra2fMmBFdoaCgQETGjh2r1370QaqhoaGX+REMBntY8/Tp0yLi9/uXLVvWVR2Xy5WdnV1aWur1emNk0q5du9xud6cVUlJS8vLyfvjDH5aUlOTk5PSwb3qs8TZlyhQRCQaDx44dE5Hy8vLy8vIermsqLS3t/T0lAPqVfe8/nz9/vojs3Lkzuqi2tlY96OrA+tZbb8VuPBQKRSzpNKV60pTpYm/Ni5jYUcxXVFtbW1pa6na7y8vL9+zZ02kLhmGUlpZmZWV1tYkHHnjA7XZv2LAh+vVa084c7WmztnD8+HH1YNq0adLFBQXdjtKsJ+QB2JN982PNmjUi8tJLLwUCgYiin/3sZyKiTsaazANiMBiMmF2JVlNTYz4+ePCgiEyaNEmvqQg9GYWo2Z7y8nLrYTcYDJpPQ6HQ9ddf73a733nnHY/Hc/PNN0c0q2rW1dWJyA033NDVhhwOx29+85vm5ubc3FzrchVL5uZUO72hRlSKSlyXy6UGeRFzdIFAIHZcqdI+v08eQJ+zb37MmzcvOzvb7/enpqYuXLiwuro6FArV1tbm5OQUFBS43e4HHnhA1Vy+fLmIPPPMM6FQKBAIRMzVqHMDu3btsi585JFHqqurDcMoKSlRramDnUZTJjVv09zc3O1LczgcKvzuuOOOQCAQCoVycnKSkpLGjx+vKuTm5jY3N7/66qtOp7OsrExEzJ6oraiDcmVlpYhkZGR0+zaWlpZaF65atUpEnn32WcMwqqur1avujcWLF6v3Mz8/X53zUL+Vkp2d3dzcnJ+fHwqFgsFgTk5OamrqTTfdFKMpFWbqZQKwtf4/Rd+NGN0wL2yNoG5HMKupm85Ukbo1wXqRUjgcNks7LPdSmJelWu//uKimIqiRSqc3rHR6/0fEjRFut1vdNhFxwW7HZy/nVZcnKebPlkRsKGLr5irW+wfNTavrdNW70en/SHTnrRXkIu//sN5f2ek7GeMyaJt8YgEoCR2f7pbxkpCQoA4KXVVQP56oznlMnDixq59QrK2tbWxsvP766x0OR3V1dUNDg/mdXa07fPjwlJSUnJyc0tJStbnq6moRiT5t3sOmovuQnJwsIo2NjRHnhAOBQE1NTVpaWsRahmEcPHiwoaFhxowZU6dOVWvt2bOntbU1KyvLbMQwDDXgUN0w+1BTUzNx4kRr/9WGos+W19bW1tbWWiurTX/wwQeLFi1Sb4X5MtWJa7OR6M5bK5SUlEycOHHOnDl1dXW1tbXRr9H6MiNKIzYkll+fjH4PpQcfFQADaRDkR9+y5kefq62t9Xg8hYWF99xzT3+0f8lTPyS8e/dulWoRyA/AVux7/mMwmjFjhs/nW7t2bfQ5f3QrEAisXbs2Ozu70/AAYDeMP/pYKBS6+uqr+TsWGmbOnNnS0vLOO+90dQsh4w/AVi67/AgGg6FQqF//CJXaxLhx48iPnjMM4/Tp07H/whX5AdjKZZcfGLz4qAC2wvkPAIAO8gMAoIP8AADoID8AADrIDwCADvIDAKCD/AAA6CA/AAA6yA8AgA7yAwCgwy4/0KR+mgIAMFgw/gAA6Ij/7ycCAAYjxh8AAB3kBwBAB/kBANBBfgAAdJAfAAAd5AcAQAf5AQDQQX4AAHSQHwAAHeQHAEAH+QEA0EF+AAB0kB8AAB3kBwBAB/kBANBBfgAAdJAfAAAd5AcAQAf5AQDQQX4AAHSQHwAAHeQHAEAH+QEA0EF+AAB0kB8AAB3kBwBAB/kBANBBfgAAdJAfAAAd5AcAQAf5AQDQQX4AAHSQHwAAHeQHAEAH+QEA0EF+AAB0kB8AAB3kBwBAB/kBANBBfgAAdJAfAAAd5AcAQMf/BwEcItROt3hIAAAAAElFTkSuQmCC"}}},{"cell_type":"markdown","source":"# 1- Install Sentence Piece from source","metadata":{}},{"cell_type":"code","source":"!sudo apt-get install cmake build-essential pkg-config libgoogle-perftools-dev -y","metadata":{"scrolled":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-08-23T11:46:27.278567Z","iopub.execute_input":"2023-08-23T11:46:27.279087Z","iopub.status.idle":"2023-08-23T11:46:37.972586Z","shell.execute_reply.started":"2023-08-23T11:46:27.279043Z","shell.execute_reply":"2023-08-23T11:46:37.971321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os","metadata":{"execution":{"iopub.status.busy":"2023-08-23T11:46:37.974301Z","iopub.execute_input":"2023-08-23T11:46:37.974702Z","iopub.status.idle":"2023-08-23T11:46:37.983282Z","shell.execute_reply.started":"2023-08-23T11:46:37.974669Z","shell.execute_reply":"2023-08-23T11:46:37.981993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! git clone https://github.com/google/sentencepiece.git \nos.chdir('sentencepiece')\n! mkdir build\nos.chdir('build')\n! cmake ..\n! make -j $(nproc)\n! sudo make install\n! sudo ldconfig -v","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2023-08-23T11:46:43.694647Z","iopub.execute_input":"2023-08-23T11:46:43.695067Z","iopub.status.idle":"2023-08-23T11:48:49.782090Z","shell.execute_reply.started":"2023-08-23T11:46:43.695017Z","shell.execute_reply":"2023-08-23T11:48:49.780599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.chdir('/kaggle/working')\n!ls","metadata":{"execution":{"iopub.status.busy":"2023-08-23T11:50:12.243960Z","iopub.execute_input":"2023-08-23T11:50:12.244510Z","iopub.status.idle":"2023-08-23T11:50:13.247687Z","shell.execute_reply.started":"2023-08-23T11:50:12.244467Z","shell.execute_reply":"2023-08-23T11:50:13.246234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2- Extract text type data from dataset and save it as txt file format","metadata":{}},{"cell_type":"code","source":"import pandas as pd","metadata":{"execution":{"iopub.status.busy":"2023-08-23T11:50:16.046775Z","iopub.execute_input":"2023-08-23T11:50:16.047315Z","iopub.status.idle":"2023-08-23T11:50:16.052939Z","shell.execute_reply.started":"2023-08-23T11:50:16.047270Z","shell.execute_reply":"2023-08-23T11:50:16.051676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.read_csv('/kaggle/input/bengaliai-speech/train.csv')['sentence'].tolist()","metadata":{"execution":{"iopub.status.busy":"2023-08-23T11:50:16.808601Z","iopub.execute_input":"2023-08-23T11:50:16.809015Z","iopub.status.idle":"2023-08-23T11:50:22.697440Z","shell.execute_reply.started":"2023-08-23T11:50:16.808982Z","shell.execute_reply":"2023-08-23T11:50:22.696294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('bn-speech.txt', 'w') as f:\n    text = '\\n'.join(data)\n    f.write(text)","metadata":{"execution":{"iopub.status.busy":"2023-08-23T11:50:22.699666Z","iopub.execute_input":"2023-08-23T11:50:22.700187Z","iopub.status.idle":"2023-08-23T11:50:23.323549Z","shell.execute_reply.started":"2023-08-23T11:50:22.700139Z","shell.execute_reply":"2023-08-23T11:50:23.322421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3- Train the tokenizer","metadata":{}},{"cell_type":"markdown","source":"## Usage: spm_train [options] files\n\n   **--help** (show help)  type: bool default: false\n   \n   **--version** (show version)  type: bool default: false\n   \n   **--minloglevel** (Messages logged at a lower level than this don't actually get logged anywhere)  type: int default: 0\n   \n   **--input** (comma separated list of input sentences)  type: std::string default: \"\"\n   \n   **--input_format** (Input format. Supported format is `text` or `tsv`.)  type: std::string default: \"\"\n   \n   **--model_prefix** (output model prefix)  type: std::string default: \"\"\n   \n   **--model_type** (model algorithm: unigram, bpe, word or char)  type: std::string default: \"unigram\"\n   \n   **--vocab_size** (vocabulary size)  type: int32 default: 8000\n   \n   **--accept_language** (comma-separated list of languages this model can accept)  type: std::string default: \"\"\n   \n   **--self_test_sample_size** (the size of self test samples)  type: int32 default: 0\n   \n   **--character_coverage** (character coverage to determine the minimum symbols)  type: double default: 0.9995\n   \n   **--input_sentence_size** (maximum size of sentences the trainer loads)  type: std::uint64_t default: 0\n   \n   **--shuffle_input_sentence** (Randomly sample input sentences in advance. Valid when --input_sentence_size > 0)  type: bool default: true\n   \n   **--seed_sentencepiece_size** (the size of seed sentencepieces)  type: int32 default: 1000000\n   \n   **--shrinking_factor** (Keeps top shrinking_factor pieces with respect to the loss)  type: double default: 0.75\n   \n   **--num_threads** (number of threads for training)  type: int32 default: 16\n   \n   **--num_sub_iterations** (number of EM sub-iterations)  type: int32 default: 2\n   \n   **--max_sentencepiece_length** (maximum length of sentence piece)  type: int32 default: 16\n   \n   **--max_sentence_length** (maximum length of sentence in byte)  type: int32 default: 4192\n   \n   **--split_by_unicode_script** (use Unicode script to split sentence pieces)  type: bool default: true\n   \n   **--split_by_number** (split tokens by numbers (0-9))  type: bool default: true\n   \n   **--split_by_whitespace** (use a white space to split sentence pieces)  type: bool default: true\n   \n   **--split_digits** (split all digits (0-9) into separate pieces)  type: bool default: false\n   \n   **--pretokenization_delimiter** (specifies the delimiter of pre-tokenization)  type: std::string default: \"\"\n   \n   **--treat_whitespace_as_suffix** (treat whitespace marker as suffix instead of prefix.)  type: bool default: false\n   \n   **--allow_whitespace_only_pieces** (allow pieces that only contain (consecutive) whitespace tokens)  type: bool default: false\n   \n   **--control_symbols** (comma separated list of control symbols)  type: std::string default: \"\"\n   \n   **--control_symbols_file** (load control_symbols from file.)  type: std::string default: \"\"\n   \n   **--user_defined_symbols** (comma separated list of user defined symbols)  type: std::string default: \"\"\n   \n   **--user_defined_symbols_file** (load user_defined_symbols from file.)  type: std::string default: \"\"\n   \n   **--required_chars** (UTF8 characters in this flag are always used in the character set regardless of --character_coverage)  type: std::string default: \"\"\n   \n   **--required_chars_file** (load required_chars from file.)  type: std::string default: \"\"\n   \n   **--byte_fallback** (decompose unknown pieces into UTF-8 byte pieces)  type: bool default: false\n   \n   **--vocabulary_output_piece_score** (Define score in vocab file)  type: bool default: true\n   \n   **--normalization_rule_name** (Normalization rule name. Choose from nfkc or identity)  type: std::string default: \"nmt_nfkc\"\n   \n   **--normalization_rule_tsv** (Normalization rule TSV file. )  type: std::string default: \"\"\n   \n   **--denormalization_rule_tsv** (Denormalization rule TSV file.)  type: std::string default: \"\"\n   \n   **--add_dummy_prefix** (Add dummy whitespace at the beginning of text)  type: bool default: true\n   \n   **--remove_extra_whitespaces** (Removes leading, trailing, and duplicate internal whitespace)  type: bool default: true\n   \n   **--hard_vocab_limit** (If set to false, --vocab_size is considered as a soft limit.)  type: bool default: true\n   \n   **--use_all_vocab** (If set to true, use all tokens as vocab. Valid for word/char models.)  type: bool default: false\n   \n   **--unk_id** (Override UNK (\\<unk>) id.)  type: int32 default: 0\n    \n   **--bos_id** (Override BOS (\\<s>) id. Set -1 to disable BOS.)  type: int32 default: 1\n    \n   **--eos_id** (Override EOS (\\</s>) id. Set -1 to disable EOS.)  type: int32 default: 2\n    \n   **--pad_id** (Override PAD (\\<pad>) id. Set -1 to disable PAD.)  type: int32 default: -1\n    \n   **--unk_piece** (Override UNK (\\<unk>) piece.)  type: std::string default: \"\\<unk>\"\n    \n   **--bos_piece** (Override BOS (\\<s>) piece.)  type: std::string default: \"\\<s>\"\n    \n   **--eos_piece** (Override EOS (\\</s>) piece.)  type: std::string default: \"\\</s>\"\n    \n   **--pad_piece** (Override PAD (\\<pad>) piece.)  type: std::string default: \"\\<pad>\"\n    \n   **--unk_surface** (Dummy surface string for \\<unk>. In decoding \\<unk> is decoded to `unk_surface`.)  type: std::string default: \" ⁇ \"\n    \n   **--train_extremely_large_corpus** (Increase bit depth for unigram tokenization.)  type: bool default: false\n    \n   **--random_seed** (Seed value for random generator.)  type: uint32 default: 4294967295\n    \n   **--enable_differential_privacy** (Whether to add DP while training. Currently supported only by UNIGRAM model.)  type: bool default: false\n    \n   **--differential_privacy_noise_level** (Amount of noise to add for DP)  type: float default: 0\n    \n   **--differential_privacy_clipping_threshold** (Threshold for clipping the counts for DP)  type: std::uint64_t default: 0\n","metadata":{"execution":{"iopub.status.busy":"2023-07-24T13:05:49.356274Z","iopub.execute_input":"2023-07-24T13:05:49.356756Z","iopub.status.idle":"2023-07-24T13:05:50.473182Z","shell.execute_reply.started":"2023-07-24T13:05:49.356720Z","shell.execute_reply":"2023-07-24T13:05:50.471789Z"}}},{"cell_type":"code","source":"!spm_train --input=bn-speech.txt --model_prefix=spm --vocab_size=8000 --character_coverage=0.9999 num_threads=4 --pad_id=0 --unk_id=3 --bos_id=1 --eos_id=2","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2023-08-23T11:50:28.283290Z","iopub.execute_input":"2023-08-23T11:50:28.284220Z","iopub.status.idle":"2023-08-23T11:51:44.622974Z","shell.execute_reply.started":"2023-08-23T11:50:28.284173Z","shell.execute_reply":"2023-08-23T11:51:44.621718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-08-23T11:53:01.507697Z","iopub.execute_input":"2023-08-23T11:53:01.508181Z","iopub.status.idle":"2023-08-23T11:53:02.537481Z","shell.execute_reply.started":"2023-08-23T11:53:01.508134Z","shell.execute_reply":"2023-08-23T11:53:02.535910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!spm_export_vocab --model=spm.model --output=spm.spm","metadata":{"execution":{"iopub.status.busy":"2023-08-23T11:53:27.299409Z","iopub.execute_input":"2023-08-23T11:53:27.299887Z","iopub.status.idle":"2023-08-23T11:53:28.356480Z","shell.execute_reply.started":"2023-08-23T11:53:27.299844Z","shell.execute_reply":"2023-08-23T11:53:28.354864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm bn-speech.txt\n!rm -r sentencepiece/","metadata":{"execution":{"iopub.status.busy":"2023-08-23T11:54:11.354974Z","iopub.execute_input":"2023-08-23T11:54:11.355511Z","iopub.status.idle":"2023-08-23T11:54:12.405872Z","shell.execute_reply.started":"2023-08-23T11:54:11.355467Z","shell.execute_reply":"2023-08-23T11:54:12.404230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls","metadata":{"execution":{"iopub.status.busy":"2023-08-23T11:54:15.029988Z","iopub.execute_input":"2023-08-23T11:54:15.030536Z","iopub.status.idle":"2023-08-23T11:54:16.050460Z","shell.execute_reply.started":"2023-08-23T11:54:15.030490Z","shell.execute_reply":"2023-08-23T11:54:16.049016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}