{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"# this is a clone of https://www.kaggle.com/miklgr500/how-to-use-translators-for-comments-translation but for google colab","execution_count":null,"outputs":[]},{"metadata":{"id":"bYROVtrT-xji","trusted":false},"cell_type":"code","source":"from google.colab import files\n\n# Install Kaggle library\n!pip install -q kaggle","execution_count":0,"outputs":[]},{"metadata":{"id":"YEpMKJIt-1bv","outputId":"aa8872b7-a2ba-4498-8968-452f8bd0777c","trusted":false},"cell_type":"code","source":"import multiprocessing\nprint(multiprocessing.cpu_count())","execution_count":null,"outputs":[]},{"metadata":{"id":"D2RDfxhS_Zl-","outputId":"cd7593c3-bf8e-43fa-9e99-f7247eabf75e","trusted":false},"cell_type":"code","source":"# Upload kaggle API key file\nuploaded = files.upload()","execution_count":null,"outputs":[]},{"metadata":{"id":"-SdjY2-N_dfS","outputId":"09786c5a-e674-4430-bde7-832d6e0ad57d","trusted":false},"cell_type":"code","source":"!ls","execution_count":null,"outputs":[]},{"metadata":{"id":"ZEBAy227ANYo","outputId":"1674468e-958e-4b79-b0eb-9019a1011f29","trusted":false},"cell_type":"code","source":"!mkdir ~/.kaggle\n!cp /content/kaggle.json ~/.kaggle/kaggle.json","execution_count":null,"outputs":[]},{"metadata":{"id":"CxjeQsbOAUum","outputId":"1556a2dd-9aae-4715-8a66-4892708eeb08","trusted":false},"cell_type":"code","source":"import kaggle","execution_count":null,"outputs":[]},{"metadata":{"id":"kApJbGjvAk3l","outputId":"77ada16c-f022-4c5e-f170-65ff09f2ae17","trusted":false},"cell_type":"code","source":"from kaggle.api.kaggle_api_extended import KaggleApi\napi = KaggleApi()\napi.authenticate()\n","execution_count":null,"outputs":[]},{"metadata":{"id":"Erh2Vr8eAqQe","outputId":"82d63353-4ffb-4dbb-a149-379ee1ae3e2e","trusted":false},"cell_type":"code","source":"api.competition_download_file('jigsaw-multilingual-toxic-comment-classification','jigsaw-unintended-bias-train.csv')","execution_count":null,"outputs":[]},{"metadata":{"id":"C_VQMngLBk_d","outputId":"5ed31d6f-3c61-4488-8b61-aef7f0f7d807","trusted":false},"cell_type":"code","source":"!ls","execution_count":null,"outputs":[]},{"metadata":{"id":"53zGt8OlB86Q","trusted":false},"cell_type":"code","source":"","execution_count":0,"outputs":[]},{"metadata":{"id":"629rSJrMCAzm","outputId":"252b8df3-ddb1-4fe4-d380-40a97d2375e8","trusted":false},"cell_type":"code","source":"!unzip jigsaw-unintended-bias-train.csv.zip","execution_count":null,"outputs":[]},{"metadata":{"id":"-LIp6nXfCOdG","outputId":"e7683821-21de-41d8-b371-3a0e43179271","trusted":false},"cell_type":"code","source":"! ls","execution_count":null,"outputs":[]},{"metadata":{"id":"gKeXCXCzCSVO","outputId":"95171bd4-66ed-4e6a-ae6b-0bf348a5f636","trusted":false},"cell_type":"code","source":"!pip install translators\n\nimport pandas as pd\n# current version have logs, which is not very comfortable\nimport translators as ts\nfrom multiprocessing import Pool\nfrom tqdm import *","execution_count":null,"outputs":[]},{"metadata":{"id":"S_G7KYpVCZ6K","trusted":false},"cell_type":"code","source":"LANG = 'ru'\nAPI = 'google'\n\n\ndef translator_constructor(api):\n    if api == 'google':\n        return ts.google\n    elif api == 'bing':\n        return ts.bing\n    elif api == 'baidu':\n        return ts.baidu\n    elif api == 'sogou':\n        return ts.sogou\n    elif api == 'youdao':\n        return ts.youdao\n    elif api == 'tencent':\n        return ts.tencent\n    elif api == 'alibaba':\n        return ts.alibaba\n    else:\n        raise NotImplementedError(f'{api} translator is not realised!')\n","execution_count":0,"outputs":[]},{"metadata":{"id":"WD5HfscpCkYD","outputId":"d5b51728-1956-4815-9e89-ae2f489d7472","trusted":false},"cell_type":"code","source":"CSV_PATH = 'jigsaw-unintended-bias-train.csv'\n\ndef translate(x):\n    try:\n        return [x[0], translator_constructor(API)(x[1], 'en', LANG), x[2]]\n    except:\n        return [x[0], None, [2]]\n\n\ndef imap_unordered_bar(func, args, n_processes: int = 48):\n    p = Pool(n_processes, maxtasksperchild=100)\n    res_list = []\n    with tqdm(total=len(args)) as pbar:\n        for i, res in tqdm(enumerate(p.imap_unordered(func, args))):\n            pbar.update()\n            res_list.append(res)\n    pbar.close()\n    p.close()\n    p.join()\n    return res_list\n\n\ndef main():\n    df = pd.read_csv(CSV_PATH).query('toxic==1') # .sample(100)\n    df.toxic = df.toxic.round().astype(int)\n    tqdm.pandas('Translation progress')\n    df[['id', 'comment_text', 'toxic']] = imap_unordered_bar(translate, df[['id', 'comment_text', 'toxic']].values)\n    df.to_csv(f'jigsaw-toxic-comment-train-{API}-{LANG}.csv')\n\n\nif __name__ == '__main__':\n    import multiprocessing\n    print(multiprocessing.cpu_count())\n    main()","execution_count":null,"outputs":[]},{"metadata":{"id":"GHlTIMCnDKsu","trusted":false},"cell_type":"code","source":"","execution_count":0,"outputs":[]}],"metadata":{"colab":{"name":"jigsaw_data.ipynb","provenance":[]},"kernelspec":{"name":"python3","display_name":"Python 3"}},"nbformat":4,"nbformat_minor":4}