{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!pip install torch > /dev/null\n!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py > /dev/null\n!python pytorch-xla-env-setup.py --version 20200515 --apt-packages libomp5 libopenblas-dev > /dev/null\n!pip install transformers > /dev/null\n!pip install pandarallel > /dev/null\n!pip install emoji\n!pip install num2words\n!pip install --upgrade language_tool_python\n!pip install --upgrade turkishnlp\n!pip install --upgrade git+https://github.com/pytorch/contrib.git > /dev/null","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nos.environ['XLA_USE_BF16'] = \"1\"\n\nfrom pathlib import Path\nimport pandas as pd\nimport gc\nimport torch\nimport json\nfrom pandarallel import pandarallel\npandarallel.initialize(nb_workers=8, progress_bar=False)\n\nfrom transformers import AutoModel, AutoTokenizer, AutoConfig\nimport torch_xla.core.xla_model as xm\nimport torch_xla.distributed.xla_multiprocessing as xmp\n\nimport data_cleaning as clean\nimport models\nimport config\nimport utility","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MODEL = 'xlm-roberta-large'\n\nroot_dir = Path(\"../input\")\n\n# *Data access\ntest_file_dir = Path(root_dir, \"jigsaw-multilingual-toxic-comment-classification\")\n\n# *Model paths\nmodel_dir = Path(root_dir, MODEL)\ncheckpoint_dir = (model_dir/'best_model.bin')\nconfig_path = (model_dir/'config.json')\nout_dir = Path('../output')\n\n# *Files\ntest_file = \"test.csv\"\n# yandex translated test file\ntest_eng_file1 = \"jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_test_translated.csv\"\n# google translated test file\ntest_eng_file2 = \"test-en-df/test_en.csv\"\n\nsub_file = \"jigsaw-toxic/sample_submission.csv\"\nopen_subtitles_file = 'open-subtitles-toxic/open-subtitles-synthesic.csv'\n\nLANGS = {\n    'en': 'english',\n    'it': 'italian', \n    'fr': 'french', \n    'es': 'spanish',\n    'tr': 'turkish', \n    'ru': 'russian',\n    'pt': 'portuguese'\n}\n\n# read config file\nwith open(config_path) as f:\n    model_config = json.load(f)\n\nMODEL_VERSION = model_config['model_version']\nMODEL_PREFIX = model_config['model_prefix']\nMAX_LENGTH = model_config['max_len']\n\nenglish_pipeline = model_config['english pipeline']\noutput_hidden_states = model_config['output_hidden_states']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Read the data and external sources\ndir_paths = {'base_dir': test_file_dir, 'base_t_dir': root_dir, 'test_file':test_file, \n             'test_file_translated1': test_eng_file1, 'test_file_translated2': test_eng_file2}\n_, _, test, _ = utility.read_data(dir_paths, list(LANGS.keys()), english_pipeline=english_pipeline)\n\nif english_pipeline:\n    test_yandex, test_google = test[0], test[1]\n    test = test_yandex.append(test_google) \n#     df_test_yandex = clean.clean_data(test_yandex, input_cols_test)\n#     df_test_google = clean.clean_data(test_google, input_cols_test)\n\ninput_cols_test = ['content']\ndf_test = clean.clean_data(test, input_cols_test, True)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer = AutoTokenizer.from_pretrained(str(model_dir))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = models.DatasetRetriever(\n    tokenizer,\n    labels_or_ids=test.index.values, \n    comment_texts=test[input_cols_test].values, \n    langs=test['lang'].values,\n    maxlen=MAX_LENGTH,\n    use_train_transforms=False,\n    test=True\n)\n\ndel test\ngc.collect();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transformer = AutoModel.from_config(AutoConfig.from_pretrained(str(model_dir)))\nnet = models.ToxicSimpleNNModel(transformer, config.TrainGlobalConfig)\ncheckpoint = torch.load(checkpoint_dir, map_location=torch.device('cpu'))\nnet.load_state_dict(checkpoint);\nconfig.TrainGlobalConfig.train_lenght = len(test_dataset)\n\ncheckpoint = None\ndel checkpoint","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\n\ndef _mp_fn(rank, flags):\n    device = xm.xla_device()\n    net.to(device)\n    \n    test_sampler = torch.utils.data.distributed.DistributedSampler(\n        test_dataset,\n        num_replicas=xm.xrt_world_size(),\n        rank=xm.get_ordinal(),\n        shuffle=False\n    )\n    test_loader = torch.utils.data.DataLoader(\n        test_dataset,\n        batch_size=16,\n        sampler=test_sampler,\n        pin_memory=False,\n        drop_last=False,\n        num_workers=1\n    )\n    \n    fitter = models.TPUFitter(model=net, device=device, config=config.TrainGlobalConfig, \n                              base_model_path=model_dir, model_name=MODEL, model_prefix=MODEL_PREFIX,\n                              model_version=MODEL_VERSION, \n                              out_path=out_dir)\n    \n    fitter.run_inference(test_loader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nFLAGS={}\nxmp.spawn(_mp_fn, args=(FLAGS,), nprocs=8, start_method='fork')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.concat([pd.read_csv(path) for path in (out_dir/'node_submissions').glob('*.csv')]).groupby('id').mean()\nsubmission['toxic'].hist(bins=100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.iloc[32]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}