{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"## Super Fast Inference [~ 2 min] using multi core TPU on PyTorch/XLA","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Main Idea:\n\nEveryone of us knows about how ensemble can improve score. But this competition has some limits on using TPU/GPU: 3 hours. I have made experiment on GPU and understood that one model inference hold 25 min for XLM-Roberta. So for KFold with 5 folds have to hold about ~2.5h. \n\nAfter reading this kernel you are able to do ensemble with ~70-80 checkpoints XLM-Roberta theoretically if you want.\n\n***So, lets use MULTI CORE TPU for Inference with data exchange in hard disk during multiprocessing!***"},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py > /dev/null\n!python pytorch-xla-env-setup.py --version 1.6 --apt-packages libomp5 libopenblas-dev > /dev/null\n!pip install transformers > /dev/null\n!pip install pandarallel > /dev/null","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport os\nos.environ['XLA_USE_BF16'] = \"1\"\n\nfrom glob import glob\nfor path in glob(f'../input/*'):\n    print(path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler\n\nimport time\nimport random\nfrom datetime import datetime\nfrom tqdm import tqdm\ntqdm.pandas()\n\nfrom transformers import XLMRobertaModel, XLMRobertaTokenizer, XLMRobertaConfig\nfrom transformers import AdamW, get_linear_schedule_with_warmup, get_constant_schedule\n\nimport re\n\n# !pip install nltk > /dev/null\nimport nltk\nnltk.download('punkt')\n\nfrom nltk import sent_tokenize\n\nfrom pandarallel import pandarallel\n\npandarallel.initialize(nb_workers=2, progress_bar=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SEED = 42\n\nMAX_LENGTH = 224\nBACKBONE_PATH = '../input/multitpu-inference'\nCHECKPOINT_PATH = '../input/multitpu-inference/checkpoint-xlm-roberta.bin'\n\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"LANGS = {\n    'en': 'english',\n    'it': 'italian', \n    'fr': 'french', \n    'es': 'spanish',\n    'tr': 'turkish', \n    'ru': 'russian',\n    'pt': 'portuguese'\n}\n\ndef get_sentences(text, lang='en'):\n    return sent_tokenize(text, LANGS.get(lang, 'english'))\n\ndef exclude_duplicate_sentences(text, lang='en'):\n    sentences = []\n    for sentence in get_sentences(text, lang):\n        sentence = sentence.strip()\n        if sentence not in sentences:\n            sentences.append(sentence)\n    return ' '.join(sentences)\n\ndef clean_text(text, lang='en'):\n    text = str(text)\n    text = re.sub(r'[0-9\"]', '', text)\n    text = re.sub(r'#[\\S]+\\b', '', text)\n    text = re.sub(r'@[\\S]+\\b', '', text)\n    text = re.sub(r'https?\\S+', '', text)\n    text = re.sub(r'\\s+', ' ', text)\n    text = exclude_duplicate_sentences(text, lang)\n    return text.strip()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DatasetRetriever(Dataset):\n\n    def __init__(self, df):\n        self.comment_texts = df['comment_text'].values\n        self.ids = df['id'].values\n        self.tokenizer = XLMRobertaTokenizer.from_pretrained(BACKBONE_PATH)\n\n    def get_tokens(self, text):\n        encoded = self.tokenizer.encode_plus(\n            text, \n            add_special_tokens=True, \n            max_length=MAX_LENGTH, \n            pad_to_max_length=True\n        )\n        return encoded['input_ids'], encoded['attention_mask']\n\n    def __len__(self):\n        return self.ids.shape[0]\n\n    def __getitem__(self, idx):\n        text = self.comment_texts[idx]\n        \n        #######################################\n        # TODO TTA transforms: about it later #\n        #######################################\n    \n        tokens, attention_mask = self.get_tokens(text)\n        tokens, attention_mask = torch.tensor(tokens), torch.tensor(attention_mask)\n\n        return self.ids[idx], tokens, attention_mask","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ndf_test = pd.read_csv(f'../input/mifi-data/mifi.csv')\n#df_test.reset_index()\n#df_test= df_test.reset_index() \n#df_test['comment_text'] = df_test.parallel_apply(lambda x: clean_text(x['content'], x['lang']), axis=1)\n#df_test = df_test.drop(columns=['content'])\ndf_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = DatasetRetriever(df_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ToxicSimpleNNModel(nn.Module):\n\n    def __init__(self, backbone):\n        super(ToxicSimpleNNModel, self).__init__()\n        self.backbone = backbone\n        self.dropout = nn.Dropout(0.3)\n        self.linear = nn.Linear(\n            in_features=self.backbone.pooler.dense.out_features*2,\n            out_features=2,\n        )\n\n    def forward(self, input_ids, attention_masks):\n        bs, seq_length = input_ids.shape\n        seq_x = self.backbone(input_ids=input_ids, attention_mask=attention_masks, output_hidden_states=True).last_hidden_state\n        apool = torch.mean(seq_x, 1)\n        mpool, _ = torch.max(seq_x, 1)\n        x = torch.cat((apool, mpool), 1)\n        x = self.dropout(x)\n        return self.linear(x)\n\n\nbackbone = XLMRobertaModel(XLMRobertaConfig.from_pretrained(BACKBONE_PATH))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings\n\nwarnings.filterwarnings(\"ignore\")\n\nimport torch_xla\nimport torch_xla.core.xla_model as xm\nimport torch_xla.distributed.parallel_loader as pl\nimport torch_xla.distributed.xla_multiprocessing as xmp\n\n\nclass MultiCoreTPUPredictor:\n    \n    def __init__(self, model, device):\n        if not os.path.exists('node_submissions'):\n            os.makedirs('node_submissions')\n\n        self.model = model\n        self.device = device\n\n        xm.master_print(f'Model prepared. Device is {self.device}')\n\n\n    def run_inference(self, test_loader, verbose=True, verbose_step=50):\n        self.model.eval()\n        result = {'id': [], 'toxic': []}\n        t = time.time()\n        for step, (ids, inputs, attention_masks) in enumerate(test_loader):\n            if verbose:\n                if step % 50 == 0:\n                    xm.master_print(f'Prediction Step {step}, time: {(time.time() - t):.5f}')\n\n            with torch.no_grad():\n                inputs = inputs.to(self.device, dtype=torch.long) \n                attention_masks = attention_masks.to(self.device, dtype=torch.long)\n                outputs = self.model(inputs, attention_masks)\n                toxics = nn.functional.softmax(outputs, dim=1).data.cpu().numpy()[:,1]\n\n            result['id'].extend(ids.numpy())\n            result['toxic'].extend(toxics)\n\n        result = pd.DataFrame(result)\n        node_count = len(glob('node_submissions/*.csv'))\n        result.to_csv(f'node_submissions/submission_{node_count}_{datetime.utcnow().microsecond}.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"net = ToxicSimpleNNModel(backbone=backbone)\ncheckpoint = torch.load(CHECKPOINT_PATH, map_location=torch.device('cpu'))\nnet.load_state_dict(checkpoint, strict=False);\n\ncheckpoint = None\ndel checkpoint","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _mp_fn(rank, flags):\n    device = xm.xla_device()\n    model = net.to(device)\n\n    test_sampler = torch.utils.data.distributed.DistributedSampler(\n        test_dataset,\n        num_replicas=xm.xrt_world_size(),\n        rank=xm.get_ordinal(),\n        shuffle=False\n    )\n    test_loader = torch.utils.data.DataLoader(\n        test_dataset,\n        batch_size=16,\n        sampler=test_sampler,\n        pin_memory=False,\n        drop_last=False,\n        num_workers=1\n    )\n\n    fitter = MultiCoreTPUPredictor(model=model, device=device)\n    fitter.run_inference(test_loader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nFLAGS={}\nxmp.spawn(_mp_fn, args=(FLAGS,), nprocs=1, start_method='fork')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.concat([pd.read_csv(path) for path in glob('node_submissions/*.csv')]).groupby('id').mean()\nsubmission['toxic'].hist(bins=100);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> On this stage my submission has 0.9417 with single checkpoint. But let me say thanks author @hamditarek for perfect ensemble https://www.kaggle.com/hamditarek/ensemble\n\n> Let's blend with 1:1"},{"metadata":{"trusted":true},"cell_type":"code","source":"ensemble = pd.read_csv('../input/multitpu-inference/submission-ensemble.csv', index_col='id')\nensemble['toxic'].hist(bins=100);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def scale_min_max_submission(submission):\n    min_, max_ = submission['toxic'].min(), submission['toxic'].max()\n    submission['toxic'] = (submission['toxic'] - min_) / (max_ - min_)\n    return submission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission['toxic'] = (scale_min_max_submission(submission)['toxic'] + scale_min_max_submission(ensemble)['toxic']) / 2\nsubmission['toxic'].hist(bins=100);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tmp_с = df_test.loc[submission[submission['toxic'] > 0.03].index]\ntmp_с[tmp_с['lang'] == 'ru']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tmp_с[tmp_с['lang'] == 'ru'].to_csv('toxic_ru_003.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tmp = df_test.loc[submission[submission['toxic'] > 0.52].index]\ntmp[tmp['lang'] == 'ru']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":false},"cell_type":"code","source":"tmp[tmp['lang'] == 'ru'].to_csv('toxic_ru_052.csv')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}