{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"VERSION = \"20200220\" \n!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py!python pytorch-xla-env-setup.py --version $VERSION","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"language_dict = {0: 'base', 1: 'spanish', 2: 'english', 3: 'german', 4: 'dutch'}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Preprocessing","execution_count":null},{"metadata":{"_uuid":"cba509cb-708d-4fd0-8e2a-e5c1c7a0982d","_cell_guid":"593619be-8090-4dad-a462-e883e560ec1c","trusted":true},"cell_type":"code","source":"import os, time\nimport pandas as pd\nfrom kaggle_datasets import KaggleDatasets\nimport torch\nfrom sklearn import metrics\nfrom transformers.data.processors import InputFeatures\n\nfrom tqdm import tqdm\nfrom transformers import XLMRobertaTokenizer, XLMRobertaForSequenceClassification, AdamW\nfrom torch.utils.data import TensorDataset, DataLoader, WeightedRandomSampler\n\nBASE_FILE_PATH = \"/kaggle/input/cleaned-data\"\nMODEL_NAME = 'xlm-roberta-base'\nTRAIN_EPOCH = 20\n\ntorch.manual_seed(12)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bc97f110-17eb-44dd-a792-d66c27a0b3a6","_cell_guid":"f5ccaf08-c532-4fde-9306-b897c890d0f8","trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\n#try:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n#    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n#    print('Running on TPU ', tpu.master())\n#except ValueError:\n#    tpu = None\n#tpu = None\n#if tpu:\n#    tf.config.experimental_connect_to_cluster(tpu)\n#    tf.tpu.experimental.initialize_tpu_system(tpu)\n#    strategy = tf.distribute.experimental.TPUStrategy(tpu)\n#else:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n#    strategy = tf.distribute.get_strategy()\n\n#print(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Tokenizing","execution_count":null},{"metadata":{"_uuid":"1392a5e0-c8e4-46ea-b45d-0d9289682e09","_cell_guid":"fcd2093b-0774-4adf-9e4c-e9096f156d32","trusted":true},"cell_type":"code","source":"whole_train = pd.read_csv(f'{BASE_FILE_PATH}/cleaned_train.csv')\n\n#positive_200 = whole_train[whole_train[\"toxic\"]==1][0:199]\n#negative_200 = whole_train[whole_train[\"toxic\"]==0][0:199]\n\npart_train = pd.read_csv(f'{BASE_FILE_PATH}/cleaned_train.csv')\npart_valid = pd.read_csv(f'{BASE_FILE_PATH}/cleaned_validation.csv')\npart_test = pd.read_csv(f'{BASE_FILE_PATH}/cleaned_test.csv')\n\n\nSEQUENCE_LENGTH = 32\n\ndef multilingual_model(max_seq_length=SEQUENCE_LENGTH, trainable=False):\n    \"\"\"Build and return a multilingual BERT model and tokenizer.\"\"\"\n    model = XLMRobertaForSequenceClassification.from_pretrained(\n        MODEL_NAME,\n        num_labels = 2, \n        output_attentions = False, # Whether the model returns attentions weights.\n        output_hidden_states = False, # Whether the model returns all hidden-states.\n    )\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#def convert_df_to_tensors(tokenizer, df):\n#    sentences = df['comment_text'].values\n#    labels = df['toxic'].values\n#    batch_encoding = tokenizer.batch_encode_plus(sentences, max_length=SEQUENCE_LENGTH, pad_to_max_length=True)\n#    return torch.tensor(batch_encoding['input_ids'], dtype=torch.long), \\\n#           torch.tensor(batch_encoding['attention_mask'], dtype=torch.long), \\\n#           torch.tensor(labels)\n\ndef convert_df_to_tensors(tokenizer, df):\n    sentences = df['comment_text'].values\n    labels = df['toxic'].values\n\n    features = []\n    print(\"Starting...\")\n    for i in tqdm(range(len(sentences))):\n        encoding = tokenizer.encode_plus(sentences[i], max_length=SEQUENCE_LENGTH, pad_to_max_length=True)\n        inputs = {k: encoding[k] for k in encoding}\n        feature = InputFeatures(**inputs, label=labels[i])\n        features.append(feature)\n\n    return features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Loading XLMRoberta tokenizer...')\ntokenizer = XLMRobertaTokenizer.from_pretrained(MODEL_NAME, do_lower_case=True)\n#part_train_dataset = convert_df_to_tensors(tokenizer, part_train)\n#part_valid_dataset = convert_df_to_tensors(tokenizer, part_valid)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#torch.save(part_train_dataset, 'train.pt')\n#torch.save(part_valid_dataset, 'valid.pt')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b24c47ad-a156-41f2-97b8-f5618181382c","_cell_guid":"48479a32-25c1-40c9-bd1c-d076eb39d86e","trusted":true},"cell_type":"markdown","source":"Load the preprocessed dataset. See the demo notebook for sample code for performing this preprocessing.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model = multilingual_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from transformers import EvalPrediction\nfrom typing import Dict\nimport numpy as np\n\ndef compute_metrics(p: EvalPrediction) -> Dict:\n    preds = np.argmax(p.predictions, axis=1)\n    return {\"auc\": metrics.roc_auc_score(preds, p.label)} #p.label_ids)}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfrom torch.utils.data.dataset import Dataset\n#> xsv select toxic cleaned_train.csv | xsv frequency\n#field,value,count\n#toxic,0,1952248\n#toxic,1,173495\n#1952248 / 173495\n\nweighted_dict = {0:1, 1:11.2524741347}\n\nclass ToxicDataset(Dataset):\n    def __init__(self, input_path):\n        features = torch.load(input_path)\n        self.input_id = torch.tensor([x.input_ids for x in tqdm(features)], dtype=torch.long)\n        self.attention_mask = torch.tensor([x.attention_mask for x in tqdm(features)], dtype=torch.long)\n        self.labels = torch.tensor([x.label for x in tqdm(features)], dtype=torch.long)\n        \n    def __len__(self):\n        return len(self.features)\n    \n    def get_tensordataset(self):\n        return TensorDataset(self.input_id, self.attention_mask, self.labels)\n\n    def __getitem__(self, i):\n        return self.features[i]\n    \n    def get_weights(self):\n        return [weighted_dict[x.label] for x in self.features]\n\n    def get_labels(self):\n        return [x.label for x in self.features]\n    \n    def read_csv(self, csv_file_path):\n        part_train = pd.read_csv(\"\")\n        #part_valid = torch.load(f'{BASE_FILE_PATH}/valid.pt')\n\n        #part_train_data = TensorDataset(*convert_df_to_tensors(tokenizer, part_train))\n        #part_valid_data = TensorDataset(*convert_df_to_tensors(tokenizer, part_valid))\n\n        part_train_data = part_train_dataset\n        part_valid_data = part_valid_dataset\n\n        weighted_list = part_train['toxic'].map(weighted_dict)\n\n        #train_dataloader = DataLoader(part_train_data, sampler=WeightedRandomSampler(weighted_list, part_train_data), batch_size=BATCH_SIZE)\n        #valid_dataloader = DataLoader(part_valid_data)\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"part_train_dataset = ToxicDataset(f'{BASE_FILE_PATH}/train.pt')\npart_valid_dataset = ToxicDataset(f'{BASE_FILE_PATH}/valid.pt').get_tensordataset();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SEQUENCE_LENGTH = 32\nBATCH_SIZE = 16\nMODEL_NAME = 'xlm-roberta-large'\ndevice = torch.device('cpu')\nLEARNING_RATE = 2e-5\n\ndef multilingual_model(max_seq_length=SEQUENCE_LENGTH):\n    \"\"\"Build and return a multilingual BERT model and tokenizer.\"\"\"\n    model = XLMRobertaForSequenceClassification.from_pretrained(\n        MODEL_NAME,\n        num_labels = 2, # The number of output labels--2 for binary classification.\n        output_attentions = False, # Whether the model returns attentions weights.\n        output_hidden_states = False, # Whether the model returns all hidden-states.\n    )\n    return model\n\ndef fit(model, num_epocs=TRAIN_EPOCH):\n    global_step = 0\n    model.train()\n    for i_ in tqdm(range(num_epocs), desc=\"Epoch\"):\n        tr_loss = 0\n        nb_tr_examples, nb_tr_steps = 0, 0\n        for step, batch in enumerate(tqdm(train_dataloader, desc=\"Iteration\")):\n            batch = tuple(t.to(device) for t in batch)\n            input_ids, attention_mask, label_ids = batch\n            loss, logit = model(input_ids, labels=label_ids)\n#            if n_gpu > 1:\n#                loss = loss.mean() # mean() to average on multi-gpu.\n            if args['gradient_accumulation_steps'] > 1:\n                loss = loss / args['gradient_accumulation_steps']\n\n            if args['fp16']:\n                optimizer.backward(loss)\n            else:\n                loss.backward()\n\n            tr_loss += loss.item()\n            nb_tr_examples += input_ids.size(0)\n            nb_tr_steps += 1\n            if (step + 1) % args['gradient_accumulation_steps'] == 0:\n                # modify learning rate with special warm up BERT uses\n                lr_this_step = LEARNING_RATE\n                for param_group in optimizer.param_groups:\n                    param_group['lr'] = lr_this_step\n                optimizer.step()\n                optimizer.zero_grad()\n                global_step += 1\n\n        print('Loss after epoc {}'.format(tr_loss / nb_tr_steps))\n\nmodel = multilingual_model(SEQUENCE_LENGTH)\nprint('Loading XLMRoberta tokenizer...')\n# tokenizer = XLMRobertaTokenizer.from_pretrained(MODEL_NAME, do_lower_case=True)\n\nargs={}\nargs['fp16'] = False\nargs['gradient_accumulation_steps'] = 1\nargs['warmup_proportion'] = 0.01\nargs['learning_rate'] = 2e-4\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n#\ntrain_dataloader = DataLoader(part_train_dataset.get_tensordataset(), sampler=WeightedRandomSampler(part_train_dataset.get_weights(), num_samples=BATCH_SIZE))\nvalid_dataloader = DataLoader(part_valid_dataset.get_tensordataset())\n\nparam_optimizer = list(model.named_parameters())\nno_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\noptimizer_grouped_parameters = [\n    {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},\n    {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n    ]\n\noptimizer = AdamW(optimizer_grouped_parameters,LEARNING_RATE)\nfit(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls output/kaggle/working","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def eval():\n    model.eval()\n    eval_loss, eval_accuracy = 0, 0\n    nb_eval_steps, nb_eval_examples = 0, 0\n    predicts = []\n    with torch.no_grad():\n        for input_ids, attention_mask, label in valid_dataloader:\n            loss, logits = model(input_ids, labels=label)\n            predicts.append(np.argmax(logits, axis=1))\n    return predicts","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predicts = eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tmp_eval_accuracy = metrics.roc_auc_score(part_valid['toxic'], predicts)\nprint(tmp_eval_accuracy)\nn_wrong = 0\nfor p in range(len(predicts)):\n    if part_valid['toxic'].iloc[p] != predicts[p]:\n        print(part_valid.iloc[p]['comment_text'])\n        print(part_valid.iloc[p]['toxic'])\n        n_wrong += 1\n    if n_wrong == 20:\n        break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.save(model, 'model.pt')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}