{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\nimport re\nimport os\nimport sys\nimport time\nimport pickle\nimport random\nimport unidecode\nfrom tqdm import tqdm\ntqdm.pandas()\nfrom scipy.stats import spearmanr\nfrom gensim.models import Word2Vec\nfrom flashtext import KeywordProcessor\nfrom keras.preprocessing import text, sequence\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom sklearn.model_selection import StratifiedKFold, KFold\nfrom transformers import (\n    BertTokenizer, BertModel, BertForSequenceClassification, BertConfig,\n    WEIGHTS_NAME, CONFIG_NAME, AdamW, get_linear_schedule_with_warmup, \n    get_cosine_schedule_with_warmup,\n)\n\nfrom math import floor, ceil\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"### just get 50,000 training data\ntrain = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv', usecols=['id', 'comment_text', 'toxic']).sample(n=25000)\ntrain_2 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv', usecols=['id', 'comment_text', 'toxic', 'rating'])\ntrain_2 = train_2.loc[train_2['rating']=='approved'].drop(columns=['rating']).sample(n=25000)\n\ntrain = pd.concat([train, train_2]).reset_index(drop=True)\ntest = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')\nsub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')\nvalidation = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train_2\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer = BertTokenizer.from_pretrained(\"/kaggle/input/bert-mmm/bert-base-multilingual-uncased-vocab.txt\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_newline(context):\n    return re.sub(r'(\\n)', ' ', context)\n\ntrain['clean_text'] = train['comment_text'].apply(lambda x: clean_newline(x))\ntest['clean_text'] = test['content'].apply(lambda x: clean_newline(x))\nvalidation['clean_text'] = validation['comment_text'].apply(lambda x: clean_newline(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['comment_text_len'] = train.clean_text.apply(lambda x : len(x.split()))\ntest['comment_text_len'] = test.clean_text.apply(lambda x : len(x.split()))\nvalidation['comment_text_len'] = train.clean_text.apply(lambda x : len(x.split()))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.hist('comment_text_len')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.hist('comment_text_len')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"validation.hist('comment_text_len')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MAX_SEQ_LENGTH = 200\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TextDataset(torch.utils.data.TensorDataset):\n\n    def __init__(self, data, idxs, targets=None):\n        self.input_ids = data[0][idxs]\n        self.input_masks = data[1][idxs]\n        self.input_segments = data[2][idxs]\n        self.targets = targets[idxs] if targets is not None else np.zeros((data[0].shape[0], 1))\n    def __getitem__(self, idx):\n        input_ids =  self.input_ids[idx]\n        input_masks = self.input_masks[idx]\n        input_segments = self.input_segments[idx]\n\n        target = self.targets[idx]\n\n        return input_ids, input_masks, input_segments, target\n\n    def __len__(self):\n        return len(self.input_ids)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_bert_tokenize(df):\n    \n    token_ids, segment_ids, attention_mask = np.zeros((len(df), MAX_SEQ_LENGTH)), np.zeros((len(df), MAX_SEQ_LENGTH)), np.zeros((len(df), MAX_SEQ_LENGTH))\n    \n    for i, content in tqdm(enumerate(df.clean_text.values), total=len(df)):\n        \n        content = tokenizer.tokenize(content)[:MAX_SEQ_LENGTH-2]\n        token_id = tokenizer.encode(content)\n        \n        token_ids[i] = token_id + [0] * (MAX_SEQ_LENGTH-len(token_id))\n        segment_ids[i] = [0] * MAX_SEQ_LENGTH               \n        attention_mask[i] = [1] * len(token_id) + [0] * (MAX_SEQ_LENGTH - len(token_id))\n        \n    return token_ids, segment_ids, attention_mask\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nx_train = get_bert_tokenize(train)\nx_test = get_bert_tokenize(test)\nx_validation = get_bert_tokenize(validation)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bert_config = BertConfig.from_pretrained('/kaggle/input/bert-pytorch/bert-base-multilingual-uncased-config.json') \nbert_config.num_labels = 1\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nfrom torch import nn\nfrom torch.nn import CrossEntropyLoss, MSELoss\nfrom transformers import BertPreTrainedModel,BertModel\n\n\nclass CustomizedBert(BertPreTrainedModel):\n    r\"\"\"\n        **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``:\n            Labels for computing the sequence classification/regression loss.\n            Indices should be in ``[0, ..., config.num_labels - 1]``.\n            If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss),\n            If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy).\n\n    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:\n        **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:\n            Classification (or regression if config.num_labels==1) loss.\n        **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)``\n            Classification (or regression if config.num_labels==1) scores (before SoftMax).\n        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)\n            list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)\n            of shape ``(batch_size, sequence_length, hidden_size)``:\n            Hidden-states of the model at the output of each layer plus the initial embedding outputs.\n        **attentions**: (`optional`, returned when ``config.output_attentions=True``)\n            list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:\n            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.\n\n    Examples::\n\n        tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n        model = BertForSequenceClassification.from_pretrained('bert-base-uncased')\n        input_ids = torch.tensor(tokenizer.encode(\"Hello, my dog is cute\", add_special_tokens=True)).unsqueeze(0)  # Batch size 1\n        labels = torch.tensor([1]).unsqueeze(0)  # Batch size 1\n        outputs = model(input_ids, labels=labels)\n        loss, logits = outputs[:2]\n\n    \"\"\"\n\n    def __init__(self, config):\n        super(CustomizedBert, self).__init__(config)\n        self.num_labels = config.num_labels\n\n        self.bert = BertModel(config)\n        self.dropout = nn.Dropout(config.hidden_dropout_prob)\n        self.classifier = nn.Linear(config.hidden_size*2, self.config.num_labels)\n\n        self.init_weights()\n    def forward(\n        self,\n        input_ids=None,\n        attention_mask=None,\n        token_type_ids=None,\n        position_ids=None,\n        head_mask=None,\n        inputs_embeds=None,\n        labels=None,\n    ):\n\n        outputs = self.bert(\n            input_ids,\n            attention_mask=attention_mask,\n            token_type_ids=token_type_ids,\n            position_ids=position_ids,\n            head_mask=head_mask,\n            inputs_embeds=inputs_embeds,\n        )\n\n        ## mean max pooling and concatenate to a vector\n        \n        avg_pool = torch.mean(outputs[0], 1)\n        max_pool, _ = torch.max(outputs[0], 1)\n        pooled_output = torch.cat((max_pool, avg_pool), 1)\n\n        pooled_output = self.dropout(pooled_output)\n        logits = self.classifier(pooled_output)\n\n        outputs = (logits,) + outputs[2:]  # add hidden states and attention if they are here\n\n        return outputs  # (loss), logits, (hidden_states), (attentions)\n\n\nclass callback:\n    def __init__(self):\n        self.score = list()\n        self.model = list()\n    \n    def put(self, model, score):\n        self.score.append(score)\n        self.model.append(model)\n\n    def get_model(self):\n        ind = np.argmin(self.score)\n        return self.model[ind]\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def model_validation(net, val_loader, validation_length):\n    \n    avg_val_loss = 0.0\n    \n    net.eval()\n    valid_preds = np.zeros((validation_length, 1))\n    true_label = np.zeros((validation_length, 1))\n    \n    for j, data in enumerate(val_loader):\n\n        # get the inputs\n        input_ids, input_masks, input_segments, labels = data\n        pred = net(input_ids = input_ids.long().cuda(),\n                         labels = None,\n                         attention_mask = input_masks.cuda(),\n                         token_type_ids = input_segments.long().cuda()\n                        )[0]\n\n        loss_val = loss_fn(pred, labels.float().cuda())\n        avg_val_loss += loss_val.item()\n\n        valid_preds[j * BATCH_SIZE:(j+1) * BATCH_SIZE] = torch.sigmoid(pred).cpu().detach().numpy()\n        true_label[j * BATCH_SIZE:(j+1) * BATCH_SIZE]  = labels.float()\n\n\n    score = roc_auc_score(true_label, valid_preds)\n    \n    return valid_preds, avg_val_loss, score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_auc_score, accuracy_score\n\ncb = callback()\n    \nBATCH_SIZE = 32\nEPOCHS = 2\n\nSEED = 2020\nlr = 3e-5\n\n\ngradient_accumulation_steps = 1\nseed_everything(SEED)\n\nmodel_list = list()\n\ny_train = train.toxic.values.reshape(-1, 1)\ny_val = validation.toxic.values.reshape(-1, 1)\n\ntest_pred = np.zeros((len(test), 1))\n\ntest_loader = torch.utils.data.DataLoader(TextDataset(x_test, test.index),batch_size=BATCH_SIZE, shuffle=False)\n\n    \n    \n## loader\ntrain_loader = torch.utils.data.DataLoader(TextDataset(x_train, train.index, y_train),batch_size=BATCH_SIZE, shuffle=True)\nval_loader = torch.utils.data.DataLoader(TextDataset(x_validation, validation.index, y_val),batch_size=BATCH_SIZE, shuffle=False)\n\nt_total = len(train_loader)//gradient_accumulation_steps*EPOCHS\nwarmup_proportion = 0.01\nnum_warmup_steps = t_total * warmup_proportion\n\nnet = CustomizedBert.from_pretrained('/kaggle/input/bert-pytorch/bert-base-multilingual-uncased-pytorch_model.bin', config=bert_config)\nnet.cuda()\n\nloss_fn = torch.nn.BCEWithLogitsLoss()\noptimizer = AdamW(net.parameters(), lr = lr)\nscheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=num_warmup_steps, num_training_steps=t_total)  # PyTorch scheduler\n\nprint('Start fine tuning BERT')\nfor epoch in range(EPOCHS):  \n\n    start_time = time.time()\n    avg_loss = 0.0\n    net.train()\n\n    for step, data in enumerate(train_loader):\n\n        # get the inputs\n        input_ids, input_masks, input_segments, labels = data\n\n\n        pred = net(input_ids = input_ids.long().cuda(),\n                         labels = None,\n                         attention_mask = input_masks.cuda(),\n                         token_type_ids = input_segments.long().cuda()\n                        )[0]\n\n\n        loss = loss_fn(pred, labels.float().cuda())\n\n        avg_loss += loss.item()\n        loss = loss / gradient_accumulation_steps\n        loss.backward()\n\n        if (step + 1) % gradient_accumulation_steps == 0:\n\n            # Calling the step function on an Optimizer makes an update to its parameters\n            optimizer.step()\n            scheduler.step()\n            optimizer.zero_grad()\n\n\n    valid_preds, val_loss, val_score  = model_validation(net, val_loader, len(validation))\n\n    elapsed_time = time.time() - start_time \n\n\n    print('Epoch {}/{} \\t loss={:.4f}\\t val_loss={:.4f} \\t val_score={:.4f}\\t time={:.2f}s'\n          .format(epoch+1, EPOCHS, avg_loss/len(train_loader), val_loss/len(val_loader), val_score , elapsed_time))\n\n    cb.put(net, val_loss/len(val_loader))\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## And Lastly, I add validation dataset to fine tuning model finally."},{"metadata":{"trusted":true},"cell_type":"code","source":"net = cb.get_model()\n\nfor epoch in range(EPOCHS):  \n\n    start_time = time.time()\n    avg_loss = 0.0\n    net.train()\n\n    for step, data in enumerate(val_loader):\n\n        # get the inputs\n        input_ids, input_masks, input_segments, labels = data\n\n\n        pred = net(input_ids = input_ids.long().cuda(),\n                         labels = None,\n                         attention_mask = input_masks.cuda(),\n                         token_type_ids = input_segments.long().cuda()\n                        )[0]\n\n\n        loss = loss_fn(pred, labels.float().cuda())\n\n        avg_loss += loss.item()\n        loss = loss / gradient_accumulation_steps\n        loss.backward()\n\n        if (step + 1) % gradient_accumulation_steps == 0:\n\n            # Calling the step function on an Optimizer makes an update to its parameters\n            optimizer.step()\n            scheduler.step()\n            optimizer.zero_grad()\n\n    elapsed_time = time.time() - start_time \n    print('Epoch {}/{} \\t loss={:.4f} \\t time={:.2f}s'\n          .format(epoch+1, EPOCHS, avg_loss/len(val_loader) elapsed_time))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pickle.dump(cb.get_model(), open('bert_model.pkl', 'wb'))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_loader = torch.utils.data.DataLoader(TextDataset(x_test, test.index), batch_size=128, shuffle=False)\n\nresult = list()\n\n# net = cb.get_model()\nnet.eval()\nwith torch.no_grad():\n    for data in test_loader:\n        input_ids, input_masks, input_segments, _ = data\n        y_pred = net(input_ids = input_ids.long().cuda(),\n                            labels = None,\n                            attention_mask = input_masks.cuda(),\n                            token_type_ids = input_segments.long().cuda(),\n                        )[0]\n        result.extend(torch.sigmoid(y_pred).cpu().detach().numpy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['toxic'] = np.array(result)\nsub.to_csv('submission.csv', index=False)\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.hist('toxic')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.describe()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}