{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport os\nimport random\n\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm, tqdm_notebook, trange\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import f1_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install transformers","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from transformers import BertConfig, BertForSequenceClassification, BertTokenizer, AdamW, WarmupLinearSchedule\n\nimport torch\nfrom torch.utils.tensorboard import SummaryWriter","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SEED = 2019\n\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    \nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_df = pd.read_csv(\"../input/quora-insincere-questions-classification/train.csv\")\ntest_df = pd.read_csv(\"../input/quora-insincere-questions-classification/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(data_df))\ndata_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(test_df))\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"N_FOLDS = 5\n\nskf = StratifiedKFold(n_splits=N_FOLDS, random_state=42, shuffle=True)\nsplits = skf.split(data_df, data_df['target'])\n\ndef get_fold(fold):    \n    for i, (train_index, valid_index) in enumerate(splits):\n        if i == fold:\n            return data_df.iloc[train_index], data_df.iloc[valid_index]\n        \ntrain_df, valid_df = get_fold(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.value_counts(train_df['target']).plot.bar()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.value_counts(valid_df['target']).plot.bar()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MAX_LENGTH = 320\nBATCH_SIZE = 24\nEPOCHS = 1\nLEARNING_RATE = 2e-5\nACCUM_STEPS = 1\nEVAL_STEPS = 10000\nMAX_STEPS = 20000","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pretrained_weights = 'bert-base-uncased'\n\nconfig = BertConfig.from_pretrained(pretrained_weights, num_labels=2)\nmodel = BertForSequenceClassification.from_pretrained(pretrained_weights, config=config)\ntokenizer = BertTokenizer.from_pretrained(pretrained_weights, do_lower_case=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def encode_text(texts):\n    \n    # encoding\n    X = [tokenizer.encode(text, add_special_tokens=True, max_length=MAX_LENGTH) \n         for text in tqdm(texts)]\n    \n    # padding\n    X = [x + [0 for _ in range(MAX_LENGTH-len(x))] for x in X]            \n    \n    return X\n\ntrain_X = encode_text(train_df['question_text'])\ntrain_y = train_df['target'].values\n\nvalid_X = encode_text(valid_df['question_text'])\nvalid_y = valid_df['target'].values\n\ntest_X = encode_text(test_df['question_text'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def evaluate(model):    \n    preds = None\n    eval_loss = 0.0\n    nb_eval_steps = 0\n    \n    valid_dataset = torch.utils.data.TensorDataset(torch.tensor(valid_X, dtype=torch.long), torch.tensor(valid_y, dtype=torch.long))\n    valid_sampler = torch.utils.data.SequentialSampler(valid_dataset)\n    valid_loader = torch.utils.data.DataLoader(valid_dataset, sampler=valid_sampler, batch_size=BATCH_SIZE)    \n    \n    for batch in tqdm_notebook(valid_loader, desc=\"Evaluating\"):\n        model.eval()\n        batch = tuple(t.to(device) for t in batch)\n\n        with torch.no_grad():\n            inputs = {'input_ids': batch[0], 'labels': batch[1]}\n            outputs = model(**inputs)\n            \n            tmp_eval_loss, logits = outputs[:2]\n            eval_loss += tmp_eval_loss.mean().item()\n\n        nb_eval_steps += 1\n        \n        if preds is None:\n            preds = logits.detach().cpu().numpy()\n        else:\n            preds = np.append(preds, logits.detach().cpu().numpy(), axis=0)\n\n    eval_loss = eval_loss / nb_eval_steps\n    \n    preds = np.argmax(preds, axis=1)\n    f1 = f1_score(valid_y, preds)\n    \n    return eval_loss, f1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = torch.utils.data.TensorDataset(torch.tensor(train_X, dtype=torch.long), torch.tensor(train_y, dtype=torch.long))\ntrain_sampler = torch.utils.data.RandomSampler(train_dataset)\ntrain_loader = torch.utils.data.DataLoader(train_dataset, sampler=train_sampler, batch_size=BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device('cuda')\nmodel = model.to(device)\n\nmodel.zero_grad()\n\nno_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\noptimizer_grouped_parameters = [\n    {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},\n    {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n    ]\n\nif MAX_STEPS > 0:\n    t_total = MAX_STEPS\nelse:\n    t_total = len(train_loader) // ACCUM_STEPS * EPOCHS\n\noptimizer = AdamW(optimizer_grouped_parameters, lr=LEARNING_RATE, eps=1e-8)\nscheduler = WarmupLinearSchedule(optimizer, warmup_steps=0, t_total=t_total)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"global_step = 0\ntr_loss, logging_loss = 0.0, 0.0\n\ntq = tqdm_notebook(range(EPOCHS))\ntb_writer = SummaryWriter()\n\nbest_f1 = -1 #0.0\noutput_model_file = 'pytorch_model.bin'\n\nfor _ in tq:\n    lossf = None\n    eval_loss, eval_f1 = None, None\n    \n    tk0 = tqdm_notebook(enumerate(train_loader), total=len(train_loader), leave=False)\n    \n    for step, batch in tk0:\n        model.train()\n        batch = tuple(t.to(device) for t in batch)\n        \n        inputs = {'input_ids': batch[0], 'labels': batch[1]}\n\n        outputs = model(**inputs)\n        loss = outputs[0] # model outputs are always tuple in transformers (see doc)\n\n        if ACCUM_STEPS > 1:\n            loss = loss / ACCUM_STEPS\n\n        loss.backward()\n        tr_loss += loss.item()\n        \n        if (step + 1) % ACCUM_STEPS == 0:\n            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n            \n            optimizer.step()\n            scheduler.step()\n            model.zero_grad()\n            \n            global_step += 1\n            \n        if EVAL_STEPS > 0 and (step + 1) % EVAL_STEPS == 0:\n            eval_loss, eval_f1 = evaluate(model)\n            \n            tb_writer.add_scalar('loss', (tr_loss - logging_loss)/EVAL_STEPS, global_step)\n            logging_loss = tr_loss\n            \n            tb_writer.add_scalar('eval_loss', eval_loss, global_step)\n            tb_writer.add_scalar('eval_f1', eval_f1, global_step)\n            \n            if eval_f1 > best_f1:\n                torch.save(model.state_dict(), output_model_file)\n                best_f1 = eval_f1\n                    \n        lossf = 0.98*lossf + 0.02*loss.item() if lossf else loss.item()        \n        tk0.set_postfix(loss=lossf, eval_loss=eval_loss, eval_f1=eval_f1)\n        \n        if MAX_STEPS > 0 and global_step > MAX_STEPS:\n            tk0.close()\n            break\n\n    if MAX_STEPS > 0 and global_step > MAX_STEPS:\n        tq.close()\n        break        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = torch.utils.data.TensorDataset(torch.tensor(test_X, dtype=torch.long))\ntest_sampler = torch.utils.data.SequentialSampler(test_dataset)\ntest_loader = torch.utils.data.DataLoader(test_dataset, sampler=test_sampler, batch_size=BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_state_dict = torch.load('pytorch_model.bin')\nmodel = BertForSequenceClassification.from_pretrained(pretrained_weights, state_dict=model_state_dict)\nmodel.to(device)\n\npreds = None\n\nfor batch in tqdm_notebook(test_loader, desc=\"testing\"):\n    model.eval()\n    batch = tuple(t.to(device) for t in batch)\n\n    with torch.no_grad():\n        inputs = {'input_ids': batch[0]}\n        outputs = model(**inputs)\n        logits = outputs[0]\n\n    if preds is None:\n        preds = logits.detach().cpu().numpy()\n    else:\n        preds = np.append(preds, logits.detach().cpu().numpy(), axis=0)\n\npreds = np.argmax(preds, axis=1)\n\nsubmission = test_df[['qid']]\nsubmission['prediction'] = preds\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.3"}},"nbformat":4,"nbformat_minor":1}