{"cells":[{"metadata":{},"cell_type":"markdown","source":"Testing out fine tuning pre-trained BERT using fastai.\n\nAlmost all the code is from https://mlexplained.com/2019/05/13/a-tutorial-to-fine-tuning-bert-with-fast-ai/"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"!pip install pytorch-pretrained-bert","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfrom fastai.text import * \nfrom fastai.callbacks import *\nfrom pytorch_pretrained_bert.modeling import BertConfig, BertForSequenceClassification\nfrom pytorch_pretrained_bert import BertTokenizer\nfrom fastai.callbacks import CSVLogger","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#reading into pandas and renaming columns for easier api access\nfilepath = Path('../input/')\ntrn = pd.read_csv(filepath/'train.csv')\n#tst = pd.read_csv(filepath/'test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trn.rename(columns={'target':'label', 'question_text':'text'},inplace=True)\ndf = trn[['label','text']]\n\ndf['1'] = df['label'].apply(lambda x: 1 if x==1 else 0)\ndf['0'] = df['label'].apply(lambda x: 1 if x==0 else 0)\n\ntrain = df[:int(len(df)*.80)]\nvalid = df[int(len(df)*.80):]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Setting path for learner\npath = Path(os.path.abspath(os.curdir))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Config(dict):\n    def __init__(self, **kwargs):\n        super().__init__(**kwargs)\n        for k, v in kwargs.items():\n            setattr(self, k, v)\n    \n    def set(self, key, val):\n        self[key] = val\n        setattr(self, key, val)\n\nconfig = Config(\n    testing=False,\n    bert_model_name=\"bert-base-uncased\",\n    max_lr=3e-5,\n    epochs=4,\n    use_fp16=True,\n    bs=32,\n    discriminative=False,\n    max_seq_len=256,\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bert_tok = BertTokenizer.from_pretrained(config.bert_model_name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FastAiBertTokenizer(BaseTokenizer): \n    \"\"\"Wrapper around BertTokenizer to be compatible with fast.ai\"\"\"\n    def __init__(self, tokenizer: BertTokenizer, max_seq_len: int=128, **kwargs): \n         self._pretrained_tokenizer = tokenizer \n         self.max_seq_len = max_seq_len \n    def __call__(self, *args, **kwargs): \n         return self \n    def tokenizer(self, t:str) -> List[str]: #Limits the maximum sequence length\n        return [\"[CLS]\"] + self._pretrained_tokenizer.tokenize(t)[:self.max_seq_len - 2] + [\"[SEP]\"]   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fastai_tokenizer = Tokenizer(tok_func=FastAiBertTokenizer(bert_tok, max_seq_len=config.max_seq_len), \n                             pre_rules=[], post_rules=[])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _join_texts(texts:Collection[str], mark_fields:bool=False, sos_token:Optional[str]=BOS):\n    \"\"\"Borrowed from fast.ai source\"\"\"\n    if not isinstance(texts, np.ndarray): texts = np.array(texts)\n    if is1d(texts): texts = texts[:,None]\n    df = pd.DataFrame({i:texts[:,i] for i in range(texts.shape[1])})\n    text_col = f'{FLD} {1} ' + df[0].astype(str) if mark_fields else df[0].astype(str)\n    if sos_token is not None: text_col = f\"{sos_token} \" + text_col\n    for i in range(1,len(df.columns)):\n        #text_col += (f' {FLD} {i+1} ' if mark_fields else ' ') + df[i]\n        text_col += (f' {FLD} {i+1} ' if mark_fields else ' ') + df[i].astype(str)\n    return text_col.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if config.testing:\n    train = train.head(1024)\n    val = val.head(1024)\n    test = test.head(1024)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fastai_bert_vocab = Vocab(list(bert_tok.vocab.keys()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fastai_tokenizer = Tokenizer(tok_func=FastAiBertTokenizer(bert_tok, max_seq_len=config.max_seq_len), \n                             pre_rules=[], post_rules=[])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_cols = [\"1\", \"0\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"databunch = TextDataBunch.from_df(\".\", train, valid, valid,\n                   tokenizer=fastai_tokenizer,\n                   vocab=fastai_bert_vocab,\n                   include_bos=False,\n                   include_eos=False,\n                   text_cols=\"text\",\n                   label_cols=label_cols,\n                   bs=config.bs,\n                   collate_fn=partial(pad_collate, pad_first=False, pad_idx=0),\n              )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class BertTokenizeProcessor(TokenizeProcessor):\n    def __init__(self, tokenizer):\n        super().__init__(tokenizer=tokenizer, include_bos=False, include_eos=False)\n\nclass BertNumericalizeProcessor(NumericalizeProcessor):\n    def __init__(self, *args, **kwargs):\n        super().__init__(*args, vocab=Vocab(list(bert_tok.vocab.keys())), **kwargs)\n\ndef get_bert_processor(tokenizer:Tokenizer=None, vocab:Vocab=None):\n    \"\"\"\n    Constructing preprocessors for BERT\n    We remove sos/eos tokens since we add that ourselves in the tokenizer.\n    We also use a custom vocabulary to match the numericalization with the original BERT model.\n    \"\"\"\n    return [BertTokenizeProcessor(tokenizer=tokenizer),\n            NumericalizeProcessor(vocab=vocab)]\n\nclass BertDataBunch(TextDataBunch):\n    @classmethod\n    def from_df(cls, path:PathOrStr, train_df:DataFrame, valid_df:DataFrame, test_df:Optional[DataFrame]=None,\n                tokenizer:Tokenizer=None, vocab:Vocab=None, classes:Collection[str]=None, text_cols:IntsOrStrs=1,\n                label_cols:IntsOrStrs=0, label_delim:str=None, **kwargs) -> DataBunch:\n        \"Create a `TextDataBunch` from DataFrames.\"\n        p_kwargs, kwargs = split_kwargs_by_func(kwargs, get_bert_processor)\n        # use our custom processors while taking tokenizer and vocab as kwargs\n        processor = get_bert_processor(tokenizer=tokenizer, vocab=vocab, **p_kwargs)\n        if classes is None and is_listy(label_cols) and len(label_cols) > 1: classes = label_cols\n        src = ItemLists(path, TextList.from_df(train_df, path, cols=text_cols, processor=processor),\n                        TextList.from_df(valid_df, path, cols=text_cols, processor=processor))\n        src = src.label_for_lm() if cls==TextLMDataBunch else src.label_from_df(cols=label_cols, classes=classes)\n        if test_df is not None: src.add_test(TextList.from_df(test_df, path, cols=text_cols))\n        return src.databunch(**kwargs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bert_model = BertForSequenceClassification.from_pretrained(config.bert_model_name, num_labels=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"loss_func = nn.BCEWithLogitsLoss()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learner = Learner(databunch, bert_model, callback_fns=[partial(CSVLogger, append=True)], \n                  loss_func=loss_func)\nif config.use_fp16: learner = learner.to_fp16()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#learner.lr_find(); learner.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learner.fit_one_cycle(1, 3e-5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_preds_as_nparray(ds_type) -> np.ndarray:\n    \"\"\"\n    the get_preds method does not yield the elements in order by default\n    we borrow the code from the RNNLearner to resort the elements into their correct order\n    \"\"\"\n    preds = learner.get_preds(ds_type)[0].detach().cpu().numpy()\n    y = learner.get_preds(ds_type)[1].detach().cpu().numpy()\n    \n    sampler = [i for i in databunch.dl(ds_type).sampler]\n    reverse_sampler = np.argsort(sampler)\n    \n    return preds[reverse_sampler, :], y[reverse_sampler, :] ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#testing on bottom 20% of data\npreds, y = get_preds_as_nparray(DatasetType.Valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import accuracy_score\nfrom sklearn.metrics import f1_score\nfrom sklearn.metrics import precision_recall_fscore_support as score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/ryanzhang/tfidf-naivebayes-logreg-baseline\n\ndef threshold_search(y_true, y_proba):\n    best_threshold = 0\n    best_score = 0\n    for threshold in [i * 0.01 for i in range(100)]:\n        score = f1_score(y_true=y_true, y_pred=y_proba > threshold)\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n    search_result = {'threshold': best_threshold, 'f1': best_score}\n    return search_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#convert predicted prob to predicted labels\nidx = np.argmax(preds, axis=-1)\ny_preds = np.zeros(preds.shape)\ny_preds[np.arange(preds.shape[0]), idx] = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"accuracy_score(y_preds, y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_proba = preds[:, 0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#change y from multi-label to single label\nidx = np.argmax(y, axis=-1)\ny_new = np.zeros((y.shape[0],1))\nfor i in range(len(idx)):\n    if idx[i]==0:y_new[i]=1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"threshold_search(y_new, y_proba)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_finalpred = np.asarray([1 if x>0.3 else 0 for x in y_proba ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"score(y_new,y_finalpred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learner.save('bert-1')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}