{"cells":[{"metadata":{},"cell_type":"markdown","source":"Testing out fine tuning pre-trained BERT using fastai.\n\nAlmost all the code is from https://mlexplained.com/2019/05/13/a-tutorial-to-fine-tuning-bert-with-fast-ai/"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-output":true,"collapsed":true},"cell_type":"code","source":"!pip install pytorch-pretrained-bert","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfrom shutil import copyfile\n\nfrom fastai.text import * \nfrom pytorch_pretrained_bert.modeling import BertConfig, BertForSequenceClassification\nfrom pytorch_pretrained_bert import BertTokenizer\n\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import f1_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Setting path for learner\npath = Path(os.path.abspath(os.curdir))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create directory\ndirName = 'models'\n \ntry:\n    # Create target Directory\n    os.mkdir(dirName)\n    print(\"Directory:\" , dirName , \"created!\") \nexcept FileExistsError:\n    print(\"Directory:\" , dirName , \"already exists!\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#copying files into working path\nmodelpath = Path('../input/bert-fastai-error-analysis')\n\ncopyfile(modelpath/\"models/bert-1.pth\", path/\"models/bert-1.pth\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#reading into pandas and renaming columns for easier api access\nfilepath = Path('../input/quora-insincere-questions-classification')\ntrn = pd.read_csv(filepath/'train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trn.rename(columns={'target':'label', 'question_text':'text'},inplace=True)\ndf = trn[['label','text']]\n\ndf['1'] = df['label'].apply(lambda x: 1 if x==1 else 0)\ndf['0'] = df['label'].apply(lambda x: 1 if x==0 else 0)\n\nvalid = df[int(len(df)*.80):]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Config(dict):\n    def __init__(self, **kwargs):\n        super().__init__(**kwargs)\n        for k, v in kwargs.items():\n            setattr(self, k, v)\n    \n    def set(self, key, val):\n        self[key] = val\n        setattr(self, key, val)\n\nconfig = Config(\n    testing=False,\n    bert_model_name=\"bert-base-uncased\",\n    max_lr=3e-5,\n    epochs=4,\n    use_fp16=True,\n    bs=32,\n    discriminative=False,\n    max_seq_len=256,\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bert_tok = BertTokenizer.from_pretrained(config.bert_model_name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FastAiBertTokenizer(BaseTokenizer): \n    \"\"\"Wrapper around BertTokenizer to be compatible with fast.ai\"\"\"\n    def __init__(self, tokenizer: BertTokenizer, max_seq_len: int=128, **kwargs): \n         self._pretrained_tokenizer = tokenizer \n         self.max_seq_len = max_seq_len \n    def __call__(self, *args, **kwargs): \n         return self \n    def tokenizer(self, t:str) -> List[str]: #Limits the maximum sequence length\n        return [\"[CLS]\"] + self._pretrained_tokenizer.tokenize(t)[:self.max_seq_len - 2] + [\"[SEP]\"]   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fastai_tokenizer = Tokenizer(tok_func=FastAiBertTokenizer(bert_tok, max_seq_len=config.max_seq_len), \n                             pre_rules=[], post_rules=[])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _join_texts(texts:Collection[str], mark_fields:bool=False, sos_token:Optional[str]=BOS):\n    \"\"\"Borrowed from fast.ai source\"\"\"\n    if not isinstance(texts, np.ndarray): texts = np.array(texts)\n    if is1d(texts): texts = texts[:,None]\n    df = pd.DataFrame({i:texts[:,i] for i in range(texts.shape[1])})\n    text_col = f'{FLD} {1} ' + df[0].astype(str) if mark_fields else df[0].astype(str)\n    if sos_token is not None: text_col = f\"{sos_token} \" + text_col\n    for i in range(1,len(df.columns)):\n        #text_col += (f' {FLD} {i+1} ' if mark_fields else ' ') + df[i]\n        text_col += (f' {FLD} {i+1} ' if mark_fields else ' ') + df[i].astype(str)\n    return text_col.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fastai_bert_vocab = Vocab(list(bert_tok.vocab.keys()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fastai_tokenizer = Tokenizer(tok_func=FastAiBertTokenizer(bert_tok, max_seq_len=config.max_seq_len), \n                             pre_rules=[], post_rules=[])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_cols = [\"1\", \"0\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"databunch = TextDataBunch.from_df(\".\", valid, valid, valid,\n                   tokenizer=fastai_tokenizer,\n                   vocab=fastai_bert_vocab,\n                   include_bos=False,\n                   include_eos=False,\n                   text_cols=\"text\",\n                   label_cols=label_cols,\n                   bs=config.bs,\n                   collate_fn=partial(pad_collate, pad_first=False, pad_idx=0),\n              )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class BertTokenizeProcessor(TokenizeProcessor):\n    def __init__(self, tokenizer):\n        super().__init__(tokenizer=tokenizer, include_bos=False, include_eos=False)\n\nclass BertNumericalizeProcessor(NumericalizeProcessor):\n    def __init__(self, *args, **kwargs):\n        super().__init__(*args, vocab=Vocab(list(bert_tok.vocab.keys())), **kwargs)\n\ndef get_bert_processor(tokenizer:Tokenizer=None, vocab:Vocab=None):\n    \"\"\"\n    Constructing preprocessors for BERT\n    We remove sos/eos tokens since we add that ourselves in the tokenizer.\n    We also use a custom vocabulary to match the numericalization with the original BERT model.\n    \"\"\"\n    return [BertTokenizeProcessor(tokenizer=tokenizer),\n            NumericalizeProcessor(vocab=vocab)]\n\nclass BertDataBunch(TextDataBunch):\n    @classmethod\n    def from_df(cls, path:PathOrStr, train_df:DataFrame, valid_df:DataFrame, test_df:Optional[DataFrame]=None,\n                tokenizer:Tokenizer=None, vocab:Vocab=None, classes:Collection[str]=None, text_cols:IntsOrStrs=1,\n                label_cols:IntsOrStrs=0, label_delim:str=None, **kwargs) -> DataBunch:\n        \"Create a `TextDataBunch` from DataFrames.\"\n        p_kwargs, kwargs = split_kwargs_by_func(kwargs, get_bert_processor)\n        # use our custom processors while taking tokenizer and vocab as kwargs\n        processor = get_bert_processor(tokenizer=tokenizer, vocab=vocab, **p_kwargs)\n        if classes is None and is_listy(label_cols) and len(label_cols) > 1: classes = label_cols\n        src = ItemLists(path, TextList.from_df(train_df, path, cols=text_cols, processor=processor),\n                        TextList.from_df(valid_df, path, cols=text_cols, processor=processor))\n        src = src.label_for_lm() if cls==TextLMDataBunch else src.label_from_df(cols=label_cols, classes=classes)\n        if test_df is not None: src.add_test(TextList.from_df(test_df, path, cols=text_cols))\n        return src.databunch(**kwargs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bert_model = BertForSequenceClassification.from_pretrained(config.bert_model_name, num_labels=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learner = Learner(databunch, bert_model, loss_func=nn.BCEWithLogitsLoss())\nif config.use_fp16: learner = learner.to_fp16()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learner.load('bert-1')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_preds_as_nparray(ds_type) -> np.ndarray:\n    \"\"\"\n    the get_preds method does not yield the elements in order by default\n    we borrow the code from the RNNLearner to resort the elements into their correct order\n    \"\"\"\n    preds = learner.get_preds(ds_type)[0].detach().cpu().numpy()\n    y = learner.get_preds(ds_type)[1].detach().cpu().numpy()\n    \n    sampler = [i for i in databunch.dl(ds_type).sampler]\n    reverse_sampler = np.argsort(sampler)\n    \n    return preds[reverse_sampler, :], y[reverse_sampler, :] ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#testing on bottom 20% of data\npreds, y = get_preds_as_nparray(DatasetType.Valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid.reset_index(inplace=True)\nvalid.rename(columns={'1':'insincere_gt', '0': 'sincere_gt'}, inplace=True); print(valid.shape); valid.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds_df = pd.DataFrame(preds,columns=['insincere_pred', 'sincere_pred']) ; print(preds_df.shape) ; preds_df.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#convert predicted prob to predicted labels\nidx = np.argmax(preds, axis=-1)\ny_preds = np.zeros(preds.shape)\ny_preds[np.arange(preds.shape[0]), idx] = 1\n\naccuracy_score(y_preds, y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_proba = preds[:, 0]\ny_finalpred = np.asarray([1 if x>0.3 else 0 for x in y_proba ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_df = pd.DataFrame(y_finalpred,columns=['label_pred']) ; print(label_df.shape) ; label_df.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final = valid.merge(preds_df, left_index=True, right_index=True)\nfinal = final.merge(label_df, left_index=True, right_index=True)\nfinal.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final.to_csv('final.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Checking if model is still correct after threshold adjustment"},{"metadata":{"trusted":true},"cell_type":"code","source":"#change y from multi-label to single label\nidx = np.argmax(y, axis=-1)\ny_new = np.zeros((y.shape[0],1))\nfor i in range(len(idx)):\n    if idx[i]==0:y_new[i]=1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"accuracy_score(y_finalpred, y_new)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f1_score(y_new, y_finalpred)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}