{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nfrom pathlib import Path\nfrom typing import *\n\nimport torch\nimport torch.optim as optim","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from fastai import *\nfrom fastai.vision import *\nfrom fastai.text import *\nfrom fastai.callbacks import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%bash\npip install pytorch-pretrained-bert","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Config(dict):\n    def __init__(self, **kwargs):\n        super().__init__(**kwargs)\n        for k, v in kwargs.items():\n            setattr(self, k, v)\n    \n    def set(self, key, val):\n        self[key] = val\n        setattr(self, key, val)\n\nconfig = Config(\n    testing=False,\n    bert_model_name=\"bert-base-uncased\",\n    max_lr=3e-5,\n    epochs=4,\n    use_fp16=True,\n    bs=32,\n    discriminative=False,\n    max_seq_len=256,\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from pytorch_pretrained_bert import BertTokenizer\nbert_tok = BertTokenizer.from_pretrained(\n    config.bert_model_name,\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _join_texts(texts:Collection[str], mark_fields:bool=False, sos_token:Optional[str]=BOS):\n    \"\"\"Borrowed from fast.ai source\"\"\"\n    if not isinstance(texts, np.ndarray): texts = np.array(texts)\n    if is1d(texts): texts = texts[:,None]\n    df = pd.DataFrame({i:texts[:,i] for i in range(texts.shape[1])})\n    text_col = f'{FLD} {1} ' + df[0].astype(str) if mark_fields else df[0].astype(str)\n    if sos_token is not None: text_col = f\"{sos_token} \" + text_col\n    for i in range(1,len(df.columns)):\n        #text_col += (f' {FLD} {i+1} ' if mark_fields else ' ') + df[i]\n        text_col += (f' {FLD} {i+1} ' if mark_fields else ' ') + df[i].astype(str)\n    return text_col.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FastAiBertTokenizer(BaseTokenizer):\n    \"\"\"Wrapper around BertTokenizer to be compatible with fast.ai\"\"\"\n    def __init__(self, tokenizer: BertTokenizer, max_seq_len: int=128, **kwargs):\n        self._pretrained_tokenizer = tokenizer\n        self.max_seq_len = max_seq_len\n\n    def __call__(self, *args, **kwargs):\n        return self\n\n    def tokenizer(self, t:str) -> List[str]:\n        \"\"\"Limits the maximum sequence length\"\"\"\n        return [\"[CLS]\"] + self._pretrained_tokenizer.tokenize(t)[:self.max_seq_len - 2] + [\"[SEP]\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\ntrain = pd.read_csv('../input/til2020/TIL_NLP_train_dataset.csv')\ntest = pd.read_csv('../input/til2020/TIL_NLP_test_dataset.csv')\nval = train # we won't be using a validation set but you can easily create one using train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if config.testing:\n    train = train.head(1024)\n    val = val.head(1024)\n    test = test.head(1024)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fastai_bert_vocab = Vocab(list(bert_tok.vocab.keys()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fastai_tokenizer = Tokenizer(tok_func=FastAiBertTokenizer(bert_tok, max_seq_len=config.max_seq_len), pre_rules=[], post_rules=[])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_cols = [\"outwear\", \"top\", \"trousers\", \"women dresses\", \"women skirts\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class BertTokenizeProcessor(TokenizeProcessor):\n    def __init__(self, tokenizer):\n        super().__init__(tokenizer=tokenizer, include_bos=False, include_eos=False)\n\nclass BertNumericalizeProcessor(NumericalizeProcessor):\n    def __init__(self, *args, **kwargs):\n        super().__init__(*args, vocab=Vocab(list(bert_tok.vocab.keys())), **kwargs)\n\ndef get_bert_processor(tokenizer:Tokenizer=None, vocab:Vocab=None):\n    \"\"\"\n    Constructing preprocessors for BERT\n    We remove sos/eos tokens since we add that ourselves in the tokenizer.\n    We also use a custom vocabulary to match the numericalization with the original BERT model.\n    \"\"\"\n    return [BertTokenizeProcessor(tokenizer=tokenizer),\n            NumericalizeProcessor(vocab=vocab)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class BertDataBunch(TextDataBunch):\n    @classmethod\n    def from_df(cls, path:PathOrStr, train_df:DataFrame, valid_df:DataFrame, test_df:Optional[DataFrame]=None,\n                tokenizer:Tokenizer=None, vocab:Vocab=None, classes:Collection[str]=None, text_cols:IntsOrStrs=1,\n                label_cols:IntsOrStrs=0, label_delim:str=None, **kwargs) -> DataBunch:\n        \"Create a `TextDataBunch` from DataFrames.\"\n        p_kwargs, kwargs = split_kwargs_by_func(kwargs, get_bert_processor)\n        # use our custom processors while taking tokenizer and vocab as kwargs\n        processor = get_bert_processor(tokenizer=tokenizer, vocab=vocab, **p_kwargs)\n        if classes is None and is_listy(label_cols) and len(label_cols) > 1: classes = label_cols\n        src = ItemLists(path, TextList.from_df(train_df, path, cols=text_cols, processor=processor),\n                        TextList.from_df(valid_df, path, cols=text_cols, processor=processor))\n        src = src.label_for_lm() if cls==TextLMDataBunch else src.label_from_df(cols=label_cols, classes=classes)\n        if test_df is not None: src.add_test(TextList.from_df(test_df, path, cols=text_cols))\n        return src.databunch(**kwargs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"databunch = BertDataBunch.from_df(\".\", train, val, test,\n                  tokenizer=fastai_tokenizer,\n                  vocab=fastai_bert_vocab,\n                  text_cols=\"word_representation\",\n                  label_cols=label_cols,\n                  bs=config.bs,\n                  collate_fn=partial(pad_collate, pad_first=False, pad_idx=0),\n             )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from pytorch_pretrained_bert.modeling import BertConfig, BertForSequenceClassification\nbert_model = BertForSequenceClassification.from_pretrained(config.bert_model_name, num_labels=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"loss_func = nn.BCEWithLogitsLoss()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from fastai.callbacks import *\n\nlearner = Learner(\n    databunch, bert_model,\n    loss_func=loss_func,\n)\nif config.use_fp16: learner = learner.to_fp16()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learner.lr_find()\nlearner.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#learner.fit_one_cycle(config.epochs, max_lr=config.max_lr)\nlearner.fit_one_cycle(cyc_len = 20, max_lr=config.max_lr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learner.save('BERT_best_model')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_preds_as_nparray(ds_type) -> np.ndarray:\n    \"\"\"\n    the get_preds method does not yield the elements in order by default\n    we borrow the code from the RNNLearner to resort the elements into their correct order\n    \"\"\"\n    preds = learner.get_preds(ds_type)[0].detach().cpu().numpy()\n    sampler = [i for i in databunch.dl(ds_type).sampler]\n    reverse_sampler = np.argsort(sampler)\n    return preds[reverse_sampler, :]\n\ntest_preds = get_preds_as_nparray(DatasetType.Test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission = pd.read_csv('../input/til2020/NLP_submission_example.csv')\nif config.testing: sample_submission = sample_submission.head(test.shape[0])\nsample_submission[label_cols] = test_preds\nsample_submission[label_cols] = [[1 if x > 0.5 else 0 for idx,x in enumerate(i) ] for i in test_preds]\nsample_submission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}