{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"! pip install pytorch-transformers","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from fastai.text import *\nfrom fastai.metrics import *\nfrom pytorch_transformers import RobertaTokenizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = Path('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/')\npath.ls()\n!mkdir data\n!pwd\n!cp -a ../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv ./data/\n!cp -a ../input/jigsaw-multilingual-toxic-comment-classification/test.csv ./data/\n!cp -a ../input/jigsaw-multilingual-toxic-comment-classification/validation.csv ./data/\n#!cp -a ../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv ./data\n#!cp -a ../input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv ./data\n!ls data\n\npath = Path('/kaggle/working/data/')\npath.ls()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Creating a config object to store task specific information\nclass Config(dict):\n    def __init__(self, **kwargs):\n        super().__init__(**kwargs)\n        for k, v in kwargs.items():\n            setattr(self, k, v)\n    \n    def set(self, key, val):\n        self[key] = val\n        setattr(self, key, val)\n        \nconfig = Config(\n    testing=False,\n    seed = 2019,\n    roberta_model_name='roberta-base', # can also be exchnaged with roberta-large \n    max_lr=1e-5,\n    epochs=1,\n    use_fp16=False,\n    bs=4, \n    max_seq_len=256, \n    num_labels = 2,\n    hidden_dropout_prob=.05,\n    hidden_size=768, # 1024 for roberta-large\n    start_tok = \"<s>\",\n    end_tok = \"</s>\",\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(path/\"jigsaw-toxic-comment-train.csv\", usecols=[1,2])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if config.testing: df = df[:5000]\nprint(df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feat_cols = \"comment_text\"\nlabel_cols = \"toxic\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FastAiRobertaTokenizer(BaseTokenizer):\n    \"\"\"Wrapper around RobertaTokenizer to be compatible with fastai\"\"\"\n    def __init__(self, tokenizer: RobertaTokenizer, max_seq_len: int=128, **kwargs): \n        self._pretrained_tokenizer = tokenizer\n        self.max_seq_len = max_seq_len \n    def __call__(self, *args, **kwargs): \n        return self \n    def tokenizer(self, t:str) -> List[str]: \n        \"\"\"Adds Roberta bos and eos tokens and limits the maximum sequence length\"\"\" \n        return [config.start_tok] + self._pretrained_tokenizer.tokenize(t)[:self.max_seq_len - 2] + [config.end_tok]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"roberta_tok = RobertaTokenizer.from_pretrained(\"roberta-base\")\n\nfastai_tokenizer = Tokenizer(tok_func=FastAiRobertaTokenizer(roberta_tok, max_seq_len=config.max_seq_len), \n                             pre_rules=[], post_rules=[])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = Path()\nroberta_tok.save_vocabulary(path)\n\nwith open('vocab.json', 'r') as f:\n    roberta_vocab_dict = json.load(f)\n    \nfastai_roberta_vocab = Vocab(list(roberta_vocab_dict.keys()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Setting up pre-processors\nclass RobertaTokenizeProcessor(TokenizeProcessor):\n    def __init__(self, tokenizer):\n         super().__init__(tokenizer=tokenizer, include_bos=False, include_eos=False)\n\nclass RobertaNumericalizeProcessor(NumericalizeProcessor):\n    def __init__(self, *args, **kwargs):\n        super().__init__(*args, vocab=fastai_roberta_vocab, **kwargs)\n\n\ndef get_roberta_processor(tokenizer:Tokenizer=None, vocab:Vocab=None):\n    \"\"\"\n    Constructing preprocessors for Roberta\n    We remove sos and eos tokens since we add that ourselves in the tokenizer.\n    We also use a custom vocabulary to match the numericalization with the original Roberta model.\n    \"\"\"\n    return [RobertaTokenizeProcessor(tokenizer=tokenizer), NumericalizeProcessor(vocab=vocab)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RobertaDataBunch(TextDataBunch):\n    \"Create a `TextDataBunch` suitable for training Roberta\"\n    @classmethod\n    def create(cls, train_ds, valid_ds, test_ds, path:PathOrStr='.', bs:int=64, val_bs:int=None, pad_idx=1,\n               pad_first=True, device:torch.device=None, no_check:bool=False, backwards:bool=False, \n               dl_tfms:Optional[Collection[Callable]]=None, **dl_kwargs) -> DataBunch:\n        \"Function that transform the `datasets` in a `DataBunch` for classification. Passes `**dl_kwargs` on to `DataLoader()`\"\n        datasets = cls._init_ds(train_ds, valid_ds, test_ds)\n        val_bs = ifnone(val_bs, bs)\n        collate_fn = partial(pad_collate, pad_idx=pad_idx, pad_first=pad_first, backwards=backwards)\n        train_sampler = SortishSampler(datasets[0].x, key=lambda t: len(datasets[0][t][0].data), bs=bs)\n        train_dl = DataLoader(datasets[0], batch_size=bs, sampler=train_sampler, drop_last=True, **dl_kwargs)\n        dataloaders = [train_dl]\n        for ds in datasets[1:]:\n            lengths = [len(t) for t in ds.x.items]\n            sampler = SortSampler(ds.x, key=lengths.__getitem__)\n            dataloaders.append(DataLoader(ds, batch_size=val_bs, sampler=sampler, **dl_kwargs))\n        return cls(*dataloaders, path=path, device=device, dl_tfms=dl_tfms, collate_fn=collate_fn, no_check=no_check)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RobertaTextList(TextList):\n    _bunch = RobertaDataBunch\n    _label_cls = TextList","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = Path('/kaggle/working/data/')\ntest = pd.read_csv(path/\"test.csv\")\ntest_datalist = TextList.from_df(test, cols='content')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# loading the tokenizer and vocab processors\nprocessor = get_roberta_processor(tokenizer=fastai_tokenizer, vocab=fastai_roberta_vocab)\n\n# creating our databunch \ndata = RobertaTextList.from_df(df, \".\", cols=feat_cols, processor=processor) \\\n    .split_by_rand_pct(seed=config.seed) \\\n    .label_from_df(cols=label_cols,label_cls=CategoryList) \\\n    .add_test(test_datalist) \\\n    .databunch(bs=config.bs, pad_first=False, pad_idx=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom pytorch_transformers import RobertaModel\n\n# defining our model architecture \nclass CustomRobertaModel(nn.Module):\n    def __init__(self,num_labels=2):\n        super(CustomRobertaModel,self).__init__()\n        self.num_labels = num_labels\n        self.roberta = RobertaModel.from_pretrained(config.roberta_model_name)\n        self.dropout = nn.Dropout(config.hidden_dropout_prob)\n        self.classifier = nn.Linear(config.hidden_size, num_labels) # defining final output layer\n        \n    def forward(self, input_ids, token_type_ids=None, attention_mask=None, labels=None):\n        _ , pooled_output = self.roberta(input_ids, token_type_ids, attention_mask) # \n        logits = self.classifier(pooled_output)        \n        return logits","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"roberta_model = CustomRobertaModel()\n\nlearn = Learner(data, roberta_model, metrics=[accuracy])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.model.roberta.train() # setting roberta to train as it is in eval mode by default\nlearn.fit_one_cycle(config.epochs, max_lr=config.max_lr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_preds_as_nparray(ds_type) -> np.ndarray:\n    learn.model.roberta.eval()\n    preds = learn.get_preds(ds_type)[0].detach().cpu().numpy()\n    sampler = [i for i in data.dl(ds_type).sampler]\n    reverse_sampler = np.argsort(sampler)\n    ordered_preds = preds[reverse_sampler, :]\n    pred_values = np.argmax(ordered_preds, axis=1)\n    return ordered_preds, pred_values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds, pred_values = get_preds_as_nparray(DatasetType.Test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"res =[preds1[1] for preds1 in preds]\n#res","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_id = test['id']\nlabel_cols = ['toxic']\n\nsubmission = pd.DataFrame({'id': test_id})\n#submission = pd.DataFrame(pred_values, columns = label_cols)\nsubmission = pd.concat([submission, pd.DataFrame(res, columns = label_cols)], axis=1)\n#header=[\"id\",\"toxic\"]\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}