{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nfrom pathlib import Path\nfrom typing import *\n\nimport torch\nimport torch.optim as optim\n\nfrom fastai import *\nfrom fastai.vision import *\nfrom fastai.text import *\nfrom fastai.callbacks import *\nfrom pytorch_pretrained_bert import BertTokenizer\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Config(dict):\n    def __init__(self, **kwargs):\n        super().__init__(**kwargs)\n        for k, v in kwargs.items():\n            setattr(self, k, v)\n    \n    def set(self, key, val):\n        self[key] = val\n        setattr(self, key, val)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"config = Config(\n    testing=False,\n    #bert_model_name=\"bert-base-uncased\",\n    bert_model_name=\"bert-base-multilingual-uncased\",\n    \n    max_lr=3e-5,\n    epochs=2,                   #4,\n    use_fp16=True,\n    bs=64,                      #32,\n    discriminative=False,\n    max_seq_len=192            #256\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FastAiBertTokenizer(BaseTokenizer):\n    \"\"\"Wrapper around BertTokenizer to be compatible with fast.ai\"\"\"\n    def __init__(self, tokenizer: BertTokenizer, max_seq_len: int=128, **kwargs):\n        self._pretrained_tokenizer = tokenizer\n        self.max_seq_len = max_seq_len\n\n    def __call__(self, *args, **kwargs):\n        return self\n\n    def tokenizer(self, t:str) -> List[str]:\n        \"\"\"Limits the maximum sequence length\"\"\"\n        return [\"[CLS]\"] + self._pretrained_tokenizer.tokenize(t)[:self.max_seq_len - 2] + [\"[SEP]\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if config.testing:\n    train = train.head(1024)\n    val = val.head(1024)\n    test = test.head(1024)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_ROOT = Path(\"..\")/\"input\"/ \"jigsaw-multilingual-toxic-comment-classification/\"\n\ndf1,df2,df3,test,sample = [pd.read_csv(DATA_ROOT / fname) for fname in [\"jigsaw-toxic-comment-train.csv\",\n                                                                        \"jigsaw-unintended-bias-train.csv\",\n                                                                        \"validation.csv\",\n                                                                        \"test.csv\",\n                                                                        \"sample_submission.csv\"\n                                                                       ]]\ndf2.toxic = df2.toxic.round().astype(int)\ntrain = pd.concat([\n    df1[['comment_text', 'toxic']],\n    df2[['comment_text', 'toxic']].query('toxic==1'),\n    df2[['comment_text', 'toxic']].query('toxic==0').sample(n=90000, random_state=0)\n])\n\n# rankings_pd.rename(columns = {'test':'TEST', 'odi':'ODI', \n#                               't20':'T20'}, inplace = True) \ntest.rename(columns={\"content\":\"comment_text\"}, inplace = True)\n\nval = df3\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bert_tok = BertTokenizer.from_pretrained(\n    config.bert_model_name,\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fastai_bert_vocab = Vocab(list(bert_tok.vocab.keys()))\nfastai_tokenizer = Tokenizer(tok_func=FastAiBertTokenizer(bert_tok, \n                                                          max_seq_len=config.max_seq_len), pre_rules=[], post_rules=[])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"databunch = TextDataBunch.from_df(\".\", train, val, test,\n                  tokenizer=fastai_tokenizer,\n                  vocab=fastai_bert_vocab,\n                  include_bos=False,\n                  include_eos=False,\n                  text_cols=\"comment_text\",\n                  label_cols=\"toxic\",\n                  bs=config.bs,\n                  collate_fn=partial(pad_collate, pad_first=False, pad_idx=0),\n             )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"databunch.show_batch()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Load the DataBunch**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# databunch = load_data(path=\"../input/jigsawprocesseddatabunch/\", file = Path(\"data-jigsaw.pkl\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# databunch.show_batch()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# databunch.device","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# VERSION = \"20200325\"  #@param [\"1.5\" , \"20200325\", \"nightly\"]\n# !curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n# !python pytorch-xla-env-setup.py --version $VERSION","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import warnings\n# import torch_xla\n# import torch_xla.debug.metrics as met\n# import torch_xla.distributed.data_parallel as dp\n# import torch_xla.distributed.parallel_loader as pl\n# import torch_xla.utils.utils as xu\n# import torch_xla.core.xla_model as xm\n# import torch_xla.distributed.xla_multiprocessing as xmp\n# import torch_xla.test.test_utils as test_utils","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# device = xm.xla_device()\n# model = mx.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from pytorch_pretrained_bert.modeling import BertConfig, BertForSequenceClassification\nbert_model = BertForSequenceClassification.from_pretrained(config.bert_model_name, num_labels=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Loss_fn(nn.BCEWithLogitsLoss):\n  __constants__ = ['weight', 'pos_weight', 'reduction']\n  \n  def __init__(self, weight=None, size_average=None, reduce=None, reduction='mean', pos_weight=None):\n    \n    super().__init__(size_average, reduce, reduction)\n    self.register_buffer('weight', weight)\n    self.register_buffer('pos_weight', pos_weight)\n\n  def forward(self, input, target):\n    # My target is of torch.Size([32])\n    target = target.unsqueeze(1)   # Convert target size  of torch.Size([32, 1])\n    target = target.float()        # BCE loss expects a Tensor of type float\n  \n    return F.binary_cross_entropy_with_logits(input, target,\n                                                  self.weight,\n                                                  pos_weight=self.pos_weight,\n                                                  reduction=self.reduction)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"loss_func = Loss_fn()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from fastai.callbacks import *\n\nlearner = Learner(\n    databunch, bert_model,\n    loss_func=loss_func,\n)\nif config.use_fp16: learner = learner.to_fp16()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learner.model_dir = '/tmp/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learner.lr_find()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learner.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learner.fit_one_cycle(config.epochs, max_lr=config.max_lr)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_preds_as_nparray(ds_type) -> np.ndarray:\n    \"\"\"\n    the get_preds method does not yield the elements in order by default\n    we borrow the code from the RNNLearner to resort the elements into their correct order\n    \"\"\"\n    preds = learner.get_preds(ds_type)[0].detach().cpu().numpy()\n    sampler = [i for i in databunch.dl(ds_type).sampler]\n    reverse_sampler = np.argsort(sampler)\n    return preds[reverse_sampler, :]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds = get_preds_as_nparray(DatasetType.Test)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv\")\nif config.testing: sample_submission = sample_submission.head(test.shape[0])\nsample_submission['toxic'] = test_preds\nsample_submission.to_csv(\"predictions.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}