{"cells":[{"metadata":{},"cell_type":"markdown","source":"**Update**:\n1. Use cached embedding\n2. Load only selected columns of train\n3. Decrease comsumed RAM (use numerized text)\n4. DataFrameDataset for torchtext"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \n\nimport os, re, gc, random, tqdm\n\nfrom nltk.tokenize import TweetTokenizer\nfrom collections import Counter\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\n\nimport torchtext\nfrom torchtext import vocab, data\n\nimport warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f9ae58b80e7a871ca069fcefdce83d24c43948e5","trusted":true},"cell_type":"code","source":"vocab.tqdm = tqdm.tqdm_notebook # Replace tqdm to tqdm_notebook in module torchtext","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b7c04a817b5ba68498dc9b30638605da891ba6c4","trusted":true},"cell_type":"code","source":"path = \"../input/jigsaw-unintended-bias-in-toxicity-classification/\"\nemb_path = \"../input/embeddings-glove-crawl-torch-cached\"\nn_folds = 5\ndevice = 'cuda'\ngc.enable();","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6ddd98901408dbe7c7fb9efdb0ec17cecd511864","trusted":true},"cell_type":"code","source":"# seed\nseed = 7777\nrandom.seed(seed)\nos.environ['PYTHONHASHSEED'] = str(seed)\nnp.random.seed(seed)\ntorch.manual_seed(seed)\ntorch.cuda.manual_seed(seed)\ntorch.cuda.manual_seed_all(seed)\ntorch.backends.cudnn.deterministic = True","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"461f84f65cf5452f7d47cd07d4e7da984529c5bf","trusted":true},"cell_type":"code","source":"class RegExCleaner():\n\n    def __init__(self, expressions=[]):\n        r\"\"\"Create class from compiled expressions: [(re.compile(pattern), repl)]\"\"\"\n        self.expressions = expressions\n\n    @staticmethod\n    def _compile(expressions):\n        regexps = []\n        for pattern, repl in expressions.items():\n            regexps.append((re.compile(pattern), repl))\n        return regexps\n        \n    @classmethod\n    def from_dict(cls, custom_dic):\n        r\"\"\"Create class from dictionary with flexible patterns {pattern : replacing}\"\"\"\n        return cls(cls._compile(custom_dic))\n    \n    @classmethod\n    def from_vocab(cls, vocab):\n        r\"\"\"Create class from vocabulary with fixed patterns {pattern : replacing}\"\"\"\n        pattern = re.compile(\"|\".join(map(re.escape, vocab.keys())))\n        repl = lambda match: vocab[match.group(0)]\n        return cls([(pattern, repl)])\n    \n    def __add__(self, b):\n        return RegExCleaner(self.expressions + b.expressions)\n\n    def __call__(self, s):\n        for regex, repl in self.expressions:\n            s = regex.sub(repl, s) \n        return s\n\ntknzr = TweetTokenizer(strip_handles=True, reduce_len=True)\ncleaner = RegExCleaner.from_dict({r'https?:/\\/\\S+':r' ',\n                                  r'[^A-Za-z0-9!.,?$\\'\\\"]+':r' '})\ndef preparation(s): \n    s = cleaner(s)\n    return ' '.join(tknzr.tokenize(s))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Thanks sakami\n# https://www.kaggle.com/c/jigsaw-unintended-bias-in-toxicity-classification/discussion/90527\n\nclass JigsawEvaluator:\n\n    def __init__(self, y_true, y_identity, power=-5, overall_model_weight=0.25):\n        self.y = (y_true >= 0.5).astype(int)\n        self.y_i = (y_identity >= 0.5).astype(int)\n        self.n_subgroups = self.y_i.shape[1]\n        self.power = power\n        self.overall_model_weight = overall_model_weight\n\n    @staticmethod\n    def _compute_auc(y_true, y_pred):\n        try:\n            return roc_auc_score(y_true, y_pred)\n        except ValueError:\n            return np.nan\n\n    def _compute_subgroup_auc(self, i, y_pred):\n        mask = self.y_i[:, i] == 1\n        return self._compute_auc(self.y[mask], y_pred[mask])\n\n    def _compute_bpsn_auc(self, i, y_pred):\n        mask = self.y_i[:, i] + self.y == 1\n        return self._compute_auc(self.y[mask], y_pred[mask])\n\n    def _compute_bnsp_auc(self, i, y_pred):\n        mask = self.y_i[:, i] + self.y != 1\n        return self._compute_auc(self.y[mask], y_pred[mask])\n\n    def compute_bias_metrics_for_model(self, y_pred):\n        records = np.zeros((3, self.n_subgroups))\n        for i in range(self.n_subgroups):\n            records[0, i] = self._compute_subgroup_auc(i, y_pred)\n            records[1, i] = self._compute_bpsn_auc(i, y_pred)\n            records[2, i] = self._compute_bnsp_auc(i, y_pred)\n        return records\n\n    def _calculate_overall_auc(self, y_pred):\n        return roc_auc_score(self.y, y_pred)\n\n    def _power_mean(self, array):\n        total = sum(np.power(array, self.power))\n        return np.power(total / len(array), 1 / self.power)\n\n    def get_final_metric(self, y_pred):\n        bias_metrics = self.compute_bias_metrics_for_model(y_pred)\n        bias_score = np.average([\n            self._power_mean(bias_metrics[0]),\n            self._power_mean(bias_metrics[1]),\n            self._power_mean(bias_metrics[2])\n        ])\n        overall_score = self.overall_model_weight * self._calculate_overall_auc(y_pred)\n        bias_score = (1 - self.overall_model_weight) * bias_score\n        return overall_score + bias_score\n    \nidentity_columns = [\n    'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',\n    'muslim', 'black', 'white', 'psychiatric_or_mental_illness']\n\ndef as_torch(x):\n    return torch.FloatTensor([float(x)])\n\n# Class for split train on folds\nclass PartialSet():\n    def __init__(self, ds, indices):\n        self.ds = ds\n        self.idx = indices\n        self.len = len(indices)\n        self.fields = self.ds.fields\n        \n    def update(self, idx):\n        self.idx = idx\n        self.len = len(idx)\n        \n    def __getitem__(self, idx):\n        return self.ds[self.idx[idx]]\n    \n    def __len__(self):\n        return self.len\n    \ndef sigmoid(x):\n    return 1/(1+np.exp(-x))\n\n# DataFrameDataset for torchtext\nclass DataFrameDataset(data.Dataset):\n    def __init__(self, df, fields, is_test=False, **kwargs):\n        keys = dict(fields)\n        for n, f in list(keys.items()):\n            if isinstance(n, tuple):\n                keys.update(zip(n, f))\n                del keys[n]\n        keys = keys.keys()\n        examples = []\n        for i, row in tqdm.tqdm_notebook(df.iterrows(), total=len(df)):\n            examples.append(data.Example.fromlist([row[k] for k in keys], fields))\n\n        super().__init__(examples, fields, **kwargs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load csv file with selected columns\n\ncols = ['id', 'comment_text', 'target'] + identity_columns\ndtypes = {'target': np.float16,'comment_text': object,'id': np.int32}\nfor c in identity_columns:\n    dtypes[c] = np.float16\ndf = pd.read_csv(os.path.join(path, 'train.csv'), usecols=cols, dtype=dtypes, index_col=[0])\n\ny = df.target.values > 0.5\n\nskf = StratifiedKFold(n_splits = n_folds, shuffle = True, random_state = seed)\n# Test csv\ntest_df = pd.read_csv(os.path.join(path, 'test.csv'),\n                dtype={'comment_text': object,'id': np.int32}, index_col=[0])\n\ntest_df['prediction'] = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# function for transform sentence to sequence of encoded words\ndef sentence2numbers(s, fill_as=1): # fill <unk> token\n    seq = []\n    for w in s:\n        try:\n            seq.append(vocabulary.stoi[w])\n        except KeyError:\n            seq.append(fill_as)\n    return np.array(seq, dtype=np.int32) ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9640b7fef754155e0162b2cd1e70cadf94c6ec6c","trusted":true},"cell_type":"code","source":"# define the columns that we want to process and how to process\npad_token = 0 # '<pad>' position token\n\ntxt_field = data.Field(sequential=True, preprocessing=sentence2numbers,\n                       pad_token=pad_token, use_vocab=False)\nnum_field = data.Field(sequential=False, dtype=torch.float,  use_vocab=False)\nidx_field = data.Field(sequential=False, dtype=torch.int64,  use_vocab=False)\n\ntrain_fields = [\n    ('id', idx_field), \n    ('target', num_field), \n    ('comment_text', txt_field),\n]\ntest_fields = [\n    ('id', idx_field), \n    ('comment_text', txt_field), \n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm._tqdm_notebook import tqdm_notebook as tqdm_pandas\ntqdm_pandas.pandas()\n# prepare text field\ndf.comment_text = df.comment_text.progress_apply(preparation)\ntest_df.comment_text = test_df.comment_text.progress_apply(preparation)\n\n# count unique words\ncounter = Counter()\nfor comment in df.comment_text:\n    counter.update(comment.split())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create vocabulary from glove cache\nvec = vocab.Vectors(os.path.join(emb_path, 'glove.840B.300d.txt'), cache=emb_path)\nvocabulary = vocab.Vocab(counter, max_size=500000, vectors=vec, specials=['<pad>', '<unk>'])\ntorch.zero_(vocabulary.vectors[1]); # fill <unk> token as 0\n\ndel vec\ngc.collect();\nprint('Embedding vocab size: ', vocabulary.vectors.size(0))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"92ea577a18faedfdccf5198d626c5c27861133ee","trusted":true},"cell_type":"code","source":"# create datasets\nds = DataFrameDataset(df.reset_index(), train_fields)\ntest_ds = DataFrameDataset(test_df.reset_index(), test_fields)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# wrapper for loaders\nclass BatchWrapper:\n    def __init__(self, dl, mode='train'):\n        self.dl, self.mode = dl, mode\n    def __iter__(self):\n        if self.mode !='test':\n            for batch in self.dl:\n                yield (batch.comment_text, batch.target, batch.id)\n        else:\n            for batch in self.dl:\n                yield (batch.comment_text, batch.id)  \n    def __len__(self):\n            return len(self.dl)\n\ndef wrapper(ds, mode='train', **args):\n    dl = data.BucketIterator(ds, **args)\n    return BatchWrapper(dl, mode)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8e5feaed1764517dbb5e51c21f09c939dbb77704","trusted":true},"cell_type":"code","source":"tloader = wrapper(test_ds, mode='test', batch_size=512, device='cuda',\n               sort_key=lambda x: len(x.comment_text),\n               sort_within_batch=True, shuffle=False, repeat=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e64a4a40f2e23749d937004f00db7210cdd3d946","scrolled":true,"trusted":true},"cell_type":"code","source":"class BaseModule(nn.Module):\n    @torch.no_grad()\n    def prediction(self, x):\n        score = self.forward(x)\n        return torch.sigmoid(score)\n    \n    @torch.no_grad()\n    def evaluate(self, x, y, func):\n        preds = self.forward(x)\n        loss = func(preds.squeeze(-1), y)\n        return preds, loss\n    \nclass RecNN(BaseModule):\n    def __init__(self, embs_vocab, hidden_size, layers=1, dropout=0., bidirectional=False):\n        super().__init__()\n\n        self.hidden_size = hidden_size\n        self.bidirectional = bidirectional\n        self.num_layers = layers\n        self.emb = nn.Embedding.from_pretrained(embs_vocab)\n        \n        self.line = nn.Linear(embs_vocab.size(1), embs_vocab.size(1))\n        \n        self.lstm = nn.LSTM(embs_vocab.size(1), self.hidden_size,\n                            num_layers=layers, bidirectional=bidirectional, dropout=dropout)\n        \n        self.gru = nn.GRU(embs_vocab.size(1), self.hidden_size,\n                            num_layers=layers, bidirectional=bidirectional, dropout=dropout)\n        \n        self.out = nn.Linear(self.hidden_size*(bidirectional + 1), 32)\n        self.last = nn.Linear(32, 1)\n                \n    def forward(self, x):\n        \n        embs = self.emb(x)\n        lstm, (h, c) = self.lstm(embs)\n        \n        x = F.relu(self.line(embs), inplace=True)\n        gru, h = self.gru(x, h)\n        lstm = lstm + gru\n        \n        lstm, _ = lstm.max(dim=0, keepdim=False) \n        out = self.out(lstm)\n        out = self.last(F.relu(out)).squeeze()\n        return out.squeeze(-1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e566ecdb5f44cf32af12127f073021566d6e66dd","trusted":true},"cell_type":"code","source":"epochs = 4\nloss_fn = torch.nn.BCEWithLogitsLoss(reduction='mean', pos_weight=(torch.Tensor([2.7])).to(device))\nbs = 512\nbidirectional=True\nn_hidden = 64\n\ntrain_ds = PartialSet(ds, [0])\nvalid_ds = PartialSet(ds, [0])\n    \nfor train_idx, valid_idx in skf.split(y, y=y):\n    \n    train_ds.update(train_idx)\n    valid_ds.update(valid_idx)\n    \n    loader = wrapper(train_ds, batch_size=bs, device=device,\n                    sort_key=lambda x: len(x.comment_text),\n                    sort_within_batch=True, shuffle=True, repeat=False)\n    \n    vloader = wrapper(valid_ds, batch_size=bs, device=device,\n                    sort_key=lambda x: len(x.comment_text),\n                    sort_within_batch=True, shuffle=False, repeat=False)\n    \n    model = RecNN(vocabulary.vectors, n_hidden, layers=2, dropout=0.2, bidirectional=bidirectional).to(device)\n\n    opt = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), 1e-3,\n                     betas=(0.75, 0.999), eps=1e-08, weight_decay=0)\n\n    print('\\n')\n    for epoch in range(epochs):      \n        y_true_train = np.empty(0)\n        y_pred_train = np.empty(0)\n        total_loss_train, total_loss_valid = 0, 0          \n        model.train()\n        tcids=[]\n        vcids=[]\n        for x, target, ids in loader:\n            tcids.append(ids.detach().cpu().numpy())\n            opt.zero_grad()\n            pred = model(x)\n            loss = loss_fn(pred, target)\n            loss.backward()\n            opt.step()\n            \n            y_true_train = np.concatenate([y_true_train, target.detach().cpu().numpy()], axis = 0)\n            y_pred_train = np.concatenate([y_pred_train, pred.detach().cpu().numpy()], axis = 0)\n            total_loss_train += loss.item()\n\n        # Get prediction for validation part\n        model.eval()\n        y_true_valid = np.empty(0)\n        y_pred_valid = np.empty(0)\n        \n        for x, target, ids in vloader:\n            vcids.append(ids.detach().cpu().numpy())\n            pred, loss = model.evaluate(x, target, loss_fn)\n            total_loss_valid += loss.item()\n            \n            y_true_valid = np.concatenate([y_true_valid, target.detach().cpu().data.numpy()], axis = 0)\n            y_pred_valid = np.concatenate([y_pred_valid, pred.cpu().data.numpy()], axis = 0)\n            \n        tcids = [item for sublist in tcids for item in sublist]\n        vcids = [item for sublist in vcids for item in sublist]\n        \n        vloss = total_loss_valid/len(vloader)\n        tloss = total_loss_train/len(loader)\n        \n        scorer = JigsawEvaluator(y_true_train, df.loc[tcids][identity_columns].values)\n        tacc = scorer.get_final_metric(sigmoid(y_pred_train))\n\n        scorer = JigsawEvaluator(y_true_valid, df.loc[vcids][identity_columns].values)\n        vacc = scorer.get_final_metric(sigmoid(y_pred_valid))\n        \n        print(f'Epoch {epoch+1}: Train loss: {tloss:.4f}, BIAS AUC: {tacc:.4f}, Valid loss: {vloss:.4f}, BIAS AUC: {vacc:.4f}')\n\n    gc.collect();\n    # Get prediction for test set\n    preds = np.empty(0)\n    cids = []\n    for x, ids in tloader:\n        cids.append(ids.detach().cpu().numpy())\n        pred = model.prediction(x)\n        preds = np.concatenate([preds, pred.detach().cpu().numpy()], axis = 0)\n\n    # Save prediction of test to DataFrame\n    cids = [item for sublist in cids for item in sublist]\n    test_df.at[cids, 'prediction']  =  test_df.loc[cids]['prediction'].values + preds/n_folds","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7c6a41bd21bf783455fd11b63745ed454e0413f0","trusted":true},"cell_type":"code","source":"test_df.to_csv('submission.csv', columns=['prediction'])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.1"}},"nbformat":4,"nbformat_minor":1}