{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Introduction\n\nIs is based on this kernel, https://www.kaggle.com/theoviel/bert-pytorch-huggingface-starter\n\nThere is a huge class imbalance.\n\nTry to do under-sampling."},{"metadata":{},"cell_type":"markdown","source":"### TODO :\n- DistillBERT"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import gc\nimport os\nimport time\nimport math\nimport random\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom datetime import date\nfrom transformers import *\nfrom sklearn.metrics import *\nfrom tqdm.notebook import tqdm\n\nimport torch\nimport torch.nn as nn\nimport torch.utils.data\nimport torch.nn.functional as F\n\nfrom torch import Tensor\nfrom torch.optim import *\nfrom torch.nn.modules.loss import *\nfrom torch.optim.lr_scheduler import * \nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.utils.data.sampler import RandomSampler","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    \"\"\"\n    Seeds basic parameters for reproductibility of results\n    \n    Arguments:\n        seed {int} -- Number of the seed\n    \"\"\"\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed = 2020\nseed_everything(seed)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"MODEL_PATHS = {\n    'bert-multi-cased': '../input/bertconfigs/multi_cased_L-12_H-768_A-12/multi_cased_L-12_H-768_A-12/',\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_PATH = '../input/jigsaw-multilingual-toxic-comment-classification/'\n\ndf_val = pd.read_csv(DATA_PATH + 'validation-processed-seqlen128.csv')\ndf_test =  pd.read_csv(DATA_PATH + 'test-processed-seqlen128.csv')\ndf_train = pd.read_csv(DATA_PATH + 'jigsaw-toxic-comment-train-processed-seqlen128.csv')\ndf_train_2 = pd.read_csv(DATA_PATH + 'jigsaw-unintended-bias-train-processed-seqlen128.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nprint('mean of severe_toxicity', np.mean(df_train_2[df_train_2['severe_toxicity']!=0]['severe_toxicity'].values))\nprint('mean of obscene', np.mean(df_train_2[df_train_2['obscene']!=0]['obscene'].values))\nprint('mean of identity_attack', np.mean(df_train_2[df_train_2['identity_attack']!=0]['identity_attack'].values))\nprint('mean of insult', np.mean(df_train_2[df_train_2['insult']!=0]['insult'].values))\nprint('mean of threat', np.mean(df_train_2[df_train_2['threat']!=0]['threat'].values))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(df_train_2[df_train_2['severe_toxicity']>0.5]))\nprint(len(df_train_2[df_train_2['obscene']>0.8]))\nprint(len(df_train_2[df_train_2['identity_attack']>0.7]))\nprint(len(df_train_2[df_train_2['insult']>0.9]))\nprint(len(df_train_2[df_train_2['threat']>0.7]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(df_train_2[df_train_2['toxic']>0]['toxic'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_2['toxic'] = df_train_2['toxic'].fillna(0).round().astype(int)\ndf_train_2['severe_toxic'] = df_train_2['severe_toxicity'].fillna(0).round().astype(int)\ndf_train_2['obscene'] = df_train_2['obscene'].fillna(0).round().astype(int)\ndf_train_2['identity_hate'] = df_train_2['identity_attack'].fillna(0).round().astype(int)\ndf_train_2['insult'] = df_train_2['insult'].fillna(0).round().astype(int)\ndf_train_2['threat'] = df_train_2['threat'].fillna(0).round().astype(int)\ndf_train_2.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\n# for i, row in df_train_2.iterrows():\n#     if row['toxic'] < 0.6:\n#         df_train_2.at[i,'toxic'] = 0\n#         continue\n#     df_train_2.at[i,'toxic'] = 1\n#     if row['severe_toxicity']>0.5:\n#         df_train_2.at[i,'severe_toxic'] = 1\n#     else:\n#         df_train_2.at[i,'severe_toxic'] = 0\n#     if row['obscene']>0.5:\n#         df_train_2.at[i,'obscene'] = 1\n#     else:\n#         df_train_2.at[i,'obscene'] = 0\n#     if row['identity_attack']>0.5:\n#         df_train_2.at[i,'identity_hate'] = 1\n#     else:\n#         df_train_2.at[i,'identity_hate'] = 0\n#     if row['insult']>0.5:\n#         df_train_2.at[i,'insult'] = 1\n#     else:\n#         df_train_2.at[i,'insult'] = 0\n#     if row['threat']>0.5:\n#         df_train_2.at[i,'threat'] = 1\n#     else:\n#         df_train_2.at[i,'threat'] = 0\n        \n# df_train_2['toxic'].astype(int)\n# df_train_2['severe_toxic'].astype(int)\n# df_train_2['obscene'].astype(int)\n# df_train_2['identity_hate'].astype(int)\n# df_train_2['insult'].astype(int)\n# df_train_2['threat'].astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_2 = df_train_2[df_train.columns]\ndf_train_2.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(df_train['toxic'])\nplt.title('Target repartition on training data')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Under-Sampling"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.toxic.value_counts()[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"count_least = min(df_train.severe_toxic.value_counts()[1],\n                  df_train.obscene.value_counts()[1],\n                  df_train.threat.value_counts()[1],\n                  df_train.insult.value_counts()[1],\n                  df_train.identity_hate.value_counts()[1])\nprint(count_least)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_types = []\nfor toxic_type in ['severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']:\n    df_class_0 = df_train[df_train[toxic_type] == 0]\n    df_class_1 = df_train[df_train[toxic_type] == 1]\n\n    df_class_0_under = df_class_0.sample(count_least)\n    df_class_1_under = df_class_1.sample(count_least)\n    \n    df_train_types.append(df_class_0_under)\n    df_train_types.append(df_class_1_under)\n        \ndf_train_under = pd.concat(df_train_types, axis=0)\nprint(df_train_under.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(df_train_under['toxic'])\nplt.title('Target repartition on training data')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(df_train_under['severe_toxic'])\nplt.title('Target repartition on training data')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(df_train_under['obscene'])\nplt.title('Target repartition on training data')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(df_train_under['threat'])\nplt.title('Target repartition on training data')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(df_train_under['insult'])\nplt.title('Target repartition on training data')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(df_train_under['identity_hate'])\nplt.title('Target repartition on training data')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Add df_2"},{"metadata":{"trusted":true},"cell_type":"code","source":"count_0 = df_train_under.toxic.value_counts()[0]\ncount_1 = df_train_under.toxic.value_counts()[1]\n\ndf_class_0 = df_train_2[df_train_2['toxic'] == 0]\ndf_class_1 = df_train_2[df_train_2['toxic'] == 1]\n\ndf_class_0_under = df_class_0.sample(count_1)\ndf_class_1_under = df_class_1.sample(count_0)\n\ndf_train_under = pd.concat([df_train_under, df_class_0_under, df_class_1_under], axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(df_train_under['toxic'])\nplt.title('Target repartition on training data')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class JigsawDataset(Dataset):\n    \"\"\"\n    Torch dataset for training and validating\n    \"\"\"\n    def __init__(self, df):\n        super().__init__()\n        self.df = df \n        self.word_ids = np.array([word_ids[1:-1].split(', ') for word_ids in df['input_word_ids']]).astype(int)\n        \n        try:\n            self.y = df['toxic'].values\n        except KeyError: # test data\n            self.y = np.zeros(len(df))\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        return torch.tensor(self.word_ids[idx]), torch.tensor(self.y[idx])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"TRANSFORMERS = {\n    \"bert-multi-cased\": (BertModel, BertTokenizer, \"bert-base-cased\"),\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_class, tokenizer_class, pretrained_weights = TRANSFORMERS[\"bert-multi-cased\"]\nbert_config = BertConfig.from_json_file(MODEL_PATHS[\"bert-multi-cased\"] + 'bert_config.json')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Transformer(nn.Module):\n    def __init__(self, model, num_classes=1):\n        \"\"\"\n        Constructor\n        \n        Arguments:\n            model {string} -- Transformer to build the model on. Expects \"camembert-base\".\n            num_classes {int} -- Number of classes (default: {1})\n        \"\"\"\n        super().__init__()\n        self.name = model\n\n        model_class, tokenizer_class, pretrained_weights = TRANSFORMERS[model]\n\n        bert_config = BertConfig.from_json_file(MODEL_PATHS[model] + 'bert_config.json')\n        bert_config.output_hidden_states = True\n        \n        bert_config.max_position_embeddings = 128\n        bert_config.hidden_dropout_prob = 0.0\n        \n        \n        self.transformer = BertModel(bert_config)\n\n        self.nb_features = self.transformer.pooler.dense.out_features\n\n        self.pooler = nn.Sequential(\n            nn.Linear(self.nb_features, self.nb_features), \n            nn.Tanh(),\n        )\n\n        self.logit = nn.Linear(self.nb_features, num_classes)\n\n    def forward(self, tokens):\n        \"\"\"\n        Usual torch forward function\n        \n        Arguments:\n            tokens {torch tensor} -- Sentence tokens\n        \n        Returns:\n            torch tensor -- Class logits\n        \"\"\"\n        _, _, hidden_states = self.transformer(\n            tokens, attention_mask=(tokens > 0).long()\n        )\n\n        hidden_states = hidden_states[-1][:, 0] # Use the representation of the first token of the last layer\n\n        ft = self.pooler(hidden_states)\n\n        return self.logit(ft)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"def fit(model, train_dataset, val_dataset, epochs=1, batch_size=8, warmup_prop=0, lr=5e-4):\n    \n    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n    val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)\n\n    optimizer = AdamW(model.parameters(), lr=lr)\n    \n    num_warmup_steps = int(warmup_prop * epochs * len(train_loader))\n    num_training_steps = epochs * len(train_loader)\n    \n    scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps, num_training_steps)\n\n    loss_fct = nn.BCEWithLogitsLoss(reduction='mean').cuda()\n    \n    for epoch in range(epochs):\n        model.train()\n        start_time = time.time()\n        \n        optimizer.zero_grad()\n        avg_loss = 0\n        \n        for step, (x, y_batch) in enumerate(train_loader): \n            y_pred = model(x.cuda())\n            \n            loss = loss_fct(y_pred.view(-1).float(), y_batch.float().cuda())\n            loss.backward()\n            avg_loss += loss.item() / len(train_loader)\n\n            optimizer.step()\n            scheduler.step()\n            model.zero_grad()\n            optimizer.zero_grad()\n                \n        model.eval()\n        preds = []\n        truths = []\n        avg_val_loss = 0.\n\n        with torch.no_grad():\n            for x, y_batch in val_loader:                \n                y_pred = model(x.cuda())\n                loss = loss_fct(y_pred.detach().view(-1).float(), y_batch.float().cuda())\n                avg_val_loss += loss.item() / len(val_loader)\n                \n                probs = torch.sigmoid(y_pred).detach().cpu().numpy()\n                preds += list(probs.flatten())\n                truths += list(y_batch.numpy().flatten())\n            score = roc_auc_score(truths, preds)\n            \n        \n        dt = time.time() - start_time\n        lr = scheduler.get_last_lr()[0]\n        print(f'Epoch {epoch + 1}/{epochs} \\t lr={lr:.1e} \\t t={dt:.0f}s \\t loss={avg_loss:.4f} \\t val_loss={avg_val_loss:.4f} \\t val_auc={score:.4f}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = 2\nbatch_size = 32\nwarmup_prop = 0.2 \nlr = 1e-4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Transformer(\"bert-multi-cased\").cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n = len(df_train_under)  # I do not train on the entier data as it will take too long (for now)\nprint('the length of train data is', n)\n#train_dataset = JigsawDataset(df_train_under.sample(n))\ntrain_dataset = JigsawDataset(df_train_under)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_dataset = JigsawDataset(df_val)\ntest_dataset = JigsawDataset(df_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fit(model, train_dataset, val_dataset, epochs=epochs, batch_size=batch_size, warmup_prop=warmup_prop, lr=lr)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predicting"},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict(model, dataset, batch_size=64):\n    \"\"\"\n    Usual predict torch function\n    \n    Arguments:\n        model {torch model} -- Model to predict with\n        dataset {torch dataset} -- Dataset to get predictions from\n    \n    Keyword Arguments:\n        batch_size {int} -- Batch size (default: {32})\n    \n    Returns:\n        numpy array -- Predictions\n    \"\"\"\n\n    model.eval()\n    preds = np.empty((0, 1))\n    loader = DataLoader(dataset, batch_size=batch_size, shuffle=False)\n\n    with torch.no_grad():\n        for x, _ in tqdm(loader):\n            probs = torch.sigmoid(model(x.cuda())).detach().cpu().numpy()\n            preds = np.concatenate([preds, probs])\n            \n    return preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_val = predict(model, val_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"score = roc_auc_score(df_val['toxic'], pred_val)\nprint(f'Scored {score:.4f} on validation data')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_test = predict(model, test_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv(DATA_PATH + 'sample_submission.csv')\nsub['toxic'] = pred_test\nsub.to_csv('submission.csv', index=False)\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}