{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Pytorch","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n!python pytorch-xla-env-setup.py --apt-packages libomp5 libopenblas-dev","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm\nimport random\nfrom sklearn.metrics import roc_auc_score\n\nimport torch\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data import Dataset\nimport transformers\nfrom transformers import BertForSequenceClassification, BertPreTrainedModel, BertConfig, BertModel\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nimport torch_xla\nimport torch_xla.core.xla_model as xm\n\nimport os\n\nbert = []\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        if (\"seqlen128\" in filename):\n            bert.append(filename)\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class config:\n    EPOCHS = 1\n    BATCH_SIZE = 32\n    VAL_BATCH_SIZE = 128\n    TEST_BATCH_SIZE = 128\n    LR = 3e-5","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"valid = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation-processed-seqlen128.csv\")\ntrain = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train-processed-seqlen128.csv\")\ntrain_bias = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train-processed-seqlen128.csv\")\ntest = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test-processed-seqlen128.csv\")\nsubmit = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv\")\nevalid = train[['id', 'comment_text', 'input_word_ids', 'input_mask','all_segment_id', 'toxic']].iloc[20001:25001]\ntrain = train[['id', 'comment_text', 'input_word_ids', 'input_mask','all_segment_id', 'toxic']].iloc[:20000]\ntrain_bias = train_bias[['id', 'comment_text', 'input_word_ids', 'input_mask','all_segment_id', 'toxic']].iloc[:20000]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"evalid = evalid.reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train = pd.concat([train[train[\"toxic\"] == 1], train[train[\"toxic\"] == 0].sample(n=21384, random_state=29)]).sample(frac=1).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 透過 info() 查看是否有 null 值\n- 數量稀少的話直接將 null drop ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":false,"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"train_distribution = train[\"toxic\"].value_counts().values\nvalid_distribution = valid[\"toxic\"].value_counts().values\n\nnon_toxic = [train_distribution[0] / sum(train_distribution) * 100, valid_distribution[0] / sum(valid_distribution) * 100]\ntoxic = [train_distribution[1] / sum(train_distribution) * 100, valid_distribution[1] / sum(valid_distribution) * 100]\n\nplt.figure(figsize=(9,6))\nplt.bar([0, 1], non_toxic, alpha=.4, color=\"r\", width=0.35, label=\"non-toxic\")\nplt.bar([0.4, 1.4], toxic, alpha=.4, width=0.35, label=\"toxic\")\nplt.xlabel(\"Dataset\")\nplt.ylabel(\"Percentage\")\nplt.xticks([0.2, 1.2], [\"train\", \"valid\"])\nplt.legend(loc=\"upper right\")\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"train: \\nnon-toxic rate: {train_distribution[0] / sum(train_distribution) * 100: .2f} %\\ntoxic rate: {train_distribution[1] / sum(train_distribution) * 100: .2f} %\")\nprint(f\"valid: \\nnon-toxic rate: {valid_distribution[0] / sum(valid_distribution) * 100: .2f} %\\ntoxic rate: {valid_distribution[1] / sum(valid_distribution) * 100: .2f} %\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lang = valid[\"lang\"].value_counts()\n\nplt.figure(figsize=(9, 6))\nplt.xlabel(\"Lang\")\nplt.ylabel(\"Num\")\nplt.xticks([0.2, 0.6, 1], [\"tr\", \"es\", \"it\"])\nplt.bar([0.2, 0.6, 1], lang, color=\"purple\", width=0.28, alpha=.4)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class TweetDataset(Dataset):\n    def __init__(self, mode, df):\n        self.mode = mode\n        self.df = df\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        token, segment, mask = self.df.loc[idx, [\"input_word_ids\", \"all_segment_id\", \"input_mask\"]].values\n        if self.mode==\"train\" or self.mode == \"valid\":\n            label_tensor = torch.tensor(self.df.loc[idx, \"toxic\"])\n        else:\n            label_tensor = torch.tensor(-1)\n        tokens_tensor = torch.tensor([int(i) for i in token[1:-1].split(\",\")])\n        segments_tensor = torch.tensor([int(i) for i in segment[1:-1].split(\",\")])\n        masks_tensor = torch.tensor([int(i) for i in mask[1:-1].split(\",\")])\n           \n        return tokens_tensor, segments_tensor, masks_tensor, label_tensor","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# DataLoader","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"lang = {'Spanish': 'es', 'Italian': 'it', 'Turkish': 'tr'}\n\nvalidsets = {}\nfor i, k in lang.items():\n    validsets[i] = TweetDataset(\"valid\", valid[valid[\"lang\"] == k].reset_index(drop=True))\ntrainset = TweetDataset(\"train\", train)\ntrainbias = TweetDataset(\"train\", train_bias)\nvalidset = TweetDataset(\"valid\", valid)\ntestset = TweetDataset(\"test\", test)\nevalidset = TweetDataset(\"valid\", evalid)\n\nvalidloaders = {}\nfor i, k in validsets.items():\n    validloaders[i] = DataLoader(k, batch_size=config.VAL_BATCH_SIZE, num_workers=4, shuffle=False)\ntrainloader = DataLoader(trainset, batch_size=config.BATCH_SIZE, num_workers=4, shuffle=False)\ntrainbiasloader = DataLoader(trainbias, batch_size=config.BATCH_SIZE, num_workers=4, shuffle=False)\nvalidloader = DataLoader(validset, batch_size=config.VAL_BATCH_SIZE, num_workers=4, shuffle=False)\ntestloader = DataLoader(testset, batch_size=config.TEST_BATCH_SIZE, num_workers=4, shuffle=False)\nevalidloader = DataLoader(evalidset, batch_size=config.VAL_BATCH_SIZE, num_workers=4, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Bert","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class Model(nn.Module):\n    \n    def __init__(self, labels=1):\n        \n        super().__init__()\n        \n        self.bert = BertModel.from_pretrained(\"bert-base-multilingual-cased\")\n        self.num_features = self.bert.pooler.dense.out_features\n        self.labels = labels\n        \n        self.drop = nn.Dropout(0.3)\n        self.fc1 = nn.Linear(self.num_features * 2, self.num_features)\n        self.logit = nn.Linear(self.num_features, self.labels)\n        \n    def forward(self, tokens_tensors, segments_tensors, masks_tensors):\n\n        hidden_states, cls = self.bert(input_ids=tokens_tensors, token_type_ids=segments_tensors, attention_mask=masks_tensors)\n        avgpool = torch.mean(hidden_states, 1)\n        maxpool, _ = torch.max(hidden_states, 1)\n        cat = torch.cat((avgpool, maxpool), 1)\n        x = self.drop(cat)\n        x = torch.tanh(self.fc1(x))\n        output = self.logit(x)\n\n        return output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\ndevice = xm.xla_device()\nmodel.to(device)\nprint(f\"Now we use {device}\\n\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def training(model, warmup_prop=0.1):\n\n    optimizer = torch.optim.AdamW(model.parameters(), lr=config.LR)\n    num_warmup_steps = int(warmup_prop * config.EPOCHS * len(trainloader))\n    num_training_steps = config.EPOCHS * len(trainloader)\n    scheduler = transformers.get_linear_schedule_with_warmup(optimizer, num_warmup_steps, num_training_steps)\n    loss_fun = torch.nn.BCEWithLogitsLoss(reduction='mean').to(device)    \n\n    for epoch in range(config.EPOCHS):\n        model.train()\n        \n        optimizer.zero_grad()\n        avg_loss = 0\n        \n        for data in tqdm(trainloader):             \n            tokens_tensor, segments_tensor, masks_tensor, labels_tensor = [k.to(device) for k in data if k is not None]\n            output = model(tokens_tensor, segments_tensor, masks_tensor)\n            loss = loss_fun(output.view(-1).float(), labels_tensor.float().to(device))\n            loss.backward()\n            avg_loss += loss.item() / len(trainloader)\n\n            xm.optimizer_step(optimizer, barrier=True)\n            scheduler.step()\n            model.zero_grad()\n            optimizer.zero_grad()\n                \n        model.eval()\n        preds = []\n        truths = []\n        avg_val_loss = 0.\n\n        with torch.no_grad():\n            for data in validloader:\n                tokens_tensor, segments_tensor, masks_tensor, labels_tensor = [k.to(device) for k in data if k is not None]\n                output = model(tokens_tensor, segments_tensor, masks_tensor)\n                loss = loss_fun(output.detach().view(-1).float(), labels_tensor.float().to(device))\n                avg_val_loss += loss.item() / len(validloader)\n                \n                probs = torch.sigmoid(output).detach().cpu().numpy()\n                preds += list(probs.flatten())\n                truths += list(labels_tensor.detach().cpu().numpy().flatten())\n            score = roc_auc_score(truths, preds)\n        \n        lr = scheduler.get_last_lr()[0]\n        print(f'[Epoch {epoch + 1}] lr={lr:.1e} loss={avg_loss:.4f} val_loss={avg_val_loss:.4f} val_auc={score:.4f}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"threshold = lambda x: 1 if x>=0.5 else 0\n\ndef predict(model, dataloader, df, isAccuracy=True):\n \n    model.eval().to(device)\n    preds = np.empty((0, 1))\n    accuracy = None\n\n    with torch.no_grad():\n        for data in tqdm(dataloader):\n            tokens_tensor, segments_tensor, masks_tensor, labels_tensor = [k.to(device) for k in data if k is not None]\n            probs = torch.sigmoid(model(tokens_tensor, segments_tensor, masks_tensor)).detach().cpu().numpy()\n            preds = np.concatenate([preds, probs])\n            \n    preds = preds.reshape(len(preds))        \n    predicts = np.array([threshold(i) for i in preds])\n    if isAccuracy:\n        accuracy = (df[\"toxic\"].values == predicts).sum() / len(df)\n\n    return preds, predicts, accuracy ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# before training model accuracy\npre, pre_class, accuracy = predict(model, trainloader, train)\nauc = roc_auc_score(train[\"toxic\"].values, pre_class)\nprint(\"Train: \")\nprint(f\"Model before fine-tune accuracy: {accuracy * 100:.3f}%\\nModel before fine-tune AUC: {auc:.3f}\")\n\nfor key, value in validloaders.items():\n    pre, pre_class, accuracy = predict(model, value, valid[valid[\"lang\"] == lang[key]].reset_index(drop=True))\n    auc = roc_auc_score(valid[valid[\"lang\"] == lang[key]].reset_index(drop=True)[\"toxic\"].values, pre_class)\n    print(f\"{key} Valid: \")\n    print(f\"Model before fine-tune accuracy: {accuracy * 100:.2f}%\\nModel before fine-tune AUC: {auc:.3f}\")\n\npre, pre_class, accuracy = predict(model, validloader, valid)\nauc = roc_auc_score(valid[\"toxic\"].values, pre_class)\nprint(f\"Combined Valid: \")\nprint(f\"Model before fine-tune accuracy: {accuracy * 100:.2f}%\\nModel before fine-tune AUC: {auc:.3f}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time \n\ntraining(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_bias[\"toxic\"] = train_bias[\"toxic\"].apply(lambda x: (1 if x >= .5 else 0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# After training model accuracy\npre, pre_class, accuracy = predict(model, trainloader, train)\nauc = roc_auc_score(train[\"toxic\"].values, pre)\nprint(\"Train: \")\nprint(f\"Model after fine-tune accuracy: {accuracy * 100:.3f}%\\nModel after fine-tune AUC: {auc:.3f}\")\n  \nfor key, value in validloaders.items():\n    pre, pre_class, accuracy = predict(model, value, valid[valid[\"lang\"] == lang[key]].reset_index(drop=True))\n    auc = roc_auc_score(valid[valid[\"lang\"] == lang[key]].reset_index(drop=True)[\"toxic\"].values, pre)\n    print(f\"{key} Valid: \")\n    print(f\"Model after fine-tune accuracy: {accuracy * 100:.2f}%\\nModel after fine-tune AUC: {auc:.3f}\")\n\npre, pre_class, accuracy = predict(model, validloader, valid)\nauc = roc_auc_score(valid[\"toxic\"].values, pre)\nprint(f\"Combined Valid: \")\nprint(f\"Model after fine-tune accuracy: {accuracy * 100:.2f}%\\nModel after fine-tune AUC: {auc:.3f}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pre, pre_class, accuracy = predict(model, evalidloader, evalid)\nauc = roc_auc_score(evalid[\"toxic\"].values, pre)\nprint(\"Evalid: \")\nprint(f\"Model after fine-tune accuracy: {accuracy * 100:.3f}%\\nModel after fine-tune AUC: {auc:.3f}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pre, pre_class, accuracy = predict(model, testloader, test, False)\nsubmit['toxic'] = pre\nsubmit.to_csv('submission.csv', index=False)\nsubmit.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\n\na = torch.tensor([[[1, 2, 3], [1, 4, 5]], [[4, 5, 6], [7, 4, 5]], [[4, 5, 6], [7, 4, 5]]]).type(torch.float32)\ntorch.max(a, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}