{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport math\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport time\nimport transformers\nimport re\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset\nfrom torch.utils.data import DataLoader\nfrom sklearn.metrics import f1_score, roc_auc_score\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"df1 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df2 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df2_cut = df2[['comment_text','toxic']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df1_cut = df1[['comment_text','toxic']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df1_cut.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_toxic_rus = pd.read_csv(\"/kaggle/input/toxic-rus-train/toxic_rus_train.csv\")\ndf_test_toxic_rus = pd.read_csv(\"/kaggle/input/toxic-rus-test/toxic_rus_test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_toxic_rus.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# df_train_toxic_rus = df_train_toxic_rus[['text','toxic']]\ndf_train_toxic_rus.columns = ['comment_text','toxic']\n\n# df_test_toxic_rus = df_test_toxic_rus[['text','toxic']]\ndf_test_toxic_rus.columns = ['comment_text','toxic']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df1_cut.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.concat([df_train_toxic_rus, df_test_toxic_rus])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.columns = ['comment_text','toxic']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_eng = pd.concat([df2_cut,df1_cut])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_eng.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.concat([df, df_eng], ignore_index = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.iloc[14400:14430]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# df = df.sample(frac=1).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean(text):\n#     text = text.fillna(\"fillna\").str.lower()\n    text = text.map(lambda x: re.sub('\\\\n',' ',str(x)))\n    text = text.map(lambda x: re.sub(\"\\[\\[User.*\",'',str(x)))\n    text = text.map(lambda x: re.sub(\"\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\",'',str(x)))\n    text = text.map(lambda x: re.sub(\"\\(http://.*?\\s\\(http://.*\\)\",'',str(x)))\n    return text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['comment_text'] = clean(df['comment_text'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_val = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv\")\ndf_test = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_val['comment_text'] = clean(df_val['comment_text'])\ndf_test['content'] = clean(df_test['content'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\ndevice","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pretrained_weights = 'xlm-roberta-large'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer = transformers.XLMRobertaTokenizer.from_pretrained(pretrained_weights)\nxlm_model = transformers.XLMRobertaModel.from_pretrained(pretrained_weights).to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def pad_or_cut(text,max_len,pad_index):\n    text = text[:1500]\n    tokenized_text = tokenizer.encode(text)\n    if len(tokenized_text) >  max_len:\n        tokenized_text = tokenized_text[:max_len]\n    elif len(tokenized_text) <  max_len:\n        tokenized_text += [pad_index] * (max_len - len(tokenized_text))\n    return tokenized_text\n\ndef sentence_embedder_model(text_batch):\n    pad_index = 1\n    max_len = 256\n\n    batch = torch.tensor([pad_or_cut(text,max_len,pad_index) for text in text_batch])\n#     print(batch.shape)\n\n    batch = batch[:, :((batch != pad_index).long()).sum(dim=1).max()].to(device)\n#     print(batch.shape)\n    pad_mask = (batch != pad_index).long().to(device)\n\n#     print(\"xlm_model ...\")\n    with torch.no_grad():\n        sequence_output, pooled_output = xlm_model(batch, attention_mask=pad_mask)\n#     print(\"sequence_output\", sequence_output.shape)\n    sequence_lengths = (pad_mask).sum(dim=1)\n    sequence_lengths[sequence_lengths == 0.] = 1\n\n    pad_mask_output = pad_mask.unsqueeze(-1).repeat(1, 1, sequence_output.size(-1))\n\n    sequence_output = sequence_output * pad_mask_output\n#     print(\"sequence_output\",sequence_output.shape)\n    lengths_scaling = sequence_lengths.float() / sequence_output.size(1)\n    lengths_scaling = lengths_scaling.unsqueeze(1).repeat((1, sequence_output.size(-1)))\n\n    sequence_output = sequence_output.mean(dim=1)\n#     print(\"sequence_output mean\",sequence_output.shape)\n\n    sequence_output = sequence_output / lengths_scaling#.to(sequence_output.device)\n\n    norm = sequence_output.norm(dim=1).unsqueeze(1).repeat((1, sequence_output.size(-1)))\n\n    sequence_output = sequence_output / norm\n\n    return  torch.tensor(sequence_output.tolist()) #[float(t) for t in sequence_output]\n# robert_vect = sentence_embedder_model([\"как купить слона\",\"hey\"])\n# robert_vect = sentence_embedder_model([\"как купить слона\"*10,\"hey\"]*32)\n# robert_vect = sentence_embedder_model([\"h\"*5000])\n# print(robert_vect.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"EPOCHS = 50\nBATCH_SIZE = 64\nLEARNING_RATE = 0.001","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RawDataset(Dataset):\n    def __init__(self, text, target):\n        self.text = text\n        self.target = target\n        \n    def __len__(self):\n        return len(self.text)\n    \n    def __getitem__(self, index):\n        return self.text[index], self.target[index]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class binaryClassification(nn.Module):\n    def __init__(self):\n        super(binaryClassification, self).__init__()\n        # Number of input features is 12.\n        self.layer_1 = nn.Linear(1024, 64) \n        self.layer_2 = nn.Linear(64, 64)\n        self.layer_out = nn.Linear(64, 1) \n        \n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(p=0.1)\n        self.batchnorm1 = nn.BatchNorm1d(64)\n        self.batchnorm2 = nn.BatchNorm1d(64)\n        \n    def forward(self, x):\n        x = self.relu(self.layer_1(x))\n        x = self.batchnorm1(x)\n        x = self.relu(self.layer_2(x))\n        x = self.batchnorm2(x)\n        x = self.dropout(x)\n        x = self.layer_out(x)\n        \n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_SPLIT = 200000\nVAL_SPLIT = -1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data = RawDataset(list(df['comment_text'])[:TRAIN_SPLIT],list(df['toxic'])[:TRAIN_SPLIT])\ntrain_loader = DataLoader(dataset=train_data, batch_size=BATCH_SIZE)\n\nval_data = RawDataset(list(df_val['comment_text'])[:VAL_SPLIT],list(df_val['toxic'])[:VAL_SPLIT])\nval_loader = DataLoader(dataset=val_data, batch_size=int(BATCH_SIZE/4))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = binaryClassification()\nmodel.to(device)\nprint(model)\ncriterion = nn.BCEWithLogitsLoss()\noptimizer = optim.Adam(model.parameters(), lr=LEARNING_RATE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def roc_auc(y_test,y_pred_tag):\n    \n    y_test = torch.round(y_test.detach().cpu())\n    print(\"y_test\", y_test[:10])\n    \n    \n    y_pred_tag= y_pred_tag.detach().cpu()\n    y_pred_tag = torch.sigmoid(y_pred_tag)\n    print(\"y_pred_tag\", y_pred_tag[:10])\n    \n    roc_auc = roc_auc_score(y_test, y_pred_tag)\n    return roc_auc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for instance in list(tqdm._instances):\n#     tqdm._decr_instances(instance)\nEPOCHS = 1\nbest_loss = 10000\nbest_roc_auc = 0\npatience = 0\nplot_every = int(len(train_loader)/3)\nfor e in range(1, EPOCHS+1):\n    model.train()\n    \n    train_epoch_loss = []\n#     train_epoch_roc_auc = []\n    progress_bar = tqdm(total=len(train_loader), desc='Train')\n    iteration = 0\n    \n    for instance in list(tqdm._instances):\n        tqdm._decr_instances(instance)\n    \n    for X_batch, y_batch in train_loader:\n        y_batch = y_batch.to(device)\n#         print(\"sent embedder ...\")\n        X_batch = sentence_embedder_model(X_batch).to(device)\n#         X_batch = torch.stack([sentence_embedder_model(sent) for sent in X_batch])\n#         X_batch = torch.tensor(X_batch.numpy()).to(device)\n        \n        optimizer.zero_grad()\n#         print(\"model forward ...\")\n        y_pred = model(X_batch)\n        y_batch = y_batch.type_as(y_pred)\n        loss = criterion(y_pred, y_batch.unsqueeze(1))\n#         roc_auc_value = roc_auc(y_pred, y_batch.unsqueeze(1))\n        \n        loss.backward()\n        optimizer.step()\n        \n        train_epoch_loss.append(loss.item())\n#         train_epoch_roc_auc.append(roc_auc_value)\n        progress_bar.update()\n        progress_bar.set_postfix(epoch = e,loss=np.mean(train_epoch_loss[-100:]))#roc = np.mean(roc_auc_value[-100:])\n        iteration += 1\n        if iteration%plot_every == 0:\n            plt.plot(train_epoch_loss)\n            plt.ylabel('Loss')\n            plt.show()\n            \n#     print(f'Epoch {e+0:03}: | Loss: {np.mean(train_epoch_loss):.5f} | Acc: {np.mean(train_epoch_f1):.3f}')\n    \n    #========================EVALUATION=============\n    model.eval()\n    eval_epoch_loss = []\n    eval_epoch_preds = []\n    eval_epoch_trues = []\n#     eval_progress_bar = tqdm(total=len(val_loader), desc='Eval')\n    print(\"EVALUATE IS RUNNING ....\")\n    for X_batch, y_batch in val_loader:\n        y_batch = y_batch.to(device)\n        \n#         X_batch = torch.stack([sentence_embedder_model(sent) for sent in X_batch])\n#         X_batch = torch.tensor(X_batch.numpy()).to(device)\n        X_batch = sentence_embedder_model(X_batch).to(device)\n        \n        y_pred = model(X_batch)\n        y_batch = y_batch.type_as(y_pred)\n        \n        ev_loss = criterion(y_pred, y_batch.unsqueeze(1))\n        eval_epoch_loss.append(ev_loss.item())\n        \n        eval_epoch_preds.extend(y_pred)\n        eval_epoch_trues.extend(y_batch.unsqueeze(1))\n        \n#         eval_progress_bar.update()\n#         eval_progress_bar.set_postfix(eval_loss=np.mean(eval_epoch_loss[-100:]))\n    \n    eval_epoch_preds = torch.cat(eval_epoch_preds)\n    eval_epoch_trues = torch.cat(eval_epoch_trues,0)\n    roc_auc_value = roc_auc(eval_epoch_trues, eval_epoch_preds)\n    \n    mean_epoch_loss = np.mean(eval_epoch_loss)\n    if mean_epoch_loss < best_loss and roc_auc_value > best_roc_auc:\n        print(\"NEW BEST RESULT! SAVING ...\")\n        best_loss = mean_epoch_loss\n        best_roc_auc = roc_auc_value\n        patience = 0\n#         torch.save(model.state_dict(),\n#                        'train_results/model_state_dict.pth')\n#         torch.save(optimizer.state_dict(),\n#                        'train_results/optimizer_state_dict.pth')\n    elif mean_epoch_loss >= best_loss and roc_auc_value <= best_roc_auc:\n        patience += 1\n    elif mean_epoch_loss >= best_loss:\n        best_roc_auc = roc_auc_value\n        patience += 1\n    elif roc_auc_value <= best_roc_auc:\n        best_loss = mean_epoch_loss\n        patience += 1\n    if patience > 3:\n        print(\"out of patience!\")\n        break\n        \n    print(f'Epoch {e+0:03}: | Validation Loss: {mean_epoch_loss:.5f} | Validation roc_auc_value: {roc_auc_value:.3f}')\n    print(\"=\"*100)\n#     break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"roc_auc_value","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = pd.read_csv(os.path.join(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/\",\"test.csv\"))\ndf_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RawDataset_test(Dataset):\n    def __init__(self, text):\n        self.text = text\n        \n    def __len__(self):\n        return len(self.text)\n    \n    def __getitem__(self, index):\n        return self.text[index]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data = RawDataset_test(list(df_test['content']))\ntest_loader = DataLoader(dataset=test_data, batch_size=int(BATCH_SIZE/4))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.eval()\ntest_epoch_preds = []\nprint(\"TEST IS RUNNING ....\")\nfor X_batch in test_loader:\n\n    X_batch = sentence_embedder_model(X_batch).to(device)\n\n    y_pred = model(X_batch)\n\n    test_epoch_preds.extend(y_pred)\n\ntest_epoch_preds = torch.cat(test_epoch_preds)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_epoch_preds[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_epoch_preds_t= test_epoch_preds.detach().cpu()\ntest_epoch_preds_t = torch.sigmoid(test_epoch_preds_t)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_test_preds = list(float(i) for i in test_epoch_preds_t)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_test_preds[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"list(df_test.id)[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"my_submission = pd.DataFrame({'Id': list(df_test.id), 'toxic': final_test_preds})\n# you could use any filename. We choose submission here\nmy_submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir(\"./\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}