{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install --upgrade pip\n!pip install -q textstat\n!pip install scispacy\n!pip install emot","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n\nimport gc\nimport os\nimport time\nimport math\nimport random\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom datetime import date\nfrom transformers import *\nfrom sklearn.metrics import *\nfrom tqdm.notebook import tqdm\n\nimport torch\nimport torch.nn as nn\nimport torch.utils.data\nimport torch.nn.functional as F\n\n\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport os\nimport gc\nimport re\nimport string\nimport folium\nimport textstat\nfrom scipy import stats\nfrom colorama import Fore, Back, Style, init\n\nimport math\nimport numpy as np\nimport scipy as sp\nimport pandas as pd\n\nimport random\nimport networkx as nx\nfrom pandas import Timestamp\n\nfrom PIL import Image\nfrom IPython.display import SVG\nfrom keras.utils import model_to_dot\n\nimport requests\nfrom IPython.display import HTML\n\nimport seaborn as sns\nfrom tqdm import tqdm\nimport matplotlib.cm as cm\nimport matplotlib.pyplot as plt\n\ntqdm.pandas()\n\nimport plotly.express as px\nimport plotly.graph_objects as go\nimport plotly.figure_factory as ff\nfrom plotly.subplots import make_subplots\n\nimport transformers\nimport tensorflow as tf\n\nfrom tensorflow.keras.callbacks import Callback\nfrom sklearn.metrics import accuracy_score, roc_auc_score\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, CSVLogger\n\nfrom tensorflow.keras.models import Model\nfrom kaggle_datasets import KaggleDatasets\nfrom tensorflow.keras.optimizers import Adam\nfrom tokenizers import BertWordPieceTokenizer\nfrom tensorflow.keras.layers import Dense, Input, Dropout, Embedding\nfrom tensorflow.keras.layers import LSTM, GRU, Conv1D, SpatialDropout1D\n\nfrom tensorflow.keras import layers\nfrom tensorflow.keras import optimizers\nfrom tensorflow.keras import activations\nfrom tensorflow.keras import constraints\nfrom tensorflow.keras import initializers\nfrom tensorflow.keras import regularizers\n\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.optimizers import *\nfrom tensorflow.keras.activations import *\nfrom tensorflow.keras.constraints import *\nfrom tensorflow.keras.initializers import *\nfrom tensorflow.keras.regularizers import *\n\n\nfrom sklearn import metrics\nfrom sklearn.utils import shuffle\nfrom gensim.models import Word2Vec\nfrom sklearn.cluster import KMeans\nfrom sklearn.decomposition import PCA\nfrom sklearn.feature_extraction.text import TfidfVectorizer,CountVectorizer,HashingVectorizer\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.naive_bayes import MultinomialNB\nfrom sklearn.model_selection import train_test_split\n\nfrom nltk.stem.wordnet import WordNetLemmatizer \nfrom nltk.tokenize import word_tokenize\nfrom nltk.tokenize import TweetTokenizer  \n\nimport nltk\nfrom textblob import TextBlob\n\n\nfrom nltk.corpus import wordnet\nfrom nltk.corpus import stopwords\nfrom nltk import WordNetLemmatizer\nfrom nltk.stem import WordNetLemmatizer,PorterStemmer\nfrom wordcloud import WordCloud, STOPWORDS\nfrom nltk.sentiment.vader import SentimentIntensityAnalyzer\nfrom nltk.tokenize import sent_tokenize, word_tokenize\n\nstopword=set(STOPWORDS)\n\nfrom scispacy.abbreviation import AbbreviationDetector\nfrom emot.emo_unicode import UNICODE_EMO, EMOTICONS\n\nlem = WordNetLemmatizer()\ntokenizer=TweetTokenizer()\n\nnp.random.seed(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train1 = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")\n\n\ntrain2 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\")\ntrain2.toxic = train2.toxic.round().astype(int)\n\n#train3 = pd.read_csv('../input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-es-cleaned.csv')\n\n\nvalid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\ntest = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')\nsub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')\n\n\ntoxic = len(train2[['comment_text', 'toxic']].query('toxic==1'))\n\nSAMPLE_SIZE = 10000\n\n# Combine train1 with a subset of train2\ntrain_cat = pd.concat([\n    train1[['comment_text', 'toxic']],\n    train2[['comment_text', 'toxic']].query('toxic==1'),\n#    train3[['comment_text', 'toxic']].query('toxic==0'),\n#    train3[['comment_text', 'toxic']].query('toxic==1'),   \n    train2[['comment_text', 'toxic']].query('toxic==0').sample(n=(toxic+(toxic//3)), random_state=101)\n]).sample(n=SAMPLE_SIZE).reset_index(drop=True) #restricting data to 600,000 records due to memory issue\n\ntest_data = test\ntrain_data = train_cat\ntrain_data_o  = train_cat","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(train_data))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid['lang'].value_counts() ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid['toxic'].value_counts() ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data_o['toxic'].value_counts() \n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Preprocessing"},{"metadata":{},"cell_type":"markdown","source":"Making Class Balance (Toxic and Non-toxic)"},{"metadata":{"trusted":true},"cell_type":"code","source":"count_toxic_0, count_toxic_1 = train_data_o.toxic.value_counts()  \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_toxic_0 = train_data_o[train_data_o['toxic'] == 0]\ndf_toxic_1 = train_data_o[train_data_o['toxic'] == 1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_toxic_0_under = df_toxic_0.sample(count_toxic_1)\ntrain_data_df = pd.concat([df_toxic_0_under, df_toxic_1], axis=0)\n\nprint('under-sampling:')\nprint(train_data_df.toxic.value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Number of words\ntrain_data['num_words'] = train_data['comment_text'].apply(lambda x: len(str(x).split()))\n#df_test['num_words'] = df_test['question_text'].apply(lambda x: len(str(x).split()))\n\n# Number of capital_letters\ntrain_data['num_capital_let'] = train_data['comment_text'].apply(lambda x: len([c for c in str(x) if c.isupper()]))\n#df_test['num_capital_let'] = df_test['question_text'].apply(lambda x: len([c for c in str(x) if c.isupper()]))\n\n# Number of special characters\ntrain_data['num_special_char'] = train_data['comment_text'].str.findall(r'[^a-zA-Z0-9 ]').str.len()\n#df_test['num_special_char'] = df_test['question_text'].str.findall(r'[^a-zA-Z0-9 ]').str.len()\n\n# Number of unique words\ntrain_data['num_unique_words'] = train_data['comment_text'].apply(lambda x: len(set(str(x).split())))\n#df_test['num_unique_words'] = df_test['question_text'].apply(lambda x: len(set(str(x).split())))\n\n# Number of numerics\ntrain_data['num_numerics'] = train_data['comment_text'].apply(lambda x: sum(c.isdigit() for c in x))\n#df_test['num_numerics'] = df_test['question_text'].apply(lambda x: sum(c.isdigit() for c in x))\n\n# Number of characters\ntrain_data['num_char'] = train_data['comment_text'].apply(lambda x: len(str(x)))\n#df_test['num_char'] = df_test['question_text'].apply(lambda x: len(str(x)))\n\n# Number of stopwords\ntrain_data['num_stopwords'] = train_data['comment_text'].apply(lambda x: len([c for c in str(x).lower().split() if c in STOPWORDS]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_boxplot(_x, _y, _data, _title):\n    sns.boxplot(x=_x, y=_y, data=_data)\n    plt.grid(True)\n    #plt.tick_params(axis='x', which='major', labelsize=15)\n    plt.title(_title,fontsize=17)\n    plt.xlabel(_x, fontsize=10)\n\n# Boxplot: Number of words\nplt.subplot(2, 3, 1)\ndisplay_boxplot('toxic', 'num_words', train_data, 'No. of words in each class')\n\n# Boxplot: Number of chars\nplt.subplot(2, 3, 2)\ndisplay_boxplot('toxic', 'num_char', train_data, 'Number of characters in each class')\n\n# Boxplot: Number of unique words\nplt.subplot(2, 3, 3)\ndisplay_boxplot('toxic', 'num_unique_words', train_data, 'Number of unique words in each class')\n\n# Boxplot: Number of special characters\nplt.subplot(2, 3, 4)\ndisplay_boxplot('toxic', 'num_special_char', train_data, 'No. of special characters in each class')\n\n# Boxplot: Number of stopwords\nplt.subplot(2, 3, 5)\ndisplay_boxplot('toxic', 'num_stopwords', train_data, 'Number of stopwords in each class')\n\n# Boxplot: Number of capital letters\nplt.subplot(2, 3, 6)\ndisplay_boxplot('toxic', 'num_capital_let', train_data, 'No. of capital letters in each class')\n\n\nplt.subplots_adjust(right=3.0)\nplt.subplots_adjust(top=2.0)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Wordcloud of all comments"},{"metadata":{"trusted":true},"cell_type":"code","source":"def nonan(x):\n    if type(x) == str:\n        return x.replace(\"\\n\", \"\")\n    else:\n        return \"\"\n\ntext = ' '.join([nonan(abstract) for abstract in train_data[\"comment_text\"][train_data[\"toxic\"]>0.5]])\nwordcloud = WordCloud(max_font_size=None, background_color='black', collocations=False,\n                      width=1200, height=1000).generate(text)\nfig_1 = px.imshow(wordcloud)\nfig_1.update_layout(title_text='Common words in toxic comments')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def new_len(x):\n    if type(x) is str:\n        return len(x.split())\n    else:\n        return 0\n\ntrain_data[\"comment_words\"] = train_data[\"comment_text\"].apply(new_len)\nnums = train_data.query(\"comment_words != 0 and comment_words < 200\").sample(frac=0.1)[\"comment_words\"]\nfig = ff.create_distplot(hist_data=[nums],\n                         group_labels=[\"All comments\"],\n                         colors=[\"coral\"])\n\nfig.update_layout(title_text=\"Comment words\", xaxis_title=\"Comment words\", template=\"simple_white\", showlegend=False)\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### remove stop words"},{"metadata":{"trusted":true},"cell_type":"code","source":"punctuations = string.punctuation\nstop_words = set(stopwords.words(\"english\"))\n\ndef remove_stop_words(words):\n    return [w for w in words if not w in stop_words]\n\ndef remove_empty(words):\n    return [w for w in words if not len(w.strip())<2]\n\ndef words_to_lowercase(words):\n    return [w.lower() for w in words]\n\ndef remove_punctuation_from_word(word):\n    return ''.join(c for c in word if c not in punctuations)\n\ndef remove_punctuation_from_words(words):\n    return [remove_punctuation_from_word(w) for w in words]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_cleaned = train_data_df.copy()\n\ntrain_cleaned['comment_text'] = train_cleaned['comment_text'].apply(word_tokenize) \\\n                                                             .apply(words_to_lowercase) \\\n                                                             .apply(remove_stop_words) \\\n                                                             .apply(remove_empty)\n                                                             #.apply(remove_punctuation_from_words) \\","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_cleaned['comment_text'][0:3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_cleaned['comment_text'][0:3])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### emojis"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Function for converting emojis into word\ndef convert_emojis(words):\n    text = ' '.join(words)\n    for emot in UNICODE_EMO:\n        text = text.replace(emot, \"_\".join(UNICODE_EMO[emot].replace(\",\",\"\").replace(\":\",\"\").split()))\n    return text\n\ntrain_cleaned['comment_text'] = train_cleaned['comment_text'].apply(convert_emojis)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"###  abbreviation"},{"metadata":{"trusted":true},"cell_type":"code","source":"# abbreviation_pipe=AbbreviationDetector(nlp)\n# nlp.add_pipe(abbreviation_pipe)\n# def replace_acronyms(text):\n#     doc=nlp(txt)\n#     altered_tok=[tok.text for tok in doc]\n#     print(doc._.abbreviations)\n#     for abrv in doc._.abbreviations:\n#         altered_tok[abrv.start]=str(abrv._.long_form)\n#     return(\" \".join(altered_tok))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### stemming"},{"metadata":{"trusted":true},"cell_type":"code","source":"ps = PorterStemmer() \n\ndef stem_words(words):\n    return [ps.stem(w) for w in words]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_cleaned['comment_text'][0:3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_cleaned['comment_text'] = train_cleaned['comment_text'].apply(stem_words)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_cleaned['comment_text'][0:3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"### Lemmatization"},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_cleaned['comment_text'] = train_cleaned['comment_text'].apply(lemmatizer.lemmatize)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_cleaned['comment_text'][0:3])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Vectorisation"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_cleaned.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_cleaned['comment_text'][0:3])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Count vectorization"},{"metadata":{"trusted":true},"cell_type":"code","source":"count_vectorizer = CountVectorizer() \ntfidf_vectorizer = TfidfVectorizer()\nhashing_vectorizer = HashingVectorizer()\n\ntfidf_vector = tfidf_vectorizer.fit_transform(train_cleaned['comment_text'])\ncount_vector = count_vectorizer.fit_transform(train_cleaned['comment_text']) \nhashing_vector = hashing_vectorizer.fit_transform(train_cleaned['comment_text']) \n\n# print(\"Vocabulary: \", count_vectorizer.vocabulary_) \n# print(\"Vocabulary: \", tfidf_vectorizer.vocabulary_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_df = train_cleaned.copy()\nx_df[\"tfidf_vector\"]= tfidf_vector\nx_df[\"count_vector\"]= count_vector\nx_df[\"hashing_vector\"]= hashing_vector\ny_df = train_cleaned.toxic","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train_1, x_test_1, y_train_1, y_test_1 = train_test_split(tfidf_vector, y_df, test_size=0.3, random_state=42)\nx_train_2, x_test_2, y_train_2, y_test_2 = train_test_split(count_vector, y_df, test_size=0.3, random_state=42)\nx_train_3, x_test_3, y_train_3, y_test_3 = train_test_split(hashing_vector, y_df, test_size=0.3, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lg_regression_classifier_tfidf = LogisticRegression().fit(x_train_1, y_train_1)\nlg_regression_classifier_count = LogisticRegression().fit(x_train_2, y_train_2)\nlg_regression_classifier_hashing = LogisticRegression().fit(x_train_3, y_train_3)\n\nmultinomial_nb_classifier_tfidf = MultinomialNB().fit(x_train_1, y_train_1)\nmultinomial_nb_classifier_count = MultinomialNB().fit(x_train_2, y_train_2)\n#multinomial_nb_classifier_hashing = MultinomialNB().fit(x_train_3, y_train_3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('lg_regression_classifier_tfidf : ',lg_regression_classifier_tfidf.score(x_test_1, y_test_1))\nprint('lg_regression_classifier_count : ',lg_regression_classifier_count.score(x_test_2, y_test_2))\nprint('lg_regression_classifier_hashing : ',lg_regression_classifier_hashing.score(x_test_3, y_test_3))\n\nprint('multinomial_nb_classifier_tfidf : ',multinomial_nb_classifier_tfidf.score(x_test_1, y_test_1))\nprint('multinomial_nb_classifier_count : ',multinomial_nb_classifier_count.score(x_test_2, y_test_2))\n#print('multinomial_nb_classifier_hashing : ',clf_regression.score(x_test_3, y_test_3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n!python pytorch-xla-env-setup.py --apt-packages libomp5 libopenblas-dev\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport torch\nimport pandas as pd\nfrom scipy import stats\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nfrom collections import OrderedDict, namedtuple\nimport torch.nn as nn\nfrom torch.optim import lr_scheduler\nimport joblib\n\nimport logging\nimport transformers\nimport sys\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\npip install transformers==2.1.0","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{"trusted":true},"cell_type":"code","source":"import transformers\n\nprint(transformers.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nclass BERTBaseUncased(nn.Module):\n    def __init__(self, bert_path):\n        super(BERTBaseUncased, self).__init__()\n        self.bert_path = bert_path\n        self.bert = transformers.BertModel.from_pretrained(self.bert_path)\n        self.bert_drop = nn.Dropout(0.3)\n        self.out = nn.Linear(768 * 2, 1)\n\n    def forward(\n            self,\n            ids,\n            mask,\n            token_type_ids\n    ):\n        o1, o2 = self.bert(\n            ids,\n            attention_mask=mask,\n            token_type_ids=token_type_ids)\n        \n        apool = torch.mean(o1, 1)\n        mpool, _ = torch.max(o1, 1)\n        cat = torch.cat((apool, mpool), 1)\n\n        bo = self.bert_drop(cat)\n        p2 = self.out(bo)\n        return p2\n\n\nclass BERTDatasetTest:\n    def __init__(self, comment_text, tokenizer, max_length):\n        self.comment_text = comment_text\n        self.tokenizer = tokenizer\n        self.max_length = max_length\n\n    def __len__(self):\n        return len(self.comment_text)\n\n    def __getitem__(self, item):\n        comment_text = str(self.comment_text[item])\n        comment_text = \" \".join(comment_text.split())\n\n        inputs = self.tokenizer.encode_plus(\n            comment_text,\n            None,\n            add_special_tokens=True,\n            max_length=self.max_length,\n        )\n        ids = inputs[\"input_ids\"]\n        token_type_ids = inputs[\"token_type_ids\"]\n        mask = inputs[\"attention_mask\"]\n        \n        padding_length = self.max_length - len(ids)\n        \n        ids = ids + ([0] * padding_length)\n        mask = mask + ([0] * padding_length)\n        token_type_ids = token_type_ids + ([0] * padding_length)\n        \n        return {\n            'ids': torch.tensor(ids, dtype=torch.long),\n            'mask': torch.tensor(mask, dtype=torch.long),\n            'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long)\n        }\ndf = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/test.csv\")\n\ntokenizer = transformers.BertTokenizer.from_pretrained(\"../input/bert-base-multilingual-uncased/\", do_lower_case=True)\n\ndevice = \"cuda\"\nmodel = BERTBaseUncased(bert_path=\"../input/bert-base-multilingual-uncased/\").to(device)\nmodel.load_state_dict(torch.load(\"../input/jbertml/model.bin\"))\nmodel.eval()\n\nvalid_dataset = BERTDatasetTest(\n        comment_text=df.content.values,\n        tokenizer=tokenizer,\n        max_length=192\n)\n\nvalid_data_loader = torch.utils.data.DataLoader(\n    valid_dataset,\n    batch_size=64,\n    drop_last=False,\n    num_workers=4,\n    shuffle=False\n)\n\nwith torch.no_grad():\n    fin_outputs = []\n    for bi, d in tqdm(enumerate(valid_data_loader)):\n        ids = d[\"ids\"]\n        mask = d[\"mask\"]\n        token_type_ids = d[\"token_type_ids\"]\n\n        ids = ids.to(device, dtype=torch.long)\n        mask = mask.to(device, dtype=torch.long)\n        token_type_ids = token_type_ids.to(device, dtype=torch.long)\n\n        outputs = model(\n            ids=ids,\n            mask=mask,\n            token_type_ids=token_type_ids\n        )\n\n        outputs_np = outputs.cpu().detach().numpy().tolist()\n        fin_outputs.extend(outputs_np)\n        \nsample = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv\")\nsample.loc[:, \"toxic\"] = fin_outputs\nsample.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}