{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Identificar comentarios de toxicidad\n\n# 1. Descripción\n- Un área principal de atención son los modelos de aprendizaje automático que pueden identificar la toxicidad en las conversaciones en línea, donde la toxicidad se define como cualquier cosa grosera, irrespetuosa o que pueda hacer que alguien abandone una discusión. Si se pueden identificar estas contribuciones tóxicas, podríamos tener una Internet más segura y colaborativa.\n\n- \"Descargo de responsabilidad\": el conjunto de datos de este concurso contiene texto que puede considerarse profano, vulgar u ofensivo.\n\n# 2. Evaluación\n- \"área bajo la curva ROC\" entre la probabilidad pronosticada y el objetivo observado.\n\n# 3. Datos\n- `comment_text`: Contiene el texto de un comentario que ha sido clasificado como tóxico o no tóxico (0 ... 1 en la columna de tóxicos). Los comentarios del conjunto de datos están completamente en inglés y provienen de Civil Comments o de ediciones de la página de discusión de Wikipedia.\n\n- ¿Qué estoy prediciendo?\n> Está prediciendo la probabilidad de que un comentario sea tóxico. Un comentario tóxico recibiría un 1.0. Un comentario benigno y no tóxico recibiría un 0,0. En el conjunto de prueba, todos los comentarios se clasifican como 1.0 o 0.0.","metadata":{}},{"cell_type":"code","source":"!pip install -q pyicu\n!pip install -q pycld2\n!pip install -q polyglot\n!pip install -q pyyaml h5py  # Required to save models in HDF5 format","metadata":{"execution":{"iopub.status.busy":"2021-09-09T21:14:58.442695Z","iopub.execute_input":"2021-09-09T21:14:58.443128Z","iopub.status.idle":"2021-09-09T21:16:34.983358Z","shell.execute_reply.started":"2021-09-09T21:14:58.443045Z","shell.execute_reply":"2021-09-09T21:16:34.982359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport re\nimport pandas as pd\nimport numpy as np\nimport tqdm\nimport transformers\nimport tensorflow as tf\n\nfrom sklearn.metrics import accuracy_score, roc_auc_score, confusion_matrix\nfrom sklearn.model_selection import train_test_split\n\nfrom polyglot.detect import Detector\n\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-09-09T21:16:34.985030Z","iopub.execute_input":"2021-09-09T21:16:34.985369Z","iopub.status.idle":"2021-09-09T21:16:40.424617Z","shell.execute_reply.started":"2021-09-09T21:16:34.985330Z","shell.execute_reply":"2021-09-09T21:16:40.423737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(tf.__version__)\nprint(transformers.__version__)\nprint(tf.keras.__version__)","metadata":{"execution":{"iopub.status.busy":"2021-09-09T21:16:40.426368Z","iopub.execute_input":"2021-09-09T21:16:40.426692Z","iopub.status.idle":"2021-09-09T21:16:40.433400Z","shell.execute_reply.started":"2021-09-09T21:16:40.426656Z","shell.execute_reply":"2021-09-09T21:16:40.431286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\";\n# os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\";","metadata":{"execution":{"iopub.status.busy":"2021-09-09T21:16:40.435436Z","iopub.execute_input":"2021-09-09T21:16:40.435917Z","iopub.status.idle":"2021-09-09T21:16:40.441978Z","shell.execute_reply.started":"2021-09-09T21:16:40.435870Z","shell.execute_reply":"2021-09-09T21:16:40.440814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Preprocessing","metadata":{}},{"cell_type":"code","source":"def get_language(text):\n    return Detector(\n        \"\".join(x for x in text if x.isprintable()), quiet=True\n    ).languages[0].name\n\nPATH =  \"../input/jigsaw-multilingual-toxic-comment-classification\"\nFILES = os.listdir(PATH)\nprint(FILES)\n\nTRAIN_PATH = os.path.join(PATH, 'jigsaw-toxic-comment-train.csv')\ndata = pd.read_csv(TRAIN_PATH)\n\ndata[\"lang\"] = data[\"comment_text\"].apply(get_language)\ndata = data[data['lang'] == 'English']","metadata":{"execution":{"iopub.status.busy":"2021-09-09T21:16:40.443643Z","iopub.execute_input":"2021-09-09T21:16:40.444047Z","iopub.status.idle":"2021-09-09T21:17:01.104087Z","shell.execute_reply.started":"2021-09-09T21:16:40.444011Z","shell.execute_reply":"2021-09-09T21:17:01.103257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-09T21:17:01.105420Z","iopub.execute_input":"2021-09-09T21:17:01.105737Z","iopub.status.idle":"2021-09-09T21:17:01.126791Z","shell.execute_reply.started":"2021-09-09T21:17:01.105700Z","shell.execute_reply":"2021-09-09T21:17:01.125995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.toxic.value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-09-09T21:17:01.128157Z","iopub.execute_input":"2021-09-09T21:17:01.128544Z","iopub.status.idle":"2021-09-09T21:17:01.139167Z","shell.execute_reply.started":"2021-09-09T21:17:01.128508Z","shell.execute_reply":"2021-09-09T21:17:01.137989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Split data to Train and Test","metadata":{}},{"cell_type":"code","source":"X = data[['comment_text']]\ny = data[['toxic']]\n\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\ny_test = y_test.toxic.values\ny_train = y_train.toxic.values\n\nprint(X_train.shape)\nprint(y_train.shape)\nprint(X_test.shape)\nprint(y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2021-09-09T21:17:01.140530Z","iopub.execute_input":"2021-09-09T21:17:01.140873Z","iopub.status.idle":"2021-09-09T21:17:01.186688Z","shell.execute_reply.started":"2021-09-09T21:17:01.140839Z","shell.execute_reply":"2021-09-09T21:17:01.185821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Tokenizing the data","metadata":{}},{"cell_type":"code","source":"def map_func(input_ids, masks, labels):\n    return {\n        'input_ids': input_ids,\n        'attention_mask': masks\n    }, labels\n\nPRE_TRAINED_MODEL_NAME = 'bert-base-cased'\ntokenizer = transformers.AutoTokenizer.from_pretrained(PRE_TRAINED_MODEL_NAME)\n\nSEQ_LEN = 128\nX_train_ids = np.zeros((len(X_train), SEQ_LEN))\nX_train_mask = np.zeros((len(X_train), SEQ_LEN))\n\nX_test_ids = np.zeros((len(X_test), SEQ_LEN))\nX_test_mask = np.zeros((len(X_test), SEQ_LEN))\n\nfor i, sequence in enumerate(X_train['comment_text']):\n    tokens = tokenizer.encode_plus(\n        sequence, max_length=SEQ_LEN,\n        truncation=True, padding='max_length',\n        add_special_tokens=True, return_token_type_ids=False,\n        return_attention_mask=True, return_tensors='tf'\n    )\n    X_train_ids[i, :], X_train_mask[i, :] = tokens['input_ids'], tokens['attention_mask']\n    \nfor i, sequence in enumerate(X_test['comment_text']):\n    tokens = tokenizer.encode_plus(\n        sequence, max_length=SEQ_LEN,\n        truncation=True, padding='max_length',\n        add_special_tokens=True, return_token_type_ids=False,\n        return_attention_mask=True, return_tensors='tf'\n    )\n    X_test_ids[i, :], X_test_mask[i, :] = tokens['input_ids'], tokens['attention_mask']\n    \ntrain_dataset = tf.data.Dataset.from_tensor_slices((X_train_ids, X_train_mask, y_train))\ntest_dataset = tf.data.Dataset.from_tensor_slices((X_test_ids, X_test_mask, y_test))\n\ntrain_dataset = train_dataset.map(map_func)\ntest_dataset = test_dataset.map(map_func)\n\ntrain_dataset = train_dataset.shuffle(100000).batch(32, drop_remainder=True)\ntest_dataset = test_dataset.shuffle(100000).batch(32, drop_remainder=True)","metadata":{"execution":{"iopub.status.busy":"2021-09-09T21:17:01.189667Z","iopub.execute_input":"2021-09-09T21:17:01.189919Z","iopub.status.idle":"2021-09-09T21:18:41.751648Z","shell.execute_reply.started":"2021-09-09T21:17:01.189893Z","shell.execute_reply":"2021-09-09T21:18:41.750813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Building","metadata":{}},{"cell_type":"code","source":"bert = transformers.TFAutoModel.from_pretrained(PRE_TRAINED_MODEL_NAME)\n\ninput_ids = tf.keras.layers.Input(shape=(SEQ_LEN, ), name='input_ids', dtype='int32')\nmask = tf.keras.layers.Input(shape=(SEQ_LEN, ), name='attention_mask', dtype='int32')\n\nembeddings = bert.bert(input_ids, attention_mask=mask)[1]\n\nX = tf.keras.layers.Dense(1024, activation='relu')(embeddings)\n# X = tf.keras.layers.GlobalMaxPool1D()(embeddings)\n# X = tf.keras.layers.BatchNormalization()(X)\n# X = tf.keras.layers.Dense(128, activation='relu')(X)\n# X = tf.keras.layers.Dropout(0.1)(X)\n# X = tf.keras.layers.Dense(32, activation='relu')(X)\ny = tf.keras.layers.Dense(1, activation='sigmoid', name='outputs')(X)\n\nmodel = tf.keras.Model(inputs=[input_ids, mask], outputs=y)\n\nmodel.layers[2].trainable = False\n\nmodel.compile(\n    optimizer=tf.keras.optimizers.Adam(lr=1e-5, decay=1e-6), \n    loss='binary_crossentropy', \n    metrics=[tf.keras.metrics.AUC(name='AUC')]\n)\n\nr = model.fit(\n    train_dataset,\n    validation_data=(test_dataset),\n    epochs=10,\n    batch_size=4096\n)","metadata":{"execution":{"iopub.status.busy":"2021-09-09T21:18:41.753333Z","iopub.execute_input":"2021-09-09T21:18:41.753630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(test_dataset)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_learning_evolution(r):\n    plt.figure(figsize=(12, 8))\n    \n    plt.subplot(2, 2, 1)\n    plt.plot(r.history['loss'], label='Loss')\n    plt.plot(r.history['val_loss'], label='val_Loss')\n    plt.title('Loss evolution during trainig')\n    plt.legend()\n\n    plt.subplot(2, 2, 2)\n    plt.plot(r.history['AUC'], label='AUC')\n    plt.plot(r.history['val_AUC'], label='val_AUC')\n    plt.title('AUC score evolution during trainig')\n    plt.legend();","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_learning_evolution(r)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# y_pred = model.predict(test_dataset)\n# confusion_matrix(y_test, y_pred.round())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Saving and Uploading the model","metadata":{}},{"cell_type":"code","source":"!pip install pyyaml h5py  # Required to save models in HDF5 format","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.models.save_model(model, \"hate_speech_10_epochs.hdf5\")\nmodel.save(\"hate_speech_10_epochs.h5\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hdf5_model = tf.keras.models.load_model(\"hate_speech_10_epochs.hdf5\")\nhdf5_model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"h5_model = tf.keras.models.load_model(\"hate_speech_10_epochs.h5\")\nh5_model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prep_sentence(sentence):\n    tokens = tokenizer.encode_plus(\n        sentence, max_length=SEQ_LEN,\n        truncation=True, padding='max_length',\n        add_special_tokens=True, return_token_type_ids=False,\n        return_attention_mask=True, return_tensors='tf'\n    )\n    return {\n        'input_ids': tf.cast(tokens['input_ids'], tf.float64),\n        'attention_mask': tf.cast(tokens['attention_mask'], tf.float64)\n    }","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Toxic Comments","metadata":{}},{"cell_type":"code","source":"toxic_speechs = [\n    'COCKSUCKER BEFORE YOU PISS AROUND ON MY WORK',\n    'MEL GIBSON IS A NAZI BITCH WHO MAKES SHITTY MOVIES. HE HAS SO MUCH BUTTSEX THAT HIS ASSHOLE IS NOW BIG ENOUGH TO BE CONSIDERED A COUNTRY.',\n    \"A block ohhhhhhhhhhhhhh noooooooooooo I'm soooo like gonna cry and like shit ... ha ha.  you think i care?  i dont even use wikipedia.  look at the serb reporting me to the geek squad, what are you like 5?  Rumor has it that you are another canadian serb.  Rumor has it that you have pissed of a select few from B93 & WP.   )  BYE BYE.\",\n    \"it is a constructive edit you idiot, every kid of every age should know that santa claus is fucking fictional. ever since i first heard of santa claus i knew that he was fictional, my parents didn't give me any delusions and if they had, i would've laughed in their faces and said it isn't logical because it fucking isn't. every kid should be logical just like i was and every kid should be able to logically fucking infer that there is no fucking santa claus in the real human universe.\",\n    'honestly ==\\nyou need to crawl under a rock and DIE YOU FAT BASTARD\\n\\n=='\n]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for speech in toxic_speechs:\n    prediction = h5_model.predict(prep_sentence(speech))\n    print(prediction)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for speech in toxic_speechs:\n    prediction = hdf5_model.predict(prep_sentence(speech))\n    print(prediction)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Non-Toxic Comments","metadata":{}},{"cell_type":"code","source":"non_toxic_speechs = [\n    \"Gale, you're living proof why wikipedia should NEVER be trusted as fact. I mean, telling someone to blindly believe whatever's said instead of verifying? You need to take a walk in traffic for saying that!\\n\\n99.149.119.168\",\n    'EastEnders Manual of Style \\n\\nHello, just wanted you to be aware of the EE MoS, which helps us work out what is appropriate for Infoboxes etc.  Cheers,  (Talk)',\n    'You need to provide high-quality secondary sources (e.g., not original publications from medical experiments, but perhaps review articles or medical textbooks) that support this significant change in definition.',\n    \"I appreciate your responses, guys. I take the recommendation as an admin as a great compliment. However, since I move around so much and my knowledge of Wikipedia isn't where I would like it to be before I went after something like that, I will get back to you if the vote ever happens. In the mean time, I definitely appreciate the compliment. }\",    \n    \"Stop reinserting harrassing content on WP:ANI \\n\\nStop readding this material.  If you continue with this from other IP ranges or addresses we will be forced to block larger IP ranges from editing.  You aren't allowed to harrass people like this on Wikipedia.\"\n]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for speech in non_toxic_speechs:\n    prediction = h5_model.predict(prep_sentence(speech))\n    print(prediction)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for speech in non_toxic_speechs:\n    prediction = hdf5_model.predict(prep_sentence(speech))\n    print(prediction)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}