{"cells":[{"metadata":{},"cell_type":"markdown","source":"**IMPORTACIONES**"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\n\nSEED = 2018\n\nnp.random.seed(SEED)\ntf.random.set_seed(SEED)\n\nfrom tqdm import tqdm\ntqdm.pandas()\nimport os\nimport os\nimport time\nimport numpy as np \nimport pandas as pd \nfrom tqdm import tqdm\nimport math\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D\nfrom keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate, Lambda\nfrom keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D, BatchNormalization\nfrom keras.optimizers import Adam, Nadam\nfrom keras.models import Model\nfrom keras import backend as K\nfrom keras.callbacks import Callback\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\nfrom keras.engine.topology import Layer\nimport gc, re\nfrom sklearn import metrics\nfrom sklearn.model_selection import GridSearchCV, StratifiedKFold\nfrom sklearn.metrics import f1_score, roc_auc_score, confusion_matrix, auc, precision_recall_curve\nimport time\nfrom tqdm import tqdm\nimport math\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D\nfrom keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate, Lambda\nfrom keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D, BatchNormalization\nfrom keras.optimizers import Adam, Nadam\nfrom keras.models import Model\nfrom keras import backend as K\nfrom keras.callbacks import Callback\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\nfrom keras.engine.topology import Layer\nimport gc, re\nfrom sklearn import metrics\nfrom sklearn.model_selection import GridSearchCV, StratifiedKFold\nfrom sklearn.metrics import f1_score, roc_auc_score, confusion_matrix, auc, precision_recall_curve\n\nprint(os.listdir(\"../input/quora-insincere-questions-classification\"))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**PRE-PROCESAMIENTO DE LOS DATOS**"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"#Caracteres especiales\npuncts  = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\n\n#Función para quitar del texto los caracteres especiales\ndef clean_text(x):\n    x = str(x)\n    for punct in puncts:\n        if punct in x:\n            x = x.replace(punct, f' {punct} ')\n    return x\n\n#Función para limpiar los números\ndef clean_numbers(x): \n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    return x\n\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\n#Contracciones comunes\nmispell_dict = {\"aren't\" : \"are not\", \n\"can't\" : \"cannot\",\n\"couldn't\" : \"could not\",\n\"didn't\" : \"did not\",\n\"doesn't\" : \"does not\",\n\"don't\" : \"do not\",\n\"isn't\" : \"is not\",\n\"it's\" : \"it is\",\n\"it'll\":\"it will\",\n\"hadn't\" : \"had not\",\n\"hasn't\" : \"has not\",\n\"haven't\" : \"have not\",\n\"he'd\" : \"he would\",\n\"he'll\" : \"he will\",\n\"he's\" : \"he is\",\n\"i'd\" : \"I would\",\n\"i'd\" : \"I had\",\n\"i'll\" : \"I will\",\n\"i'm\" : \"I am\",\n\"i've\" : \"I have\",\n\"let's\" : \"let us\",\n\"mightn't\" : \"might not\",\n\"mustn't\" : \"must not\",\n\"shan't\" : \"shall not\",\n\"she'd\" : \"she would\",\n\"she'll\" : \"she will\",\n\"she's\" : \"she is\",\n\"shouldn't\" : \"should not\",\n\"that's\" : \"that is\",\n\"there's\" : \"there is\",\n\"they'd\" : \"they would\",\n\"they'll\" : \"they will\",\n\"they're\" : \"they are\",\n\"they've\" : \"they have\",\n\"we'd\" : \"we would\",\n\"we're\" : \"we are\",\n\"weren't\" : \"were not\",\n\"we've\" : \"we have\",\n\"what'll\" : \"what will\",\n\"what're\" : \"what are\",\n\"what's\" : \"what is\",\n\"what've\" : \"what have\",\n\"where's\" : \"where is\",\n\"who'd\" : \"who would\",\n\"who'll\" : \"who will\",              \n\"who're\" : \"who are\",\n\"who's\" : \"who is\",\n\"who've\" : \"who have\",\n\"won't\" : \"will not\",\n\"wouldn't\" : \"would not\",\n\"you'd\" : \"you would\",\n\"you'll\" : \"you will\",\n\"you're\" : \"you are\",\n\"you've\" : \"you have\",\n\"'re\": \" are\",\n\"wasn't\": \"was not\",\n\"we'll\":\" will\",\n\"ain't\" :  \"will not\",\n\"didn't\": \"did not\"}\n\nmispellings, mispellings_re = _get_mispell(mispell_dict)\n\n#Función para Reemplazar faltas de ortografía comunes\ndef replace_typical_misspell(text): \n    def replace(match):\n        return mispellings[match.group(0)]\n    return mispellings_re.sub(replace, text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Extrayendo data de entrenamiento y validación\ntrain_df = pd.read_csv(\"../input/quora-insincere-questions-classification/train.csv\")\ntest_df = pd.read_csv(\"../input/quora-insincere-questions-classification/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Tamaño training y test set\nprint(\"Train shape : \",train_df.shape)\nprint(\"Test shape : \",test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Porcentaje Preguntas Toxicas / Preguntas no toxicas**"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['target'].value_counts().plot(kind = 'pie', labels = ['No Ofensiva', 'Ofensiva'],\n     startangle = 100, autopct = '%1.1f%%')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Podemos ver que la proporción entre preguntas Ofensivas y No ofensivas está muy desproporcionada."},{"metadata":{"trusted":true},"cell_type":"code","source":"#Quitar caracteres especiales\ntrain_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: clean_text(x))\ntest_df[\"question_text\"] = test_df[\"question_text\"].progress_apply(lambda x: clean_text(x))\n    \n#Limpieza numero\ntrain_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: clean_numbers(x))\ntest_df[\"question_text\"] = test_df[\"question_text\"].progress_apply(lambda x: clean_numbers(x))\n    \n#Quitar contracciones\ntrain_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: replace_typical_misspell(x))\ntest_df[\"question_text\"] = test_df[\"question_text\"].progress_apply(lambda x: replace_typical_misspell(x))\n\n#Pasar letras a minuscula\ntrain_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: x.lower())\ntest_df[\"question_text\"] = test_df[\"question_text\"].progress_apply(lambda x: x.lower())\n    \ntrain_X = train_df[\"question_text\"].fillna(\"_##_\").values\nsplits = list(StratifiedKFold(n_splits=10,random_state=2018).split(train_X,train_df['target'].values))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Se divide la data en un set de train y uno de validación\ntrain_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Los valores faltantes se llenan con 'na'\ntrain_X = train_df[\"question_text\"].fillna(\"_na_\").values\nval_X = val_df[\"question_text\"].fillna(\"_na_\").values\ntest_X = test_df[\"question_text\"].fillna(\"_na_\").values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Configuración\nembed_size = 300      # Tamaño máximo de cada vector embedding\nmax_features = 50000  # Número total de palabras únicas \nmaxlen = 100          # Número máximo de palabras que tiene la pregunta","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**TOKENIZACIÓN**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Tokenización de las oraciones\ntokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(train_X))\ntrain_X = tokenizer.texts_to_sequences(train_X)\nval_X = tokenizer.texts_to_sequences(val_X)\ntest_X = tokenizer.texts_to_sequences(test_X)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**PADDING**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Si el número de palabras en el texto es mayor que 'max_len' se trunca a 'max_len'. \n# Si el número de palabras en el texto es menor que 'max_len' se agrega ceros para completar valores restantes.\ntrain_X = pad_sequences(train_X, maxlen=maxlen)\nval_X = pad_sequences(val_X, maxlen=maxlen)\ntest_X = pad_sequences(test_X, maxlen=maxlen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Consigue los valores objetivos\ntrain_y = train_df['target'].values\nval_y = val_df['target'].values","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**MODELO SIN EMBEDDINGS PRE-ENTRENADOS**"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Se utilizó un modelo GRU bidireccional.\ninp = tf.keras.layers.Input(shape=(maxlen,))\nx = tf.keras.layers.Embedding(max_features, embed_size)(inp)\nx = tf.keras.layers.Bidirectional(tf.compat.v1.keras.layers.CuDNNGRU(64, return_sequences=True))(x)\nx = tf.keras.layers.GlobalMaxPooling1D()(x)\nx = tf.keras.layers.Dense(16, activation=\"relu\")(x)\nx = tf.keras.layers.Dropout(0.1)(x)\nx = tf.keras.layers.Dense(1, activation=\"sigmoid\")(x)\nmodel = tf.keras.models.Model(inputs=inp, outputs=x)\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**PARAMETROS PARA TODOS LOS MODELOS**"},{"metadata":{"trusted":true},"cell_type":"code","source":"bathsize = 512\nepoch = 1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**ENTRENAMIENTO**"},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(train_X, train_y, batch_size=bathsize, epochs=epoch, validation_data=(val_X, val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Ahora se obtiene las predicciones de la muestra de validación y también se obtiene el mejor umbral para la puntuación de F\npred_noemb_val_y = model.predict([val_X], batch_size=1024, verbose=0) #verbose=1\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score en el límite {0} es {1}\".format(thresh, metrics.f1_score(val_y, (pred_noemb_val_y>thresh).astype(int))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Obtenemos las predicciones y se guardan (no se uso un embedding pre entrenado)\n#pred_noemb_test_y = model.predict([test_X], batch_size=1024, verbose=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Borrar de la memoria \ndel model, inp, x\nimport gc; gc.collect()\ntime.sleep(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**EMBEDDINGS DISPONIBLES**"},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ../input/quora-insincere-questions-classification/embeddings/","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**IMPORTANDO EMBEDDING GLOVE**\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"EMBEDDING_FILE = '../input/quora-insincere-questions-classification/embeddings/glove.840B.300d/glove.840B.300d.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**SE RESCONSTRUYE EL MODELO USANDO EL EMBEDDING PRE-ENTRENADO: GLOVE**\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"inp = tf.keras.layers.Input(shape=(maxlen,))\nx = tf.keras.layers.Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\nx = tf.keras.layers.Bidirectional(tf.compat.v1.keras.layers.CuDNNGRU(64, return_sequences=True))(x)\nx = tf.keras.layers.GlobalMaxPooling1D()(x)\nx = tf.keras.layers.Dense(16, activation=\"relu\")(x)\nx = tf.keras.layers.Dropout(0.1)(x)\nx = tf.keras.layers.Dense(1, activation=\"sigmoid\")(x)\nmodel = tf.keras.models.Model(inputs=inp, outputs=x)\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**ENTRENAMIENTO**"},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(train_X, train_y, batch_size=bathsize, epochs=epoch, validation_data=(val_X, val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=0)\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score en el límite {0} es {1}\".format(thresh, metrics.f1_score(val_y, (pred_glove_val_y>thresh).astype(int))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Los resultados parecen ser mejores que el modelo sin incrustaciones pre-entrenadas, guardados las predicciones\n#pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Borrar de la memoria\ndel word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x\nimport gc; gc.collect()\ntime.sleep(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**IMPORTANDO EMBEDDING WIKI NEWS**"},{"metadata":{"trusted":true},"cell_type":"code","source":"EMBEDDING_FILE = '../input/quora-insincere-questions-classification/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**SE RESCONSTRUYE EL MODELO USANDO EL EMBEDDING PRE-ENTRENADO: WIKI NEWS**"},{"metadata":{"trusted":true},"cell_type":"code","source":"inp = tf.keras.layers.Input(shape=(maxlen,))\nx = tf.keras.layers.Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\nx = tf.keras.layers.Bidirectional(tf.compat.v1.keras.layers.CuDNNGRU(64, return_sequences=True))(x)\nx = tf.keras.layers.GlobalMaxPooling1D()(x)\nx = tf.keras.layers.Dense(16, activation=\"relu\")(x)\nx = tf.keras.layers.Dropout(0.1)(x)\nx = tf.keras.layers.Dense(1, activation=\"sigmoid\")(x)\nmodel = tf.keras.models.Model(inputs=inp, outputs=x)\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**ENTRENAMIENTO**"},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(train_X, train_y, batch_size=bathsize, epochs=epoch, validation_data=(val_X, val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_fasttext_val_y = model.predict([val_X], batch_size=1024, verbose=0)\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score en el límite {0} es {1}\".format(thresh, metrics.f1_score(val_y, (pred_fasttext_val_y>thresh).astype(int))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#pred_fasttext_test_y = model.predict([test_X], batch_size=1024, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Resultados:\n\nEl modelos en los que se utilizó embeding pre-entrenados dan mejores resultados en comparación con modelo sin embedding pre-entrenados.\n\nEl rendimiento de los diferentes embedding pre-entrenadas es bastante similar."}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}