{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Importacion de Librerias"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport time\nimport numpy as np\nimport pandas as pd \nfrom tqdm import tqdm\nfrom keras.engine.topology import Layer\nimport math\nimport operator \nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn import metrics\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D, TimeDistributed, CuDNNLSTM,Conv2D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, GlobalAveragePooling1D, concatenate, Flatten, Reshape, AveragePooling2D, Average\nfrom keras.models import Model\nfrom keras.layers import Wrapper\nfrom keras.models import Model\nfrom keras.layers import Dense, Embedding, Bidirectional, CuDNNGRU, GlobalAveragePooling1D, GlobalMaxPooling1D, concatenate, Input, Dropout\nfrom keras.optimizers import Adam\nimport keras.backend as K\nimport matplotlib as plt\nfrom keras.optimizers import Adam\nfrom keras.callbacks import ModelCheckpoint, ReduceLROnPlateau\nfrom keras import initializers, regularizers, constraints, optimizers, layers\ntqdm.pandas()\nimport pandas as pd\nimport numpy as np\nimport operator \nimport re\nimport gc\nimport keras\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nsns.set_style('whitegrid')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"##  Carga y Dimensiones de Train y Test"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")\nprint(\"Train shape : \",train_df.shape)\nprint(\"Test shape : \",test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"##  Ver las frases tóxicas"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[train_df.target==1].head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"##  Ver las frases no tóxicas"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[train_df.target==0].head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Contar datos tóxicos vs no tóxicos"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['target'].value_counts().plot(kind = 'pie', labels = ['no tóxica 0', 'tóxica 1 '],\n     startangle = 90, autopct = '%1.0f%%')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Se puede apreciar que el 94% corresponde a frases no toxicas y el 6% para frases toxicas. No hay una cantidad buena cantidad de datos de palabras tóxicas, en comparacion a las no tóxicas. Esto va a afectar la precisión del modelo. Mejoraremos esto, preprocesando los datos"},{"metadata":{},"cell_type":"markdown","source":"# PARTE 1: PREPROCESAMIENTO"},{"metadata":{},"cell_type":"markdown","source":"### Cargar embeddings preentrenados"},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndef load_embed(file):\n    def get_coefs(word,*arr): \n        return word, np.asarray(arr, dtype='float32')\n    \n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file, encoding='latin'))\n    \n    return embeddings_index\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# No extraemos paragram y wikinews, Solo Glove debido a que ocupa demasiada memoria los otros dos y tardan demasiado en cargar"},{"metadata":{"trusted":true},"cell_type":"code","source":"glove = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n#paragram =  '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n#wiki_news = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Glove\nembed_glove = load_embed(glove)\n#Paragram\n#embed_paragram = load_embed(paragram)\n#FastText\n#embed_fasttext = load_embed(wiki_news)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Funciones de Vocabulario\n"},{"metadata":{},"cell_type":"markdown","source":"### Usamos la siguiente función para rastrear nuestro vocabulario de entrenamiento, que recorre todo nuestro texto y cuenta la ocurrencia de las palabras contenidas."},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_vocab(texts):\n    sentences = texts.apply(lambda x: x.split()).values\n    vocab = {}\n    for sentence in sentences:\n        for word in sentence:\n            try:\n                vocab[word] += 1\n            except KeyError:\n                vocab[word] = 1\n    return vocab","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Esta función comprueba la intersección entre nuestro vocabulario y los embeddings preentrandos. La salida es una lista de palabras fuera de vocabulario (oov)"},{"metadata":{"trusted":true},"cell_type":"code","source":"def check_coverage(vocab, embeddings_index):\n    known_words = {}\n    unknown_words = {}\n    nb_known_words = 0\n    nb_unknown_words = 0\n    for word in vocab.keys():\n        try:\n            known_words[word] = embeddings_index[word]\n            nb_known_words += vocab[word]\n        except:\n            unknown_words[word] = vocab[word]\n            nb_unknown_words += vocab[word]\n            pass\n\n    print('Se encontraron embeddings para {:.3%} of vocab'.format(len(known_words) / len(vocab)))\n    #print('Se encontraron embeddings para  {:.3%} of all text (palabras conocidas y desconocidas)'.format(nb_known_words / (nb_known_words + nb_unknown_words)))\n    unknown_words = sorted(unknown_words.items(), key=operator.itemgetter(1))[::-1]\n\n    return unknown_words","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Concatenamos en train df y el test df y creamos el vocabulario con las palabras del dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.concat([train_df ,test_df])\nvocab = build_vocab(df['question_text'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Glove : \")\noov_glove = check_coverage(vocab, embed_glove)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Pasar todo a minuscula"},{"metadata":{"trusted":true},"cell_type":"code","source":"df['question_text'] = df['question_text'].apply(lambda x: x.lower())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" ### Pasamos todas las letras a minúscula. Si aplica eso, pierde un poco de información en otros embedding:\n### Hay palabras conocidas que se conocen con letras superiores y desconocidas sin. Vamos a arreglar eso cpm word.lower () toma el embedding de word si word.lower () no tiene una incrustación. Para eso usamos una función. \n"},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_lower(embedding, vocab):\n    count = 0\n    for word in vocab:\n        if word in embedding and word.lower() not in embedding:  \n            embedding[word.lower()] = embedding[word]\n            count += 1\n    print(f\"Anadidas {count} palabras al embedding\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Glove : \")\noov_glove = check_coverage(vocab, embed_glove)\nadd_lower(embed_glove, vocab)\noov_glove = check_coverage(vocab, embed_glove)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### También eliminamos las palabras con contracciones y signos de puntuación. Eliminarlos ya que no aportan valor para la predicción de si es ofensivo o no.\n"},{"metadata":{},"cell_type":"markdown","source":"### oov_glove son palabras del diccionario que no estan en el embedding"},{"metadata":{"trusted":true},"cell_type":"code","source":"oov_glove[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"contraction_mapping = {\n    \"ain't\": \"is not\",\n    \"aren't\": \"are not\",\n    \"can't\": \"cannot\",\n    \"'cause\": \"because\",\n    \"could've\": \"could have\",\n    \"couldn't\": \"could not\",\n    \"didn't\": \"did not\",\n    \"doesn't\": \"does not\",\n    \"don't\": \"do not\",\n    \"hadn't\": \"had not\",\n    \"hasn't\": \"has not\",\n    \"haven't\": \"have not\",\n    \"he'd\": \"he would\",\n    \"he'll\": \"he will\",\n    \"he's\": \"he is\",\n    \"how'd\": \"how did\",\n    \"how'd'y\": \"how do you\",\n    \"how'll\": \"how will\",\n    \"how's\": \"how is\",\n    \"I'd\": \"I would\",\n    \"I'd've\": \"I would have\",\n    \"I'll\": \"I will\",\n    \"I'll've\": \"I will have\",\n    \"I'm\": \"I am\",\n    \"I've\": \"I have\",\n    \"i'd\": \"i would\",\n    \"i'd've\": \"i would have\",\n    \"i'll\": \"i will\",\n    \"i'll've\": \"i will have\",\n    \"i'm\": \"i am\",\n    \"i've\": \"i have\",\n    \"isn't\": \"is not\",\n    \"it'd\": \"it would\",\n    \"it'd've\": \"it would have\",\n    \"it'll\": \"it will\",\n    \"it'll've\": \"it will have\",\n    \"it's\": \"it is\",\n    \"let's\": \"let us\",\n    \"ma'am\": \"madam\",\n    \"mayn't\": \"may not\",\n    \"might've\": \"might have\",\n    \"mightn't\": \"might not\",\n    \"mightn't've\": \"might not have\",\n    \"must've\": \"must have\",\n    \"mustn't\": \"must not\",\n    \"mustn't've\": \"must not have\",\n    \"needn't\": \"need not\",\n    \"needn't've\": \"need not have\",\n    \"o'clock\": \"of the clock\",\n    \"oughtn't\": \"ought not\",\n    \"oughtn't've\": \"ought not have\",\n    \"shan't\": \"shall not\",\n    \"sha'n't\": \"shall not\",\n    \"shan't've\": \"shall not have\",\n    \"she'd\": \"she would\",\n    \"she'd've\": \"she would have\",\n    \"she'll\": \"she will\",\n    \"she'll've\": \"she will have\",\n    \"she's\": \"she is\",\n    \"should've\": \"should have\",\n    \"shouldn't\": \"should not\",\n    \"shouldn't've\": \"should not have\",\n    \"so've\": \"so have\",\n    \"so's\": \"so as\",\n    \"this's\": \"this is\",\n    \"that'd\": \"that would\",\n    \"that'd've\": \"that would have\",\n    \"that's\": \"that is\",\n    \"there'd\": \"there would\",\n    \"there'd've\": \"there would have\",\n    \"there's\": \"there is\",\n    \"here's\": \"here is\",\n    \"they'd\": \"they would\",\n    \"they'd've\": \"they would have\",\n    \"they'll\": \"they will\",\n    \"they'll've\": \"they will have\",\n    \"they're\": \"they are\",\n    \"they've\": \"they have\",\n    \"to've\": \"to have\",\n    \"wasn't\": \"was not\",\n    \"we'd\": \"we would\",\n    \"we'd've\": \"we would have\",\n    \"we'll\": \"we will\",\n    \"we'll've\": \"we will have\",\n    \"we're\": \"we are\",\n    \"we've\": \"we have\",\n    \"weren't\": \"were not\",\n    \"what'll\": \"what will\",\n    \"what'll've\": \"what will have\",\n    \"what're\": \"what are\",\n    \"what's\": \"what is\",\n    \"what've\": \"what have\",\n    \"when's\": \"when is\",\n    \"when've\": \"when have\",\n    \"where'd\": \"where did\",\n    \"where's\": \"where is\",\n    \"where've\": \"where have\",\n    \"who'll\": \"who will\",\n    \"who'll've\": \"who will have\",\n    \"who's\": \"who is\",\n    \"who've\": \"who have\",\n    \"why's\": \"why is\",\n    \"why've\": \"why have\",\n    \"will've\": \"will have\",\n    \"won't\": \"will not\",\n    \"won't've\": \"will not have\",\n    \"would've\": \"would have\",\n    \"wouldn't\": \"would not\",\n    \"wouldn't've\": \"would not have\",\n    \"y'all\": \"you all\",\n    \"y'all'd\": \"you all would\",\n    \"y'all'd've\": \"you all would have\",\n    \"y'all're\": \"you all are\",\n    \"y'all've\": \"you all have\",\n    \"you'd\": \"you would\",\n    \"you'd've\": \"you would have\",\n    \"you'll\": \"you will\",\n    \"you'll've\": \"you will have\",\n    \"you're\": \"you are\",\n    \"you've\": \"you have\",\n    \"who'd\": \"who would\",\n    \"who're\": \"who are\",\n    \"'re\": \" are\",\n    \"tryin'\": \"trying\",\n    \"doesn'\": \"does not\",\n    'howdo': 'how do',\n    'whatare': 'what are',\n    'howcan': 'how can',\n    'howmuch': 'how much',\n    'howmany': 'how many',\n    'whydo': 'why do',\n    'doI': 'do I',\n    'theBest': 'the best',\n    'howdoes': 'how does',\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def known_contractions(embed):\n    known = []\n    for contract in contraction_mapping:\n        if contract in embed:\n            known.append(contract)\n    return known","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"- Contracciones Conocidas  -\")\nprint(\"   Glove :\")\nprint(known_contractions(embed_glove))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### No se conocen muchas contracciones en Glove, entonces las mapearemos  con \"\""},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_contractions(text, mapping):\n    specials = [\"’\", \"‘\", \"´\", \"`\"]\n    for s in specials:\n        text = text.replace(s, \"'\")\n    text = ' '.join([mapping[t] if t in mapping else t for t in text.split(\" \")])\n    return text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['question_text'] = df['question_text'].apply(lambda x: clean_contractions(x, contraction_mapping))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Ahora hacemos lo mismo con los caracteres especiales"},{"metadata":{"trusted":true},"cell_type":"code","source":"punct = \"/-'?!.,#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~\" + '\"\"“”’' + '∞θ÷α•à−β∅³π‘₹´°£€\\×™√²—–&'\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def unknown_punct(embed, punct):\n    unknown = ''\n    for p in punct:\n        if p not in embed:\n            unknown += p\n            unknown += ' '\n    return unknown","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Glove :\")\nprint(unknown_punct(embed_glove, punct))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"punct_mapping = {\"‘\": \"'\", \"₹\": \"e\", \"´\": \"'\", \"°\": \"\", \"€\": \"e\", \"™\": \"tm\", \"√\": \" sqrt \", \"×\": \"x\", \"²\": \"2\", \"—\": \"-\", \"–\": \"-\", \"’\": \"'\", \"_\": \"-\", \"`\": \"'\", '“': '\"', '”': '\"', '“': '\"', \"£\": \"e\", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_special_chars(text, punct, mapping):\n    for p in mapping:\n        text = text.replace(p, mapping[p])\n    \n    for p in punct:\n        text = text.replace(p, f' {p} ')\n    \n    specials = {'\\u200b': ' ', '…': ' ... ', '\\ufeff': '', 'करना': '', 'है': ''}  \n    for s in specials:\n        text = text.replace(s, specials[s])\n    \n    return text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['question_text'] = df['question_text'].apply(lambda x: clean_special_chars(x, punct, punct_mapping))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vocab = build_vocab(df['question_text'])\nprint(\"Glove : \")\noov_glove = check_coverage(vocab, embed_glove)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov_glove[:100]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mispell_dict = {'advanatges': 'advantages', \n                'irrationaol': 'irrational' ,\n                'defferences': 'differences',\n                'lamboghini':'lamborghini',\n                'hypothical':'hypothetical',\n                'colour': 'color', \n                'centre': 'center', \n                'favourite': 'favorite',\n                'travelling': 'traveling',\n                'counselling': 'counseling', \n                'theatre': 'theater',\n                'cancelled': 'canceled', \n                'labour': 'labor',\n                'organisation': 'organization',\n                'wwii': 'world war 2',\n                'citicise': 'criticize', \n                'youtu ': 'youtube ', \n                'Qoura': 'Quora', \n                'sallary': 'salary',\n                'Whta': 'What', \n                'narcisist': 'narcissist',\n                'howdo': 'how do',\n                'whatare': 'what are', \n                'howcan': 'how can',\n                'howmuch': 'how much',\n                'howmany': 'how many',\n                'whydo': 'why do',\n                'doI': 'do I',\n                'theBest': 'the best',\n                'howdoes': 'how does',\n                'mastrubation': 'masturbation',\n                'mastrubate': 'masturbate',\n                \"mastrubating\": 'masturbating',\n                'pennis': 'penis',\n                'Etherium': 'Ethereum',\n                'narcissit': 'narcissist',\n                'bigdata': 'big data',\n                '2k17': '2017',\n                '2k18': '2018',\n                'qouta': 'quota',\n                'exboyfriend': 'ex boyfriend',\n                'airhostess': 'air hostess',\n                \"whst\": 'what',\n                'watsapp': 'whatsapp', \n                'demonitisation': 'demonetization',\n                'demonitization': 'demonetization',\n                'demonetisation': 'demonetization',\n                'pokémon': 'pokemon'}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def correct_spelling(x, dic):\n    for word in dic.keys():\n        x = x.replace(word, dic[word])\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['question_text'] = df['question_text'].apply(lambda x: correct_spelling(x, mispell_dict))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vocab = build_vocab(df['question_text'])\nprint(\"Glove : \")\noov_glove = check_coverage(vocab, embed_glove)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_numbers(x):\n\n    x = re.sub('[0-9]{5,}', ' number ', x)\n    x = re.sub('[0-9]{4}', ' number ', x)\n    x = re.sub('[0-9]{3}', ' number ', x)\n    x = re.sub('[0-9]{2}', ' number ', x)\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['question_text'] = df['question_text'].apply(lambda x: clean_numbers(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vocab = build_vocab(df['question_text'])\nprint(\"Glove : \")\noov_glove = check_coverage(vocab, embed_glove)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" ### Asignamos los cambios al set de train y test por separado, que antes lo haciamos al df con train y test pegados, para ver como mejoraba las palabras que aplicabamos al embedding"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Pasar a minusculas\ntrain_df['treated_question'] = train_df['question_text'].apply(lambda x: x.lower())\n# Eliminar Contracciones\ntrain_df['treated_question'] = train_df['treated_question'].apply(lambda x: clean_contractions(x, contraction_mapping))\n# Eliminar Caracteres Especiales\ntrain_df['treated_question'] = train_df['treated_question'].apply(lambda x: clean_special_chars(x, punct, punct_mapping))\n# Errores de Spelling\ntrain_df['treated_question'] = train_df['treated_question'].apply(lambda x: correct_spelling(x, mispell_dict))\n#Eliminar Numeros\ntrain_df['treated_question'] = train_df['treated_question'].apply(lambda x: clean_numbers(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Pasar a minusculas\ntest_df['treated_question'] = test_df['question_text'].apply(lambda x: x.lower())\n# Eliminar Contracciones\ntest_df['treated_question'] = test_df['treated_question'].apply(lambda x: clean_contractions(x, contraction_mapping))\n# Eliminar Caracteres Especiales\ntest_df['treated_question'] = test_df['treated_question'].apply(lambda x: clean_special_chars(x, punct, punct_mapping))\n# Errores de Spelling\ntest_df['treated_question'] = test_df['treated_question'].apply(lambda x: correct_spelling(x, mispell_dict))\n#Eliminar Numeros\ntest_df['treated_question'] = test_df['treated_question'].apply(lambda x: clean_numbers(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''def add_features(df):\n    \n    df['question_text'] = df['question_text'].progress_apply(lambda x:str(x))\n    df['total_length'] = df['question_text'].progress_apply(len)\n    df['capitals'] = df['question_text'].progress_apply(lambda comment: sum(1 for c in comment if c.isupper()))\n    df['caps_vs_length'] = df.progress_apply(lambda row: float(row['capitals'])/float(row['total_length']),\n                                axis=1)\n    df['num_words'] = df.question_text.str.count('\\S+')\n    df['num_unique_words'] = df['question_text'].progress_apply(lambda comment: len(set(w for w in comment.split())))\n    df['words_vs_unique'] = df['num_unique_words'] / df['num_words']  \n\n    return df'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train_df = add_features(train_df)\n#test_df = add_features(test_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Set de entrenamiento y validacion"},{"metadata":{"trusted":true},"cell_type":"code","source":"train, val = train_test_split(train_df, test_size=0.2, random_state=2)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Completar valores nulos (lo podiamos hacer antes)"},{"metadata":{"trusted":true},"cell_type":"code","source":"xtrain = train['question_text'].fillna('_na_').values\nxval = val['question_text'].fillna('_na_').values\nxtest = test_df['question_text'].fillna('_na_').values","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2da Fase Tokeninzar"},{"metadata":{},"cell_type":"markdown","source":"### Defino los siguientes hiperparametros"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Tamaño del vector embedding\nEMBED_SIZE = 300\n# Palabras únicas\nMAX_FEATURES = 100000 \n# Longitud máxima de la frase\nMAXLEN = 60 \n\n#Metodo para decirle de cuantas palabras va a truncar el diccionario al Tokenizar\ntokenizer = Tokenizer(num_words=MAX_FEATURES)\n\n#Metodo para contar la frecuencia de palabras y asignarle el indice segun la frecuencia \ntokenizer.fit_on_texts(list(xtrain))\n\nxtrain = tokenizer.texts_to_sequences(xtrain)\nxval = tokenizer.texts_to_sequences(xval)\nxtest = tokenizer.texts_to_sequences(xtest)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Padding\n#### Pad_sequences es una funcion de Keras para setear la longitud de la frase. Las secuencias que son más cortas que num_timesteps o maxlen se rellenan con un 0 al final. Las secuencias más largas que num_timesteps se truncan para que se ajusten a la longitud deseada. \n"},{"metadata":{"trusted":true},"cell_type":"code","source":"xtrain = pad_sequences(xtrain, maxlen=MAXLEN)\nxval = pad_sequences(xval, maxlen=MAXLEN)\nxtest = pad_sequences(xtest, maxlen=MAXLEN)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Guardo las variables y que incluye el 1 para toxicas y 0 para las no toxicas"},{"metadata":{"trusted":true},"cell_type":"code","source":"ytrain = train['target'].values\nyval = val['target'].values","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Matriz de Embeddings"},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_glove_matrix(word_index, embeddings_index):\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean, emb_std = all_embs.mean(), all_embs.std()\n    EMBED_SIZE = all_embs.shape[1]\n    \n    nb_words = min(MAX_FEATURES, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, EMBED_SIZE))\n\n    for word, i in word_index.items():\n        if i >= MAX_FEATURES:\n            continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[i] = embedding_vector\n\n    return embedding_matrix\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.random.seed(2)\n\ntrn_idx = np.random.permutation(len(xtrain))\nval_idx = np.random.permutation(len(xval))\n\nxtrain = xtrain[trn_idx]\nytrain = ytrain[trn_idx]\nxval = xval[val_idx]\nyval = yval[val_idx]\n\nembedding_matrix_glove = load_glove_matrix(tokenizer.word_index, embed_glove)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Attention(Layer):\n    def __init__(self, step_dim,\n                 W_regularizer=None, b_regularizer=None,\n                 W_constraint=None, b_constraint=None,\n                 bias=True, **kwargs):\n        self.supports_masking = True\n        self.init = initializers.get('glorot_uniform')\n\n        self.W_regularizer = regularizers.get(W_regularizer)\n        self.b_regularizer = regularizers.get(b_regularizer)\n\n        self.W_constraint = constraints.get(W_constraint)\n        self.b_constraint = constraints.get(b_constraint)\n\n        self.bias = bias\n        self.step_dim = step_dim\n        self.features_dim = 0\n        super(Attention, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight((input_shape[-1],),\n                                 initializer=self.init,\n                                 name='{}_W'.format(self.name),\n                                 regularizer=self.W_regularizer,\n                                 constraint=self.W_constraint)\n        self.features_dim = input_shape[-1]\n\n        if self.bias:\n            self.b = self.add_weight((input_shape[1],),\n                                     initializer='zero',\n                                     name='{}_b'.format(self.name),\n                                     regularizer=self.b_regularizer,\n                                     constraint=self.b_constraint)\n        else:\n            self.b = None\n\n        self.built = True\n\n    def compute_mask(self, input, input_mask=None):\n        return None\n\n    def call(self, x, mask=None):\n        features_dim = self.features_dim\n        step_dim = self.step_dim\n\n        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n\n        if self.bias:\n            eij += self.b\n\n        eij = K.tanh(eij)\n        a = K.exp(eij)\n\n        if mask is not None:\n            a *= K.cast(mask, K.floatx())\n\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n\n        a = K.expand_dims(a)\n        weighted_input = x * a\n        return K.sum(weighted_input, axis=1)\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[0], self.features_dim\ndef f1(y_true, y_pred):\n\n    def recall(y_true, y_pred):\n        \n        true_positives = K.sum(K.round(K.clip(y_true*y_pred, 0, 1)))\n        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n        recall = true_positives/(possible_positives + K.epsilon())\n        return recall\n\n    def precision(y_true, y_pred):\n        \n        true_positives = K.sum(K.round(K.clip(y_true*y_pred, 0, 1)))\n        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n        precision = true_positives/(predicted_positives + K.epsilon())\n        return precision\n\n    precision = precision(y_true, y_pred)\n    recall = recall(y_true, y_pred)\n\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def f1(y_true, y_pred):\n\n    def recall(y_true, y_pred):\n        \n        true_positives = K.sum(K.round(K.clip(y_true*y_pred, 0, 1)))\n        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n        recall = true_positives/(possible_positives + K.epsilon())\n        return recall\n\n    def precision(y_true, y_pred):\n        \n        true_positives = K.sum(K.round(K.clip(y_true*y_pred, 0, 1)))\n        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n        precision = true_positives/(predicted_positives + K.epsilon())\n        return precision\n\n    precision = precision(y_true, y_pred)\n    recall = recall(y_true, y_pred)\n\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def model_lstm_att(embedding_matrix):\n    \n    inp = Input(shape=(MAXLEN,))\n    x = Embedding(MAX_FEATURES, EMBED_SIZE, weights=[embedding_matrix], trainable=False)(inp)\n    x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)\n    x = Bidirectional(CuDNNLSTM(32, return_sequences=True))(x)\n    \n    att = Attention(MAXLEN)(x)\n    \n    y = Dense(32, activation='relu')(att)\n    #y = Dense(32, activation='relu')(y)\n    \n    y = Dropout(0.1)(y)\n    outp = Dense(1, activation='sigmoid')(y)    \n\n    model = Model(inputs=inp, outputs=outp)\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['binary_accuracy', 'accuracy'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"paragram = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\nembedding_matrix_para = load_glove_matrix(tokenizer.word_index, load_embed(paragram))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\n#### Al inicio nosotros dijimos que  Glove y Paragram presentan una proporción aceptable de palabras conocidas. Por lo tanto, para tener la mayor cantidad posible de embeddings, usamos ambas como incrustaciones. Se puede concatenar, sumar o promediar estos dos embeddings. Nosotros los promediamos."},{"metadata":{"trusted":true},"cell_type":"code","source":"embedding_matrix = np.mean([embedding_matrix_glove, embedding_matrix_para], axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_lstm = model_lstm_att(embedding_matrix)\nmodel_lstm.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''checkpoints = ModelCheckpoint('weights.hdf5', monitor=\"val_f1\", mode=\"max\", verbose=True, save_best_only=True)\nreduce_lr = ReduceLROnPlateau(monitor='val_f1', factor=0.1, patience=2, verbose=1, min_lr=0.000001)'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''epochs = 8\nbatch_size = 512'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''history = model_lstm.fit(xtrain, ytrain, batch_size=batch_size, epochs=epochs, \n                    validation_data=[xval, yval], callbacks=[checkpoints, reduce_lr])'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_pred(model, epochs=2):\n    \n    for e in range(epochs):\n        model.fit(xtrain, ytrain, batch_size=512, epochs=3, validation_data=(xval, yval))\n        pred_val_y = model.predict([xval], batch_size=1024, verbose=1)\n\n        best_thresh = 0.5\n        best_score = 0.0\n        for thresh in np.arange(0.1, 0.501, 0.01):\n            thresh = np.round(thresh, 2)\n            score = metrics.f1_score(yval, (pred_val_y > thresh).astype(int))\n            if score > best_score:\n                best_thresh = thresh\n                best_score = score\n\n        print(\"Val puntuacion F1: {:.4f}\".format(best_score))\n\n    pred_test_y = model.predict([xtest], batch_size=1024, verbose=1)\n\n    return pred_val_y, pred_test_y, best_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"outputs = []\npred_val_y, pred_test_y, best_score = train_pred(model_lstm, epochs=2)\noutputs.append([pred_val_y, pred_test_y, best_score, 'model_lstm_att only Glove'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Grafica de Accuracy"},{"metadata":{"trusted":true},"cell_type":"code","source":"'''plt.figure(figsize=(12,8))\nplt.plot(history.history['acc'], label='Entrenamiento Accuracy')\nplt.plot(history.history['val_acc'], label='Validacion Accuracy')\nplt.show()'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"outputs.sort(key=lambda x: x[2]) \nweights = [i for i in range(1, len(outputs) + 1)]\nweights = [float(i) / sum(weights) for i in weights] \n\npred_val_y = np.mean([outputs[i][0] for i in range(len(outputs))], axis = 0)\n\nthresholds = []\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    res = metrics.f1_score(yval, (pred_val_y > thresh).astype(int))\n    thresholds.append([thresh, res])\n    print(\"La puntuación F1 en los limites {0} y {1}\".format(thresh, res))\n    \nthresholds.sort(key=lambda x: x[1], reverse=True)\nbest_thresh = thresholds[0][0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Mejor limite:\", best_thresh, \"y puntuacion F1 \", thresholds[0][1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_test_y = np.mean([outputs[i][1] for i in range(len(outputs))], axis = 0)\npred_test_y = (pred_test_y > best_thresh).astype(int)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Guardar resultados predichos"},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv('../input/sample_submission.csv')\nout_df = pd.DataFrame({\"qid\":sub[\"qid\"].values})\nout_df['prediction'] = pred_test_y\nout_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}