{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Librerias"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport os\nimport time\nimport math\nimport operator \nimport re\nimport gc\nimport numpy as np\nimport pandas as pd\nimport keras\nimport tensorflow as tf\nimport keras.backend as K\nimport matplotlib\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom sklearn import metrics\nfrom sklearn.model_selection import train_test_split\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Input, Embedding, Dropout, Dense, Flatten, Activation, CuDNNGRU, CuDNNLSTM, Bidirectional, Average, SpatialDropout1D, Average\nfrom keras.models import Model\nfrom keras.optimizers import Adam\nfrom keras import initializers, regularizers, constraints, optimizers, layers\ntqdm.pandas()\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Carga de Datos"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Cargando el Embedding"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Adapted from Viel (2018)\nglove_path = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\ndef load_embed(file):\n    def get_coefs(word,*arr): \n        return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file, encoding='latin'))    \n    return embeddings_index\nprint('Loading Glove...')\nembed_glove = load_embed(glove_path)\nprint('Glove Loaded')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Limpienza de Datos"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Diccionarios. Viel (2018)\ncontraction_mapping = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\" }\npunct_mapping = {\"‘\": \"'\", \"₹\": \"e\", \"´\": \"'\", \"°\": \"\", \"€\": \"e\", \"™\": \"tm\", \"√\": \" sqrt \", \"×\": \"x\", \"²\": \"2\", \"—\": \"-\", \"–\": \"-\", \"’\": \"'\", \"_\": \"-\", \"`\": \"'\", '“': '\"', '”': '\"', '“': '\"', \"£\": \"e\", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi'}\nmispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \"whst\": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization', 'pokémon': 'pokemon'}\npunct = \"/-'?!.,#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~\" + '\"\"“”’' + '∞θ÷α•à−β∅³π‘₹´°£€\\×™√²—–&'\n# Funciones Auxiliares\ndef clean_contractions(text, mapping):\n    specials = [\"’\", \"‘\", \"´\", \"`\"]\n    for s in specials:\n        text = text.replace(s, \"'\")\n    text = ' '.join([mapping[t] if t in mapping else t for t in text.split(\" \")])\n    return text\n\ndef unknown_punct(embed, punct):\n    unknown = ''\n    for p in punct:\n        if p not in embed:\n            unknown += p\n            unknown += ' '\n    return unknown\n\ndef clean_special_chars(text, punct, mapping):\n    for p in mapping:\n        text = text.replace(p, mapping[p])\n    \n    for p in punct:\n        text = text.replace(p, f' {p} ')\n    \n    specials = {'\\u200b': ' ', '…': ' ... ', '\\ufeff': '', 'करना': '', 'है': ''}  # Other special characters that I have to deal with in last\n    for s in specials:\n        text = text.replace(s, specials[s])\n    \n    return text\n\ndef correct_spelling(x, dic):\n    for word in dic.keys():\n        x = x.replace(word, dic[word])\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Pasar a Minusculas los datos\ntrain['treated_question'] = train['question_text'].apply(lambda x: x.lower())\ntest['treated_question'] = test['question_text'].apply(lambda x: x.lower())\n# Eliminar las Contracciones\ntrain['treated_question'] = train['treated_question'].apply(lambda x: clean_contractions(x, contraction_mapping))\ntest['treated_question'] = test['treated_question'].apply(lambda x: clean_contractions(x, contraction_mapping))\n# Eliminar los Caracteres Especiales\ntrain['treated_question'] = train['treated_question'].apply(lambda x: clean_special_chars(x, punct, punct_mapping))\ntest['treated_question'] = test['treated_question'].apply(lambda x: clean_special_chars(x, punct, punct_mapping))\n# Eliminar los Errores Ortograficos\ntest['treated_question'] = test['treated_question'].apply(lambda x: correct_spelling(x, mispell_dict))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Tokenizacion"},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"VOCABULARY=100000\ntokenizer = Tokenizer(num_words=VOCABULARY, char_level=False, oov_token='<OOV>')\ntokenizer.fit_on_texts(list(train['treated_question']))\nX = tokenizer.texts_to_sequences(train['treated_question'])\nX = pad_sequences(X, maxlen=50)\nY = train['target'].values","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Splitting"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_dev, Y_train, Y_dev = train_test_split(X, Y, test_size=0.05, random_state=27014)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Matriz de Embedding"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Funcion de Viel (2018)\ndef make_embed_matrix(embeddings_index, word_index, len_voc):\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n    word_index = word_index\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (len_voc, embed_size))\n    \n    for word, i in word_index.items():\n        if i >= len_voc:\n            continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: \n            embedding_matrix[i] = embedding_vector\n    \n    return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embedding = make_embed_matrix(embed_glove, tokenizer.word_index, VOCABULARY)\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Implementacion de Attention"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Attention(keras.layers.Layer):\n    def __init__(self, step_dim,\n                 W_regularizer=None, b_regularizer=None,\n                 W_constraint=None, b_constraint=None,\n                 bias=True, **kwargs):\n        self.supports_masking = True\n        self.init = initializers.get('glorot_uniform')\n\n        self.W_regularizer = regularizers.get(W_regularizer)\n        self.b_regularizer = regularizers.get(b_regularizer)\n\n        self.W_constraint = constraints.get(W_constraint)\n        self.b_constraint = constraints.get(b_constraint)\n\n        self.bias = bias\n        self.step_dim = step_dim\n        self.features_dim = 0\n        super(Attention, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight((input_shape[-1],),\n                                 initializer=self.init,\n                                 name='{}_W'.format(self.name),\n                                 regularizer=self.W_regularizer,\n                                 constraint=self.W_constraint)\n        self.features_dim = input_shape[-1]\n\n        if self.bias:\n            self.b = self.add_weight((input_shape[1],),\n                                     initializer='zero',\n                                     name='{}_b'.format(self.name),\n                                     regularizer=self.b_regularizer,\n                                     constraint=self.b_constraint)\n        else:\n            self.b = None\n\n        self.built = True\n\n    def compute_mask(self, input, input_mask=None):\n        return None\n\n    def call(self, x, mask=None):\n        features_dim = self.features_dim\n        step_dim = self.step_dim\n\n        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n\n        if self.bias:\n            eij += self.b\n\n        eij = K.tanh(eij)\n        a = K.exp(eij)\n\n        if mask is not None:\n            a *= K.cast(mask, K.floatx())\n\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n\n        a = K.expand_dims(a)\n        weighted_input = x * a\n        return K.sum(weighted_input, axis=1)\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[0], self.features_dim\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Modelo"},{"metadata":{"trusted":true},"cell_type":"code","source":"def model(embedding_matrix, EMBED_SIZE=300, MAX_LEN=50, VOCABULARY=100000):\n    inp = Input(shape=(MAX_LEN, ), name='input')\n    X = Embedding(VOCABULARY, EMBED_SIZE, trainable=True, weights=[embedding_matrix], name='Embedding')(inp)\n    X = Bidirectional(CuDNNLSTM(128, return_sequences=True, name='LSTM'), name='bid1')(X)\n    A_LSTM = Attention(MAX_LEN)(X)\n    X = SpatialDropout1D(0.35)(X)\n    X = Bidirectional(CuDNNGRU(128, return_sequences=True, name='GRU'), name='bid2')(X)\n    A_GRU = Attention(MAX_LEN)(X)\n    A = Average()([A_LSTM, A_GRU, X])\n    A = Flatten()(A)\n    A = Dense(256, activation='relu', name='fc1')(A)\n    A = Dropout(0.2)(A)\n    A = Dense(128, activation='relu', name='fc2')(A)\n    O = Dense(1, activation='sigmoid', name='output')(A)\n    model = Model(inputs=inp, outputs=O)\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['binary_accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = model(embedding)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"t_start = time.time()\nhistory = model.fit(x=X_train, y=Y_train, batch_size=512, epochs=5,validation_data=(X_dev, Y_dev))\nprint(t_start - time.time())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def graf_model(train_history):\n    f = plt.figure(figsize=(15,10))\n    ax = f.add_subplot(121)\n    ax2 = f.add_subplot(122)\n    # summarize history for accuracy\n    ax.plot(train_history.history['binary_accuracy'])\n    ax.plot(train_history.history['val_binary_accuracy'])\n    ax.set_title('model accuracy')\n    ax.set_ylabel('accuracy')\n    ax.set_xlabel('epoch')\n    ax.legend(['train', 'test'], loc='upper left')\n    # summarize history for loss\n    ax2.plot(train_history.history['loss'])\n    ax2.plot(train_history.history['val_loss'])\n    ax2.set_title('model loss')\n    ax2.set_ylabel('loss')\n    ax2.set_xlabel('epoch')\n    ax2.legend(['train', 'test'], loc='upper left')\n    plt.show()\ngraf_model(history)    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_valid = tokenizer.texts_to_sequences(test['treated_question'])\nX_valid = pad_sequences(X_valid, maxlen=50)\nY_hat = model.predict(X_valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Y_hat = Y_hat.ravel()\nfor i in range(len(Y_hat)):\n    if Y_hat[i] < 0.5: \n        Y_hat[i] = 0\n    else:\n        Y_hat[i] = 1\nsub = pd.DataFrame({ 'qid': test['qid'].values, 'prediction': Y_hat })\nsub['prediction'] = sub['prediction'].astype('int32')\nsub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Bibliografia\n\n1. [Wang, A., Ranganathan, V. (2019). Is this question sincere? Identifying insincere questions on Quora using BERT and variations](http://web.stanford.edu/class/cs224n/reports/custom/15763730.pdf)\n2. [Kumar, A. (2019). Quora Insincere Question Classification](https://mc.ai/quora-insincere-question-classification/)\n3. [Viel, T. (2018). Improve your Score with some Text Preprocessing](https://www.kaggle.com/theoviel/improve-your-score-with-text-preprocessing-v2)"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}