{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport re\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom nltk.stem import WordNetLemmatizer\nfrom gensim.models import KeyedVectors\nfrom sklearn.model_selection import train_test_split\n\nimport keras\nfrom keras.models import Sequential, Model\nfrom keras.layers import Dense, Dropout, Embedding, SpatialDropout1D, GlobalAveragePooling1D, Bidirectional\nfrom keras.layers import GlobalMaxPooling1D, Input, concatenate, Lambda\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score, precision_score, recall_score\nfrom tensorflow.python.keras.layers import LSTM, CuDNNLSTM, GRU, CuDNNGRU\nfrom keras import callbacks\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/quora-insincere-questions-classification/train.csv')\ntrain_df, val_df = train_test_split(train_df, test_size=0.2) # 20%\ntest_df = pd.read_csv('../input/quora-insincere-questions-classification/test.csv')\ntrain_df.head(7)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Cleaning**"},{"metadata":{"trusted":true},"cell_type":"code","source":"puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\n\n## Puncts function\ndef clean_text(txt):\n    txt = str(txt)\n    for punct in puncts:\n        txt = txt.replace(punct, f' {punct} ')\n    return txt\n\n## Replace numbers with '#' function\ndef clean_numbers(txt):\n    txt = re.sub('[0-9]{5,}', '#####', txt)\n    txt = re.sub('[0-9]{4}', '####', txt)\n    txt = re.sub('[0-9]{3}', '###', txt)\n    txt = re.sub('[0-9]{2}', '##', txt)\n    return txt\n\n## change shortcuts words to there full form\ndef correct(txt):\n    txt = re.sub(\"ain't\", \"is not\", txt)\n    txt = re.sub(\"aren't\", \"are not\", txt)\n    txt = re.sub(\"can't\", \"cannot\", txt)\n    txt = re.sub(\"'cause\", \"because\", txt)\n    txt = re.sub(\"could've\", \"could have\", txt)\n    txt = re.sub(\"couldn't\", \"could not\", txt)\n    txt = re.sub(\"didn't\", \"did not\", txt)\n    txt = re.sub(\"doesn't\", \"does not\", txt)\n    txt = re.sub(\"don't\", \"do not\", txt)\n    txt = re.sub(\"hadn't\", \"had not\", txt)\n    txt = re.sub(\"hasn't\", \"has not\", txt)\n    txt = re.sub(\"haven't\", \"have not\", txt)\n    txt = re.sub(\"he'd\", \"he would\", txt)\n    txt = re.sub(\"he'll\", \"he will\", txt)\n    txt = re.sub(\"he's\", \"he is\", txt)\n    txt = re.sub(\"how'd\", \"how did\", txt)\n    txt = re.sub(\"how'd'y\", \"how do you\", txt)\n    txt = re.sub(\"how'll\", \"how will\", txt)\n    txt = re.sub(\"how's\", \"how is\", txt)\n    txt = re.sub(\"I'd\", \"I would\", txt)\n    txt = re.sub(\"I'd've\", \"I would have\", txt)\n    txt = re.sub(\"I'll\", \"I will\", txt)\n    txt = re.sub(\"I'll've\", \"I will have\", txt)\n    txt = re.sub(\"I'm\", \"I am\", txt)\n    txt = re.sub(\"I've\", \"I have\", txt)\n    txt = re.sub(\"i'd\", \"i would\", txt)\n    txt = re.sub(\"i'd've\", \"i would have\", txt)\n    txt = re.sub(\"i'll\", \"i will\", txt)\n    txt = re.sub(\"i'll've\", \"i will have\", txt)\n    txt = re.sub(\"i'm\", \"i am\", txt)\n    txt = re.sub(\"i've\", \"i have\", txt)\n    txt = re.sub(\"it'd\", \"it would\", txt)\n    txt = re.sub(\"it'd've\", \"it would have\", txt)\n    txt = re.sub(\"it'll\", \"it will\", txt)\n    txt = re.sub(\"it'll've\", \"it will have\", txt)\n    txt = re.sub(\"it's\", \"it is\", txt)\n    txt = re.sub(\"let's\", \"let us\", txt)\n    txt = re.sub(\"ma'am\", \"madam\", txt)\n    txt = re.sub(\"mayn't\", \"may not\", txt)\n    txt = re.sub(\"might've\", \"might have\", txt)\n    txt = re.sub(\"mightn't\", \"might not\", txt)\n    txt = re.sub(\"mightn't've\", \"might not have\", txt)\n    txt = re.sub(\"must've\", \"must have\", txt)\n    txt = re.sub(\"mustn't\", \"must not\", txt)\n    txt = re.sub(\"mustn't've\", \"must not have\", txt)\n    txt = re.sub(\"needn't\", \"need not\", txt)\n    txt = re.sub(\"needn't've\", \"need not have\", txt)\n    txt = re.sub(\"o'clock\", \"of the clock\", txt)\n    txt = re.sub(\"oughtn't\", \"ought not\", txt)\n    txt = re.sub(\"oughtn't've\", \"ought not have\", txt)\n    txt = re.sub(\"shan't\", \"shall not\", txt)\n    txt = re.sub(\"sha'n't\", \"shall not\", txt)\n    txt = re.sub(\"shan't've\", \"shall not have\", txt)\n    txt = re.sub(\"she'd\", \"she would\", txt)\n    txt = re.sub(\"she'd've\", \"she would have\", txt)\n    txt = re.sub(\"she'll\", \"she will\", txt)\n    txt = re.sub(\"she'll've\", \"she will have\", txt)\n    txt = re.sub(\"she's\", \"she is\", txt)\n    txt = re.sub(\"should've\", \"should have\", txt)\n    txt = re.sub(\"shouldn't\", \"should not\", txt)\n    txt = re.sub(\"shouldn't've\", \"should not have\", txt)\n    txt = re.sub(\"so've\", \"so have\", txt)\n    txt = re.sub(\"so's\", \"so as\", txt)\n    txt = re.sub(\"this's\", \"this is\", txt)\n    txt = re.sub(\"that'd\", \"that would\", txt)\n    txt = re.sub(\"that'd've\", \"that would have\", txt)\n    txt = re.sub(\"that's\", \"that is\", txt)\n    txt = re.sub(\"there'd\", \"there would\", txt)\n    txt = re.sub(\"there'd've\", \"there would have\", txt)\n    txt = re.sub(\"there's\", \"there is\", txt)\n    txt = re.sub(\"here's\", \"here is\", txt)\n    txt = re.sub(\"they'd\", \"they would\", txt)\n    txt = re.sub(\"they'd've\", \"they would have\", txt)\n    txt = re.sub(\"they'll\", \"they will\", txt)\n    txt = re.sub(\"they'll've\", \"they will have\", txt)\n    txt = re.sub(\"they're\", \"they are\", txt)\n    txt = re.sub(\"they've\", \"they have\", txt)\n    txt = re.sub(\"to've\", \"to have\", txt)\n    txt = re.sub(\"wasn't\", \"was not\", txt)\n    txt = re.sub(\"we'd\", \"we would\", txt)\n    txt = re.sub(\"we'd've\", \"we would have\", txt)\n    txt = re.sub(\"we'll\", \"we will\", txt)\n    txt = re.sub(\"we'll've\", \"we will have\", txt)\n    txt = re.sub(\"we've\", \"we have\", txt)\n    txt = re.sub(\"weren't\", \"were not\", txt)\n    txt = re.sub(\"what'll\", \"what will\", txt)\n    txt = re.sub(\"what'll've\", \"what will have\", txt)\n    txt = re.sub(\"what're\", \"what are\", txt)\n    txt = re.sub(\"what's\", \"what is\", txt)\n    txt = re.sub(\"what've\", \"what have\", txt)\n    txt = re.sub(\"when's\", \"when is\", txt)\n    txt = re.sub(\"when've\", \"when have\", txt)\n    txt = re.sub(\"where'd\", \"where did\", txt)\n    txt = re.sub(\"where's\", \"where is\", txt)\n    txt = re.sub(\"where've\", \"where have\", txt)\n    txt = re.sub(\"who'll\", \"who will\", txt)\n    txt = re.sub(\"who'll've\", \"who will have\", txt)\n    txt = re.sub(\"who's\", \"who is\", txt)\n    txt = re.sub(\"who've\", \"who have\", txt)\n    txt = re.sub(\"why's\", \"why is\", txt)\n    txt = re.sub(\"why've\", \"why have\", txt)\n    txt = re.sub(\"will've\", \"will have\", txt)\n    txt = re.sub(\"won't\", \"will not\", txt)\n    txt = re.sub(\"won't've\", \"will not have\", txt)\n    txt = re.sub(\"would've\", \"would have\", txt)\n    txt = re.sub(\"wouldn't\", \"would not\", txt)\n    txt = re.sub(\"wouldn't've\", \"would not have\", txt)\n    txt = re.sub(\"y'all\", \"you all\", txt)\n    txt = re.sub(\"y'all'd\", \"you all would\", txt)\n    txt = re.sub(\"y'all'd've\", \"you all would have\", txt)\n    txt = re.sub(\"y'all're\", \"you all are\", txt)\n    txt = re.sub(\"y'all've\", \"you all have\", txt)\n    txt = re.sub(\"you'd\", \"you would\", txt)\n    txt = re.sub(\"you'd've\", \"you would have\", txt)\n    txt = re.sub(\"you'll\", \"you will\", txt)\n    txt = re.sub(\"you'll've\", \"you will have\", txt)\n    txt = re.sub(\"you're\", \"you are\", txt)\n    txt = re.sub(\"you've\", \"you have\", txt)\n    return txt\n\n# To lower\ndef toLowCase(txt):\n    txt = txt.lower()\n    return txt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.question_text = train_df.question_text.apply(clean_text)\ntrain_df.question_text = train_df.question_text.apply(clean_numbers)\ntrain_df.question_text = train_df.question_text.apply(correct)\ntrain_df.question_text = train_df.question_text.apply(toLowCase)\ntrain_df.head(7)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.question_text = test_df.question_text.apply(clean_text)\ntest_df.question_text = test_df.question_text.apply(clean_numbers)\ntest_df.question_text = test_df.question_text.apply(correct)\ntest_df.question_text = test_df.question_text.apply(toLowCase)\ntest_df.head(7)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_df.question_text = val_df.question_text.apply(clean_text)\nval_df.question_text = val_df.question_text.apply(clean_numbers)\nval_df.question_text = val_df.question_text.apply(correct)\nval_df.question_text = val_df.question_text.apply(toLowCase)\nval_df.head(7)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Tokenisation"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Tokenizing the vocabulary\n#Tokenization is the process of dividing text into a set of meaningful pieces. These pieces are called tokens. For example, we can divide a chunk of text into words, or we can divide it into sentences.\n#Now we will tokenize our sentences and create the vocabulary. You could set a max. number for the word count in your vocalubary \n# how many unique words to use\nmax_features = 95000\n\n## Specifying how many words we're going to use and filtering the once that we won't use\ntokenizer = Tokenizer(num_words = max_features, filters='')\n\n## This will give you a list of integer sequences encoding the words in your sentence\n#fit_on_texts fits on sentences when list of sentences is passed to fit_on_texts() function. \n#ie - fit_on_texts( [ sent1, sent2, sent3,....sentN ] )\ntokenizer.fit_on_texts(list(train_df.question_text))\ntokenizer.fit_on_texts(list(test_df.question_text))\ntokenizer.fit_on_texts(list(val_df.question_text))\n\n# (texts_to_sequences) Transforms each text in texts to a sequence of integers. So it basically takes each word in the text and replaces it with its corresponding integer value from the word_index dictionary. Nothing more, nothing less, certainly no magic involved.\ntrain_X = tokenizer.texts_to_sequences(train_df.question_text)\ntest_X = tokenizer.texts_to_sequences(test_df.question_text)\nval_X = tokenizer.texts_to_sequences(val_df.question_text)\n\ntrain_Y = train_df.target.values\nval_Y = val_df.target.values\n\n# Create vocabulary from all words\n# vocabulary is used to create the tokenized input sentences. Finally, the tokens of these sentences are passed as inputs to the model\nvocabulary = tokenizer.word_index\nprint(train_X[:3])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Padding"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Padding sequences \n# Rememeber : The maximum length of our input can not be greater than 100 so we need to pad the incoming sequences at 100\n# maximum number of words in a question. This will be our input size\nmaxlen = 50\ntrain_X = pad_sequences(train_X, maxlen = maxlen, padding='post')\ntest_X = pad_sequences(test_X, maxlen = maxlen, padding='post')\nval_X = pad_sequences(val_X, maxlen = maxlen, padding='post')\nprint(train_X[:3])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Embedding"},{"metadata":{"trusted":true},"cell_type":"code","source":"!unzip ../input/quora-insincere-questions-classification/embeddings.zip","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"news_path = 'GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\nembedding_data = KeyedVectors.load_word2vec_format(news_path, binary=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\"# Load embedding\nembedding_data = KeyedVectors.load(\"../input/gensim-embeddings-dataset/paragram_300_sl999.gensim\")\"\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import operator \n\ndef check_coverage(vocab, embedding_item):\n\n    in_vocab = {}\n    out_of_vocab = {}\n    for word in vocab:\n        try:\n            in_vocab[word] = embedding_item[word]\n        except:\n            out_of_vocab[word] = vocab[word]\n            pass\n\n    print(f\"{len(in_vocab) / len(vocab):.0%} of vocabulary is covered\")\n    \n    sorted_x = sorted(out_of_vocab.items(), key=operator.itemgetter(1))[::-1]\n    \n    return sorted_x\n\noov = check_coverage(vocabulary, embedding_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov[:20]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_embedding_matrix(vocab, embedding_index):\n\n    # Initialize embedding matrix\n    embedding_matrix = np.zeros((len(vocab) + 1, 300))\n\n    print(embedding_matrix)\n    \n    print(\"-----------------------------------------------------\")\n    \n    for word, i in vocab.items(): \n        if word in embedding_index:\n            embedding_matrix[i] = embedding_index[word]\n            \n    print(embedding_matrix)\n\n    return embedding_matrix\n\nembedding_matrix = get_embedding_matrix(vocabulary, embedding_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embedding_matrix[:1]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# building our model\n\ndef model_builder():\n    inp = Input(shape=(maxlen,))\n    x = Embedding(len(embedding_matrix), 300)(inp)\n    \n    #Bidirectional LSTMs are an extension of traditional LSTMs that can improve model performance on sequence classification problems.\n    #In problems where all timesteps of the input sequence are available,\n    #Bidirectional LSTMs train two instead of one LSTMs on the input sequence\n    x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)\n    \n    #That return sequences return the hidden state output for each input time step.\n    x = GlobalAveragePooling1D()(x)\n    \n    #Activation functions are mathematical equations that determine the output of a neural network\n    #rerlu filtering the negative elements\n    x = Dense(16, activation = 'relu')(x)\n    \n    #drop 10% of the neurons\n    x = Dropout(0.1)(x)\n    \n    #transform the input into values between 0 and 1\n    x = Dense(1, activation = 'sigmoid')(x)\n    model = Model(inputs = inp, outputs = x)\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n    print(model.summary())\n    return model\n\nmodel = model_builder()\n    ## to prevent over-fitting\n## A fully connected layer occupies most of the parameters, and hence,\n## neurons develop co-dependency amongst each other during training\n## which curbs the individual power of each neuron leading to over-fitting of training data.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"e_stop = callbacks.EarlyStopping(mode='min', patience=3, restore_best_weights=True)\nmodel_checkp = callbacks.ModelCheckpoint('./w.h5', save_best_only=True, save_weights_only=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(train_X, train_Y, batch_size=1024, epochs=1, callbacks=[e_stop, model_checkp ], \n          validation_data=(val_X, val_Y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction = model.predict([test_X], batch_size=1024, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def pp(y_pred):\n    for k in range(len(y_pred)):\n        if y_pred[k]>0.5:\n            y_pred[k] = 1\n        else:\n            y_pred[k] = 0\n    return y_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction=pp(prediction)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df = pd.DataFrame()\nsub_df['qid'] = test_df.qid.values\nsub_df['prediction'] = prediction.astype(int)\nsub_df.to_csv(\"submission.csv\", columns=['qid', 'prediction'], index_label=False, index=False)\npd.read_csv('./submission.csv')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}