{"cells":[{"metadata":{"_uuid":"eb09242de11b37373b1976650a32b5dcbc5d0228"},"cell_type":"markdown","source":"# Quora Insincere Questions Challenge\nSelf-trained word embeddings using gensim word2vec and Keras CNN "},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# import packages\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport spacy\nimport nltk\nimport re\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score\n\nnp.random.seed(27)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"af20f5eb275e6089a7bc2f2f79585acaf8c18290"},"cell_type":"code","source":"# text cleaning\ncontractions = {\n\"ain't\": \"is not\",\n\"aren't\": \"are not\",\n\"can't\": \"cannot\",\n\"can't've\": \"cannot have\",\n\"'cause\": \"because\",\n\"could've\": \"could have\",\n\"couldn't\": \"could not\",\n\"couldn't've\": \"could not have\",\n\"didn't\": \"did not\",\n\"doesn't\": \"does not\",\n\"don't\": \"do not\",\n\"hadn't\": \"had not\",\n\"hadn't've\": \"had not have\",\n\"hasn't\": \"has not\",\n\"haven't\": \"have not\",\n\"he'd\": \"he would\",\n\"he'd've\": \"he would have\",\n\"he'll\": \"he will\",\n\"he'll've\": \"he he will have\",\n\"he's\": \"he is\",\n\"how'd\": \"how did\",\n\"how'd'y\": \"how do you\",\n\"how'll\": \"how will\",\n\"how's\": \"how is\",\n\"I'd\": \"I would\",\n\"I'd've\": \"I would have\",\n\"I'll\": \"I will\",\n\"I'll've\": \"I will have\",\n\"I'm\": \"I am\",\n\"I've\": \"I have\",\n\"i'd\": \"i would\",\n\"i'd've\": \"i would have\",\n\"i'll\": \"i will\",\n\"i'll've\": \"i will have\",\n\"i'm\": \"i am\",\n\"i've\": \"i have\",\n\"isn't\": \"is not\",\n\"it'd\": \"it would\",\n\"it'd've\": \"it would have\",\n\"it'll\": \"it will\",\n\"it'll've\": \"it will have\",\n\"it's\": \"it is\",\n\"let's\": \"let us\",\n\"ma'am\": \"madam\",\n\"mayn't\": \"may not\",\n\"might've\": \"might have\",\n\"mightn't\": \"might not\",\n\"mightn't've\": \"might not have\",\n\"must've\": \"must have\",\n\"mustn't\": \"must not\",\n\"mustn't've\": \"must not have\",\n\"needn't\": \"need not\",\n\"needn't've\": \"need not have\",\n\"o'clock\": \"of the clock\",\n\"oughtn't\": \"ought not\",\n\"oughtn't've\": \"ought not have\",\n\"shan't\": \"shall not\",\n\"sha'n't\": \"shall not\",\n\"shan't've\": \"shall not have\",\n\"she'd\": \"she would\",\n\"she'd've\": \"she would have\",\n\"she'll\": \"she will\",\n\"she'll've\": \"she will have\",\n\"she's\": \"she is\",\n\"should've\": \"should have\",\n\"shouldn't\": \"should not\",\n\"shouldn't've\": \"should not have\",\n\"so've\": \"so have\",\n\"so's\": \"so as\",\n\"that'd\": \"that would\",\n\"that'd've\": \"that would have\",\n\"that's\": \"that is\",\n\"there'd\": \"there would\",\n\"there'd've\": \"there would have\",\n\"there's\": \"there is\",\n\"they'd\": \"they would\",\n\"they'd've\": \"they would have\",\n\"they'll\": \"they will\",\n\"they'll've\": \"they will have\",\n\"they're\": \"they are\",\n\"they've\": \"they have\",\n\"to've\": \"to have\",\n\"wasn't\": \"was not\",\n\"we'd\": \"we would\",\n\"we'd've\": \"we would have\",\n\"we'll\": \"we will\",\n\"we'll've\": \"we will have\",\n\"we're\": \"we are\",\n\"we've\": \"we have\",\n\"weren't\": \"were not\",\n\"what'll\": \"what will\",\n\"what'll've\": \"what will have\",\n\"what're\": \"what are\",\n\"what's\": \"what is\",\n\"what've\": \"what have\",\n\"when's\": \"when is\",\n\"when've\": \"when have\",\n\"where'd\": \"where did\",\n\"where's\": \"where is\",\n\"where've\": \"where have\",\n\"who'll\": \"who will\",\n\"who'll've\": \"who will have\",\n\"who's\": \"who is\",\n\"who've\": \"who have\",\n\"why's\": \"why is\",\n\"why've\": \"why have\",\n\"will've\": \"will have\",\n\"won't\": \"will not\",\n\"won't've\": \"will not have\",\n\"would've\": \"would have\",\n\"wouldn't\": \"would not\",\n\"wouldn't've\": \"would not have\",\n\"y'all\": \"you all\",\n\"y'all'd\": \"you all would\",\n\"y'all'd've\": \"you all would have\",\n\"y'all're\": \"you all are\",\n\"y'all've\": \"you all have\",\n\"you'd\": \"you would\",\n\"you'd've\": \"you would have\",\n\"you'll\": \"you will\",\n\"you'll've\": \"you will have\",\n\"you're\": \"you are\",\n\"you've\": \"you have\"\n}\n\nc_re = re.compile('(%s)' % '|'.join(contractions.keys()))\n\ndef expandContractions(text, c_re=c_re):\n    def replace(match):\n        return contractions[match.group(0)]\n    return c_re.sub(replace, text)\n\nfrom gensim.parsing.preprocessing import preprocess_string\nfrom gensim.parsing.preprocessing import strip_tags, strip_punctuation, strip_numeric\nfrom gensim.parsing.preprocessing import strip_multiple_whitespaces, strip_non_alphanum, remove_stopwords, strip_short\n\nCUSTOM_FILTERS = [lambda x: x.lower(), #lowercase\n                  strip_tags, # remove html tags\n                  strip_punctuation, # replace punctuation with space\n                  strip_multiple_whitespaces,# remove repeating whitespaces\n                  strip_non_alphanum, # remove non-alphanumeric characters\n                  strip_numeric, # remove numbers\n                  remove_stopwords,# remove stopwords\n                  strip_short # remove words less than minsize=3 characters long\n                 ]\ndef gensim_preprocess(docs):\n    docs = [expandContractions(doc) for doc in docs]\n    docs = [preprocess_string(text, CUSTOM_FILTERS) for text in docs]\n    docs = [' '.join(text) for text in docs]\n    return pd.Series(docs)\n\ngensim_preprocess(train.question_text.iloc[10:15])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e4fa65e2cf7370c807b41f549067bf443b1ed6b7"},"cell_type":"code","source":"train['clean'] = gensim_preprocess(train.question_text)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9963a7574a5763b4561d110fd6833e220d085e96"},"cell_type":"code","source":"test['clean'] = gensim_preprocess(test.question_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"04900ea774f81ced844d8163f6cdd53ea440bf4c"},"cell_type":"code","source":"#define our vocab\nfrom collections import Counter\nvocab = Counter()\n\ntexts = ' '.join(train.clean).split()\nvocab.update(texts)\n\nprint(len(vocab))\nprint(vocab.most_common(50))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d84c45f8a59104275ab2d26c235bfffd8a90f5ad"},"cell_type":"code","source":"# keep tokens with a min occurrence\nmin_occurrence = 2\ntokens = [k for k,c in vocab.items() if c >= min_occurrence]\nprint(len(tokens))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"655f38cdf93e2624ae3eb9045dd7d9c3a9e46acb"},"cell_type":"code","source":"# get rid of duplicate words\nvocab = set((' '.join(tokens)).split())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4b166767b5d5a8171924a8c1f6ac40db4b351061"},"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.utils.vis_utils import plot_model\nfrom keras.models import Sequential\nfrom keras.layers import Dense\nfrom keras.layers import Flatten\nfrom keras.layers import Embedding\nfrom keras.layers.convolutional import Conv1D\nfrom keras.layers.convolutional import MaxPooling1D\n\n\n# fit a tokenizer\ndef create_tokenizer(text):\n    tokenizer = Tokenizer()\n    tokenizer.fit_on_texts(text)\n    return tokenizer\ntokenizer = create_tokenizer(train.clean)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"85bf5c7c8a1d9a0a135bd5e7a8a29b61734e64eb"},"cell_type":"code","source":"# find max length of training dataset\nmax_length = max([len(s.split()) for s in train.clean])\nmax_length","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5011d45bc20827afa190842fc8428bb60b2fc3fd"},"cell_type":"code","source":"#integer encode and pad documents\ndef encode_docs(tokenizer, max_length, docs):\n    # integer encode\n    encoded = tokenizer.texts_to_sequences(docs)\n    # pad sequences\n    padded = pad_sequences(encoded, maxlen = max_length, padding='post')\n    return padded\n\n\n# define vocab size\nvocab_size = len(tokenizer.word_index) + 1\nprint('Vocabulary size: ', vocab_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d53eb4e82d1f799bd3936615f1f453431640a317"},"cell_type":"code","source":"# define the model\ndef define_model(vocab_size, max_length):\n    model = Sequential()\n    model.add(Embedding(vocab_size, 100, input_length=max_length))\n    model.add(Conv1D(filters=32, kernel_size=8, activation='relu'))\n    model.add(MaxPooling1D(pool_size=2))\n    model.add(Flatten())\n    model.add(Dense(10, activation='relu'))\n    model.add(Dense(1, activation='sigmoid'))\n    # compile network\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) # summarize defined model\n    model.summary()\n    plot_model(model, to_file='model.png', show_shapes=True)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a7e7066665197754c746a346c8a34c53d6798c45"},"cell_type":"code","source":"vocab_size = len(tokenizer.word_index) + 1\nprint('Vocab Size: ', vocab_size)\nmax_length = max([len(s.split()) for s in train.clean])\nprint('Max Length: ', max_length)\nX_train = encode_docs(tokenizer, max_length, train.clean)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4ace56bb6d9468661c94e901cb5eecd1d26f00db"},"cell_type":"code","source":"model = define_model(vocab_size, max_length)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ff15f2285a41fc35a85b66cd6cadfdd2617caaef"},"cell_type":"code","source":"model.fit(X_train, train.target, epochs=5, verbose=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"81fe02d7ad1fa6e9344e868c15fd472f947032ab"},"cell_type":"code","source":"pred = encode_docs(tokenizer, max_length, test.clean)\nprediction = model.predict_classes(pred).ravel()\nsubmission = pd.DataFrame({'qid':test.qid, 'prediction':prediction})\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}