{"cells":[{"metadata":{"_uuid":"eb721e8c304d6beeaecf948903154e25bddbb53a"},"cell_type":"markdown","source":"# CNN + Google News Vectors\nThanks to [Dieter's Kernel](https://www.kaggle.com/christofhenkel/how-to-preprocessing-when-using-embeddings) for preprocessing with embeddings tutorial."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\nimport re\ntqdm.pandas()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")\nprint(\"Train shape : \",train.shape)\nprint(\"Test shape : \",test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f81231d640d4ec4916570a19a1f1752aa9732de"},"cell_type":"code","source":"# text preprocessing\ncontractions = {\n\"ain't\": \"is not\",\n\"aren't\": \"are not\",\n\"can't\": \"cannot\",\n\"can't've\": \"cannot have\",\n\"'cause\": \"because\",\n\"could've\": \"could have\",\n\"couldn't\": \"could not\",\n\"couldn't've\": \"could not have\",\n\"didn't\": \"did not\",\n\"doesn't\": \"does not\",\n\"don't\": \"do not\",\n\"hadn't\": \"had not\",\n\"hadn't've\": \"had not have\",\n\"hasn't\": \"has not\",\n\"haven't\": \"have not\",\n\"he'd\": \"he would\",\n\"he'd've\": \"he would have\",\n\"he'll\": \"he will\",\n\"he'll've\": \"he he will have\",\n\"he's\": \"he is\",\n\"how'd\": \"how did\",\n\"how'd'y\": \"how do you\",\n\"how'll\": \"how will\",\n\"how's\": \"how is\",\n\"I'd\": \"I would\",\n\"I'd've\": \"I would have\",\n\"I'll\": \"I will\",\n\"I'll've\": \"I will have\",\n\"I'm\": \"I am\",\n\"I've\": \"I have\",\n\"i'd\": \"i would\",\n\"i'd've\": \"i would have\",\n\"i'll\": \"i will\",\n\"i'll've\": \"i will have\",\n\"i'm\": \"i am\",\n\"i've\": \"i have\",\n\"isn't\": \"is not\",\n\"it'd\": \"it would\",\n\"it'd've\": \"it would have\",\n\"it'll\": \"it will\",\n\"it'll've\": \"it will have\",\n\"it's\": \"it is\",\n\"let's\": \"let us\",\n\"ma'am\": \"madam\",\n\"mayn't\": \"may not\",\n\"might've\": \"might have\",\n\"mightn't\": \"might not\",\n\"mightn't've\": \"might not have\",\n\"must've\": \"must have\",\n\"mustn't\": \"must not\",\n\"mustn't've\": \"must not have\",\n\"needn't\": \"need not\",\n\"needn't've\": \"need not have\",\n\"o'clock\": \"of the clock\",\n\"oughtn't\": \"ought not\",\n\"oughtn't've\": \"ought not have\",\n\"shan't\": \"shall not\",\n\"sha'n't\": \"shall not\",\n\"shan't've\": \"shall not have\",\n\"she'd\": \"she would\",\n\"she'd've\": \"she would have\",\n\"she'll\": \"she will\",\n\"she'll've\": \"she will have\",\n\"she's\": \"she is\",\n\"should've\": \"should have\",\n\"shouldn't\": \"should not\",\n\"shouldn't've\": \"should not have\",\n\"so've\": \"so have\",\n\"so's\": \"so as\",\n\"that'd\": \"that would\",\n\"that'd've\": \"that would have\",\n\"that's\": \"that is\",\n\"there'd\": \"there would\",\n\"there'd've\": \"there would have\",\n\"there's\": \"there is\",\n\"they'd\": \"they would\",\n\"they'd've\": \"they would have\",\n\"they'll\": \"they will\",\n\"they'll've\": \"they will have\",\n\"they're\": \"they are\",\n\"they've\": \"they have\",\n\"to've\": \"to have\",\n\"wasn't\": \"was not\",\n\"we'd\": \"we would\",\n\"we'd've\": \"we would have\",\n\"we'll\": \"we will\",\n\"we'll've\": \"we will have\",\n\"we're\": \"we are\",\n\"we've\": \"we have\",\n\"weren't\": \"were not\",\n\"what'll\": \"what will\",\n\"what'll've\": \"what will have\",\n\"what're\": \"what are\",\n\"what's\": \"what is\",\n\"what've\": \"what have\",\n\"when's\": \"when is\",\n\"when've\": \"when have\",\n\"where'd\": \"where did\",\n\"where's\": \"where is\",\n\"where've\": \"where have\",\n\"who'll\": \"who will\",\n\"who'll've\": \"who will have\",\n\"who's\": \"who is\",\n\"who've\": \"who have\",\n\"why's\": \"why is\",\n\"why've\": \"why have\",\n\"will've\": \"will have\",\n\"won't\": \"will not\",\n\"won't've\": \"will not have\",\n\"would've\": \"would have\",\n\"wouldn't\": \"would not\",\n\"wouldn't've\": \"would not have\",\n\"y'all\": \"you all\",\n\"y'all'd\": \"you all would\",\n\"y'all'd've\": \"you all would have\",\n\"y'all're\": \"you all are\",\n\"y'all've\": \"you all have\",\n\"you'd\": \"you would\",\n\"you'd've\": \"you would have\",\n\"you'll\": \"you will\",\n\"you'll've\": \"you will have\",\n\"you're\": \"you are\",\n\"you've\": \"you have\"\n}\n\nc_re = re.compile('(%s)' % '|'.join(contractions.keys()))\n\ndef expandContractions(text, c_re=c_re):\n    def replace(match):\n        return contractions[match.group(0)]\n    return c_re.sub(replace, text)\n\nfrom gensim.parsing.preprocessing import preprocess_string\nfrom gensim.parsing.preprocessing import strip_tags, strip_punctuation, strip_numeric\nfrom gensim.parsing.preprocessing import strip_multiple_whitespaces, strip_non_alphanum, remove_stopwords, strip_short\n\nCUSTOM_FILTERS = [#lambda x: x.lower(), #lowercase\n                  strip_tags, # remove html tags\n                  #strip_punctuation, # replace punctuation with space\n                  strip_multiple_whitespaces,# remove repeating whitespaces\n                  strip_non_alphanum, # remove non-alphanumeric characters\n                  #strip_numeric, # remove numbers\n                  #remove_stopwords,# remove stopwords\n                  strip_short # remove words less than minsize=3 characters long\n                 ]\ndef gensim_preprocess(docs):\n    docs = [expandContractions(doc) for doc in docs]\n    docs = [preprocess_string(text, CUSTOM_FILTERS) for text in docs]\n    docs = [' '.join(text) for text in docs]\n    return pd.Series(docs)\n\ntrain_clean = gensim_preprocess(train.question_text)\n\ngensim_preprocess(train.question_text.iloc[10:15])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"80515b1714cd847c2f5a90f501d5f10290dd4af8"},"cell_type":"code","source":"# creating vocab from train dataframe\nfrom collections import Counter\nvocab = Counter()\n\ntexts = ' '.join(train_clean).split()\nvocab.update(texts)\n\nprint(len(vocab))\nprint(vocab.most_common(50))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e01724db7dc07927f0c7246edc08bfdea1c54c0e"},"cell_type":"code","source":"# load google news vectors\nfrom gensim.models import KeyedVectors\nnews_path = '../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\nembeddings_index = KeyedVectors.load_word2vec_format(news_path, binary=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c5a8d54d8670ea0e54726ad09d28e6090a0cd5f6"},"cell_type":"code","source":"# function to check coverage of embedding vs train vocabulary\nimport operator \n\ndef check_coverage(vocab,embeddings_index):\n    a = {}\n    oov = {}\n    k = 0\n    i = 0\n    for word in tqdm(vocab):\n        try:\n            a[word] = embeddings_index[word]\n            k += vocab[word]\n        except:\n\n            oov[word] = vocab[word]\n            i += vocab[word]\n            pass\n\n    print('Found embeddings for {:.2%} of vocab'.format(len(a) / len(vocab)))\n    print('Found embeddings for  {:.2%} of all text'.format(k / (k + i)))\n    sorted_x = sorted(oov.items(), key=operator.itemgetter(1))[::-1]\n\n    return sorted_x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"13a63f9a6eda70696968fd94032dc64cafeadc0d"},"cell_type":"code","source":"oov = check_coverage(vocab,embeddings_index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"33145b852cb128a8f03088582b4ee06d7a1741a2"},"cell_type":"code","source":"# view out of vocab words\noov[:20]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c050f63f248cef758cd2328e044c8ee9160343a8"},"cell_type":"code","source":"# function to correct misspellings and out of vocab words\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\n\nmispell_dict = {'colour':'color',\n                'centre':'center',\n                'favourite':'favorite',\n                'travelling':'traveling',\n                'counselling':'counseling',\n                'theatre':'theater',\n                'cancelled':'canceled',\n                'labour':'labor',\n                'organisation':'organization',\n                'wwii':'world war 2',\n                'citicise':'criticize',\n                'instagram': 'social medium',\n                'whatsapp': 'social medium',\n                'snapchat': 'social medium',\n                'Snapchat': 'social medium',\n                'quora': 'social medium',\n                'Quora': 'social medium',\n                'mediumns': 'mediums',\n                'bitcoin': 'currency',\n                'cryptocurrency': 'currency',\n                'upsc': 'union public service commission',\n                'mbbs': 'bachelor medicine',\n                'ece': 'educational credential evaluators',\n                'aiims': 'all india institute medical science',\n                'iim': 'india institute management',\n                'sbi': 'state bank india',\n                'blockchain': 'crytography',\n                'and': '',\n                'reducational':'educational',\n                'neducational':'educational',\n                'greeducational': 'greed educational',\n                'pieducational': 'educational',\n                'deducational': 'educational',\n                'Quorans': 'Quoran'   \n                }\nmispellings, mispellings_re = _get_mispell(mispell_dict)\n\ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n\n    return mispellings_re.sub(replace, text)\n\n# replace numbers > 9 with #### to match embedding\ndef clean_numbers(x):\n\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    return x\n\ntrain_clean = train_clean.apply(lambda x: replace_typical_misspell(x))\ntrain_clean = train_clean.apply(lambda x: clean_numbers(x))\n\nvocab = Counter()\ntexts = ' '.join(train_clean).split()\nvocab.update(texts)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e8ff208ed3b95f97cdb89c45b7071f95d1e32a5a"},"cell_type":"code","source":"# check out of vocab words again\noov = check_coverage(vocab,embeddings_index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f4800e22d86ffd281b0fcfdc64ed165fac34f222"},"cell_type":"code","source":"# view top 20 oov words\noov[:20]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5db29514870b62a7649595b54f44f30ed9009d2e"},"cell_type":"code","source":"# clean up our vocab\n# keep tokens with a min occurrence\nmin_occurrence = 5\ntokens = [k for k,c in vocab.items() if c >= min_occurrence]\nprint(len(tokens))\n\nvocab = set((' '.join(tokens)).split())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87eab7bdd710519890330e59ac1a84cb56f8fe07"},"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.utils.vis_utils import plot_model\nfrom keras.models import Sequential\nfrom keras.layers import Dense\nfrom keras.layers import Flatten\nfrom keras.layers import Embedding\nfrom keras.layers.convolutional import Conv1D\nfrom keras.layers.convolutional import MaxPooling1D\n\n\n# fit a tokenizer using keras\ndef create_tokenizer(text):\n    tokenizer = Tokenizer()\n    tokenizer.fit_on_texts(text)\n    return tokenizer\ntokenizer = create_tokenizer(train_clean)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bb7e48a6327403ae338bb97baed81ec9bd2f8f4d"},"cell_type":"code","source":"def encode_docs(tokenizer, max_length, docs):\n    # integer encode\n    encoded = tokenizer.texts_to_sequences(docs)\n    # pad sequences\n    padded = pad_sequences(encoded, maxlen = max_length, padding='post')\n    return padded","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e618afdaa9d4a72227c05341b0cc099de96d40ef"},"cell_type":"code","source":"vocab_size = len(tokenizer.word_index) + 1\nprint('Vocab Size: ', vocab_size)\nmax_length = max([len(s.split()) for s in train_clean])\nprint('Max Length: ', max_length)\nX_train = encode_docs(tokenizer, max_length, train_clean)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3596acf4964083f50c3323b1b3113ef861a5deda"},"cell_type":"code","source":"from keras.initializers import Constant\ntype(embeddings_index.vocab)\nEMBEDDING_DIM = 300\nMAX_NUM_WORDS = 20000\nword_index = tokenizer.word_index\nnum_words = vocab_size\nembedding_matrix = np.zeros((vocab_size, EMBEDDING_DIM))\nfor word, i in word_index.items():\n    if i > MAX_NUM_WORDS:\n        continue\n    try:\n        embedding_vector = embeddings_index.get_vector(word)\n    \n        # words not found in embedding index will be all-zeros.\n        embedding_matrix[i] = embedding_vector\n    except (KeyError):\n        continue\n        \n# load pre-trained word embeddings into an Embedding layer\n# note that we set trainable = False so as to keep the embeddings fixed\nembedding_layer = Embedding(num_words,\n                            EMBEDDING_DIM,\n                            embeddings_initializer=Constant(embedding_matrix),\n                            input_length=max_length,\n                            trainable=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fa4f52b9b1e057d3e388f000f4e5ae14b3e3f40a"},"cell_type":"code","source":"# define the model\ndef define_model(vocab_size, max_length):\n    model = Sequential()\n    model.add(embedding_layer)\n    model.add(Conv1D(filters=32, kernel_size=8, activation='relu'))\n    model.add(MaxPooling1D(pool_size=2))\n    model.add(Flatten())\n    model.add(Dense(10, activation='relu'))\n    model.add(Dense(1, activation='sigmoid'))\n    # compile network\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) # summarize defined model\n    model.summary()\n    plot_model(model, to_file='model.png', show_shapes=True)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d7cd49e78752694f50595324cbbaa30056233406"},"cell_type":"code","source":"model = define_model(vocab_size, max_length)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cdabe1c5f90a94c294162abe40f4fdd660ab09ef","scrolled":true},"cell_type":"code","source":"# fit model\nmodel.fit(X_train, train.target, epochs=4, verbose=1, validation_split=0.1, class_weight={1:0.6, 0:0.4})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78f83df345d3896addf57dedc4b4a7d8b1422657"},"cell_type":"code","source":"test_clean = gensim_preprocess(test.question_text)\npred = encode_docs(tokenizer, max_length, test_clean)\nprediction = model.predict_classes(pred).ravel()\nsubmission = pd.DataFrame({'qid':test.qid, 'prediction':prediction})\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b28053804bb66833d65f9fdc9241d17c7763fc0b"},"cell_type":"markdown","source":"This kernel scored 0.604 on the public leaderboard!"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}