{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom gensim.models import KeyedVectors\nfrom keras.models import Model\nfrom keras.layers import Input, Dense,Dropout,Embedding,CuDNNGRU,Bidirectional,GlobalMaxPooling1D,GRU\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embedding = KeyedVectors.load_word2vec_format('../input/quora-insincere-questions-classification/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin',binary=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/quora-insincere-questions-classification/train.csv')\ntest_df = pd.read_csv('../input/quora-insincere-questions-classification/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_df = train_df.iloc[:50]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import defaultdict\ndef build_vocab(sentences):\n    vocab = defaultdict(int)\n    for sentence in sentences:\n        for word in sentence.split():\n            vocab[word] +=1\n#     sorted_d = sorted(vocab.items(), key=lambda x: x[1],reverse=True)\n    return vocab","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def oov_vocab(vocab,embedding):\n    i = 0\n    k = 0\n    oov = defaultdict(int)\n    a = defaultdict(int)\n    for word in vocab:\n        try:\n            a[word] = embedding[word]\n            i += vocab[word]\n        except:\n            oov[word] += vocab[word]\n            k +=vocab[word]\n    sorted_d = sorted(oov.items(), key=lambda x: x[1],reverse=True)\n    print('Found embedding on {:.2%} of vocab'.format((len(a)/len(vocab))))\n    print('Found embedding on {:.2%} of all Text'.format(i/(i+k)))\n    \n    return sorted_d\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_text1(x):\n\n    x = str(x)\n    for punct in \"/-\":\n        x = x.replace(punct, ' ')\n    for punct in '&':\n        x = x.replace(punct, f' {punct} ')\n    for punct in '?!.,\"#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~' + '“”’':\n        x = x.replace(punct, '')\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_text(x):\n    liste = x.split()\n    newtext = []\n#     print(liste)\n    for word in liste:\n        \n        try:\n            \n            embedding[word]\n            newtext.append(word)\n#             print(ckword)\n        except KeyError:\n            try:\n                ckword = word[:-2]+word[-1:]\n                embedding[ckword]\n                newtext.append(ckword)\n#                 print(ckword)\n            except KeyError:\n                try:\n                    ckword = word[:-2]\n                    embedding[ckword]\n                    newtext.append(ckword)\n                except:\n                    newtext.append(word)\n    return ' '.join(newtext)\n\nimport re\n\ndef clean_numbers(x):\n\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['question_text'] = train_df['question_text'].apply(lambda x: clean_text(x))\ntrain_df['question_text'] = train_df['question_text'].apply(lambda x: clean_numbers(x))\ntrain_df['question_text'] = train_df['question_text'].apply(lambda x: clean_text1(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## split to train and val\ntrain_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)\n\n## some config values \nembed_size = 300 # how big is each word vector\nmax_features = 50000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 100 # max number of words in a question to use\n\n## fill up the missing values\ntrain_X = train_df[\"question_text\"].fillna(\"_na_\").values\nval_X = val_df[\"question_text\"].fillna(\"_na_\").values\ntest_X = test_df[\"question_text\"].fillna(\"_na_\").values\n\n## Tokenize the sentences\ntokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(train_X))\ntrain_X = tokenizer.texts_to_sequences(train_X)\nval_X = tokenizer.texts_to_sequences(val_X)\ntest_X = tokenizer.texts_to_sequences(test_X)\n\n## Pad the sentences \ntrain_X = pad_sequences(train_X, maxlen=maxlen)\nval_X = pad_sequences(val_X, maxlen=maxlen)\ntest_X = pad_sequences(test_X, maxlen=maxlen)\n\n## Get the target values\ntrain_y = train_df['target'].values\nval_y = val_df['target'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"word_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.zeros((max_features, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    try:\n        embedding_vector = embedding[word]\n    except:\n        embedding_vector = None\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"inp = Input(shape=(maxlen,))\nx = Embedding(max_features,embed_size, weights=[embedding_matrix])(inp)\nx = Bidirectional(CuDNNGRU(64,return_sequences=True))(x)\n# x = GRU(64,return_sequences=True)(x)\nx = GlobalMaxPooling1D()(x)\nx = Dense(6,activation='relu')(x)\nx = Dropout(0.1)(x)\nx = Dense(1,activation='sigmoid')(x)\nmodel = Model(inputs=inp, outputs=x)\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_paragram_test_y = model.predict(test_X, batch_size=1024, verbose=1)\nx = (pred_paragram_test_y>0.5).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = pd.DataFrame({'qid':test_df['qid'].values})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data['prediction'] = x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}