{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Deep Learning libraries\nfrom keras.preprocessing import sequence\nfrom keras.models import Sequential\nfrom keras.layers import Dense\nfrom keras.layers import Embedding\nfrom keras.layers import GlobalAveragePooling1D\nfrom keras.layers import Dropout, LSTM\nfrom keras.callbacks import ModelCheckpoint","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3bf41e3a8f0043064e72ea916136c991400796c"},"cell_type":"code","source":"print(os.listdir(\"../input/embeddings\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")\nsub = pd.read_csv('../input/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"45697062eec0b71407fee394e5644fceddc7625f"},"cell_type":"code","source":"ngram_range = 1\nmax_features = 50000\nmaxlen = 50\nbatch_size = 32\nembedding_dims = 300\nepochs = 4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8438450a857e3dfefb5b70c8e306a163d9969994"},"cell_type":"code","source":"x_train = train[\"question_text\"].values\nx_test = test[\"question_text\"].values\ny_train = train[\"target\"].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"52ec068c662d55f3dfde2c01efee3e7df2983d45"},"cell_type":"code","source":"from gensim.models import KeyedVectors\n\nnews_path = '../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\nembeddings_index = KeyedVectors.load_word2vec_format(news_path, binary=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"21c6f836db5bbbee0e1617a448950e3f093ae626"},"cell_type":"code","source":"def clean_text(x):\n\n    x = str(x)\n    for punct in \"/-'\":\n        x = x.replace(punct, ' ')\n    for punct in '&':\n        x = x.replace(punct, f' {punct} ')\n    for punct in '?!.,\"#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~' + '“”’':\n        x = x.replace(punct, '')\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"82a60d4b17a5106c30ab3eb51d06d6834a894bf4"},"cell_type":"code","source":"train[\"question_text\"] = train[\"question_text\"].apply(lambda x: clean_text(x))\ntest[\"question_text\"] = test[\"question_text\"].apply(lambda x: clean_text(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8018656e15ee6211170c0b37ac02c700dd4e3202"},"cell_type":"code","source":"import re\n\ndef clean_numbers(x):\n\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ff91a4a9f19a0315f07dd7a2fc5b88b1a4580877"},"cell_type":"code","source":"train[\"question_text\"] = train[\"question_text\"].apply(lambda x: clean_numbers(x))\ntest[\"question_text\"] = test[\"question_text\"].apply(lambda x: clean_numbers(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"40bc64ffdfb8352b01b7d9bfe9eb515ea8932d73"},"cell_type":"code","source":"def _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\n\nmispell_dict = {'colour':'color',\n                'centre':'center',\n                'didnt':'did not',\n                'doesnt':'does not',\n                'isnt':'is not',\n                'shouldnt':'should not',\n                'favourite':'favorite',\n                'travelling':'traveling',\n                'counselling':'counseling',\n                'theatre':'theater',\n                'cancelled':'canceled',\n                'labour':'labor',\n                'organisation':'organization',\n                'wwii':'world war 2',\n                'citicise':'criticize',\n                'instagram': 'social medium',\n                'whatsapp': 'social medium',\n                'snapchat': 'social medium',\n                'facebook': 'social medium'\n                }\nmispellings, mispellings_re = _get_mispell(mispell_dict)\n\ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n\n    return mispellings_re.sub(replace, text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"791ad04e18d68468cc944fc830c41f18a2e2b94e"},"cell_type":"code","source":"train[\"question_text\"] = train[\"question_text\"].apply(lambda x: replace_typical_misspell(x))\ntest[\"question_text\"] = test[\"question_text\"].apply(lambda x: replace_typical_misspell(x))\n\nto_remove = ['a','to','of','and']\n\ntrain_sentences = train[\"question_text\"].apply(lambda x: x.split())\ntrain_sentences = [[word for word in sentence if not word in to_remove] for sentence in train_sentences]\n\ntest_sentences = test[\"question_text\"].apply(lambda x: x.split())\ntest_sentences = [[word for word in sentence if not word in to_remove] for sentence in test_sentences]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef80633cc94dd2d16bd39ea829b0269e2277581b"},"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\ntk = Tokenizer(num_words=max_features, lower = True, filters='')\nfull_text = list(train_sentences) + list(test_sentences)\ntk.fit_on_texts(full_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5a7318c928b7fd37c233aa70a5a5ce91001a8b2c"},"cell_type":"code","source":"# create a weight matrix for words in training docs\nembedding_matrix = np.zeros((len(tk.word_index)+1, embedding_dims))\ncount = 0\nfor word, i in tk.word_index.items():\n    if word in embeddings_index:\n        embedding_matrix[i] = embeddings_index[word]\n    else:\n#         print(word)\n        count += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0190bf286f592c0790335c4dfc971ab6f6e2c8f5"},"cell_type":"code","source":"del embeddings_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d2bc8bf137c12aa0106eea61fa778e43afc7bcac"},"cell_type":"code","source":"train_tokenized = tk.texts_to_sequences(train['question_text'])\ntest_tokenized = tk.texts_to_sequences(test['question_text'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"607f227c63f24410ff3e54236411132980bcac88"},"cell_type":"code","source":"train_tokenized[100]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"182fc42a4e457b2fd1db9863a7ee49800cbb8887"},"cell_type":"code","source":"train_tokenized = sequence.pad_sequences(train_tokenized, maxlen=maxlen)\ntest_tokenized = sequence.pad_sequences(test_tokenized, maxlen=maxlen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cd56328a304d7aa130be0690cb339c8cce330336"},"cell_type":"code","source":"embedding_layer = Embedding(len(tk.word_index) + 1,\n                            embedding_dims,\n                            weights=[embedding_matrix],\n                            input_length=maxlen,\n                            trainable=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"86da3264b34bd7d85745f3d9b080030db01eeb67"},"cell_type":"code","source":"from keras import backend as K\n\ndef f1(y_true, y_pred):\n    def recall(y_true, y_pred):\n        \"\"\"Recall metric.\n\n        Only computes a batch-wise average of recall.\n\n        Computes the recall, a metric for multi-label classification of\n        how many relevant items are selected.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n        recall = true_positives / (possible_positives + K.epsilon())\n        return recall\n\n    def precision(y_true, y_pred):\n        \"\"\"Precision metric.\n\n        Only computes a batch-wise average of precision.\n\n        Computes the precision, a metric for multi-label classification of\n        how many selected items are relevant.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n        precision = true_positives / (predicted_positives + K.epsilon())\n        return precision\n    precision = precision(y_true, y_pred)\n    recall = recall(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3013342441acde0685d01793fee6428d479ced95"},"cell_type":"code","source":"from keras.layers import Input, Conv2D, MaxPool2D, Reshape, Concatenate, Flatten, BatchNormalization, Dropout\nfrom keras.models import Model","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":false,"trusted":true,"_uuid":"436ccb2fb548b0244d4a55d1bd98bb368032019a","scrolled":false},"cell_type":"code","source":"filter_sizes = [1,2,3,4]\nnum_filters = 64\n\nsequence_input = Input(shape=(maxlen,), dtype='int32')\nembedded_sequences = embedding_layer(sequence_input)\nx = Reshape((maxlen, embedding_dims, 1))(embedded_sequences)\n\nmaxpool_pool = []\nfor i in range(len(filter_sizes)):\n    conv1 = Conv2D(num_filters, kernel_size=(filter_sizes[i], embedding_dims),\n                                 kernel_initializer='he_normal', activation='elu')(x)\n    maxpool_pool.append(MaxPool2D(pool_size=(maxlen - filter_sizes[i] + 1, 1))(conv1))\n    \n\nz = Concatenate(axis=1)(maxpool_pool)   \nz = Flatten()(z)\nz = Dropout(0.1)(z)\n\npreds = Dense(1, activation=\"sigmoid\")(z)\n\nmodel = Model(sequence_input, preds)\ncheckpoint = ModelCheckpoint('CNN2D-{epoch:03d}.h5', verbose=1, monitor='val_loss',save_best_only=True, mode='auto')\nmodel.compile(loss='binary_crossentropy',\n              optimizer='adam',\n              metrics=[f1])\n\nclass_weight = {0: 1.,\n                1: 3.}\n\n# happy learning!\nmodel.fit(train_tokenized, y_train,\n          batch_size=128,\n          epochs=epochs, validation_split=0.01, class_weight=class_weight, callbacks=[checkpoint])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"416917dff82e9f97f43ee2d2b1b1442233e17f81"},"cell_type":"code","source":"from sklearn.metrics import f1_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f4728fdf6a2f9dcd4fe64ced0e8ef8ac289d0c2c"},"cell_type":"code","source":"train_pred = model.predict(train_tokenized[:50000], batch_size = 1024, verbose = 1)\ntrain_predictions = np.squeeze(train_pred>0.47).astype(int)\nf1_score(train_predictions, y_train[:50000])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d578c12822d042f6ab7f738158aa59ace1d022f6"},"cell_type":"code","source":"pred = model.predict(test_tokenized, batch_size = 1024, verbose = 1)\npredictions = np.squeeze(pred>0.47).astype(int)\nsub['prediction'] = predictions\nsub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b6e10197f4ceea58989843bd5667332e44e63eca"},"cell_type":"code","source":"test['question_text'].iloc[np.where(predictions==1)[0][:10]].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"76b565a5720259f9df6fa394ac89f73ebb2adb7f"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}