{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"# Reading the data"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\ndf = pd.read_csv('../input/train.csv')\ndf_test = pd.read_csv('../input/test.csv')\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2bb59e7f6a76851fc3d6f73b5148ee4c0d34fd84"},"cell_type":"markdown","source":"# Cleaning question text\n\nUsing [Dieter's kernel](https://www.kaggle.com/christofhenkel/how-to-preprocessing-when-using-embeddings), the cleaning has to be changed:"},{"metadata":{"trusted":true,"_uuid":"e2cadb2ff611d5000f41d3dbd5b8167542450c23"},"cell_type":"code","source":"import re\nimport string\n\nfrom nltk.corpus import stopwords\nfrom nltk.stem import SnowballStemmer\n\nstops = set(stopwords.words('english'))\nstemmer = SnowballStemmer('english')\n\ndef clean_question(question):\n    question = question.translate(string.punctuation)\n    \n    words = question.lower().split()\n    question = [w for w in words if w not in stops and len(w) >= 3]\n    question = ' '.join(words)\n    \n    question = re.sub(r'[^A-Za-z0-9^,!.\\/\\'+-=]', ' ', question)\n    question = re.sub(r'what\\'s', 'what is', question)\n    question = re.sub(r'\\'s', ' ', question)\n    question = re.sub(r'\\'ve', ' have', question)\n    question = re.sub(r'n\\'t', ' not', question)\n    question = re.sub(r'i\\'m', 'i am', question)\n    question = re.sub(r'\\'re', ' are', question)\n    question = re.sub(r'\\'d', ' would', question)\n    question = re.sub(r'\\'ll', ' will', question)\n    \n    # remove morphological affixes\n    words = question.split()\n    stemmed_words = [stemmer.stem(w) for w in words]\n    question = ' '.join(stemmed_words)\n    \n    return question\n\ndf['question_text'] = df['question_text'].map(lambda q: clean_question(q))\ndf_test['question_text'] = df_test['question_text'].map(lambda q: clean_question(q))\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"60809977d84cd1345403cad90fe046d54261c781"},"cell_type":"markdown","source":"# Building the embedding"},{"metadata":{"trusted":true,"_uuid":"dc628d49c2cec4fe6773584823af052ec9f91e1e"},"cell_type":"code","source":"# loading embedding: https://www.kaggle.com/sudalairajkumar/a-look-at-different-embeddings\nEMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n\n'Loaded %s word vectors' % len(embeddings_index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d50ad982c9ba7f9a2d52eac20def82f042e0ab6e"},"cell_type":"code","source":"embedding_matrix = np.zeros((vocabulary_size, 300))\nfor word, index in tokenizer.word_index.items():\n    if index <= vocabulary_size - 1:\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[index] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b4054b678f12df40d5a41102dce8a78f85beb1fd"},"cell_type":"markdown","source":"# Building the Model"},{"metadata":{"_uuid":"dd0e294e3819224b8fdc2c481ca3c585b720e6aa"},"cell_type":"markdown","source":"## Tokenizing"},{"metadata":{"trusted":true,"_uuid":"d4182778d1a43486cd11eead9e50f2e637f1cece"},"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\n\n\nvocabulary_size = 20000\ntokenizer = Tokenizer(num_words=vocabulary_size)\ntokenizer.fit_on_texts(df['question_text'].append(df_test['question_text']))\n\nsequences = tokenizer.texts_to_sequences(df['question_text'])\npadded_data = pad_sequences(sequences, maxlen=50)\n\npadded_data.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d1c4ba1bd0d2f49cb98c047297dcf50c409b30da"},"cell_type":"markdown","source":"## Training the model"},{"metadata":{"trusted":true,"_uuid":"76b074482cf9dafe44f4a56f631580585ec02528"},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers.embeddings import Embedding\nfrom keras.layers import Dropout, Conv1D, MaxPooling1D, LSTM, Dense\n\n\nmodel = Sequential()\nmodel.add(Embedding(vocabulary_size, 300, input_length=50,\n                    weights=[embedding_matrix], trainable=False))\nmodel.add(Dropout(0.2))\nmodel.add(Conv1D(64, 5, activation='relu'))\nmodel.add(MaxPooling1D(pool_size=4))\nmodel.add(LSTM(300))\nmodel.add(Dense(1, activation='sigmoid'))\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"5be04e02fa568a4a60bf59106304096f790bf9dc"},"cell_type":"code","source":"labels = df['target']\nmodel.fit(padded_data, np.array(labels), validation_split=0.4, epochs=3)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"317193bbc1e044ad1eadf9f112177f951aa5ca7f"},"cell_type":"markdown","source":"## Predicting and adjusting"},{"metadata":{"trusted":true,"_uuid":"ce0c2f5bcd0106e9bdae83a3bc4db8c0b239c574"},"cell_type":"code","source":"sequences = tokenizer.texts_to_sequences(df_test['question_text'])\npadded_test = pad_sequences(sequences, maxlen=50)\n\npadded_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06ef5c54376bbf59cdef155b610408abb7a4aa64"},"cell_type":"code","source":"predicted = model.predict(padded_test, batch_size=1024, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"49a063d53387b1ec0dbc7a5544b9ec5ac89f1875"},"cell_type":"code","source":"sample = df.sample(int(len(df) * 0.2))\nsample_label = sample['target']\n\nsample_sequences = tokenizer.texts_to_sequences(sample['question_text'])\npadded_sample = pad_sequences(sample_sequences, maxlen=50)\n\npadded_sample.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18501dce39ab52feed2a03d772627cd44e8479dc"},"cell_type":"code","source":"predicted_sample = model.predict(padded_sample, batch_size=1024, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f5c6a8f8859a5d0735c66ef86a37d1b509e3bd91"},"cell_type":"code","source":"from sklearn.metrics import f1_score\n\n\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    print('F1 score at threshold {} is {}'.format(thresh, f1_score(sample_label,\n                                                             (predicted_sample > thresh).astype(int))))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bca9e2b8a9e65efc1dfdeec152e936276ea9a21f"},"cell_type":"markdown","source":"## Preparing submission"},{"metadata":{"trusted":true,"_uuid":"a0ecceb5a7a2b39526e607ae7be5c679bf0e53b7"},"cell_type":"code","source":"output = (predicted > 0.31).astype(int)\noutput","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"96a17d841f17dc57f209d903e6909a33b7931bb0"},"cell_type":"code","source":"df_test['prediction'] = output\nsubmission = df_test.drop(columns=['question_text'])\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1583e6c2a892c399445bcb20abdd4fef0172444a"},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}