{"cells":[{"metadata":{"_uuid":"c54fd2c4596469817530c325826df94953021b4c"},"cell_type":"markdown","source":"<center><h1> Simple LSTM that does the job</h1></center> "},{"metadata":{"_uuid":"a370e96c3208be1c143c38c09984399beb2d0f71"},"cell_type":"markdown","source":"# Dependencies"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n                    \nimport os\nimport seaborn as sns\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score,confusion_matrix,classification_report,f1_score, precision_score, recall_score\n\n\nfrom keras.layers import Input\nfrom keras import Model\nfrom keras.preprocessing import sequence,text\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.models import Sequential\nfrom keras.layers import Dense,Dropout,Embedding,LSTM,Conv1D,GlobalMaxPooling1D,Flatten,MaxPooling1D,GRU,SpatialDropout1D,Bidirectional\nfrom keras.callbacks import EarlyStopping\nfrom keras.utils import to_categorical\nfrom keras.losses import categorical_crossentropy\nfrom keras.optimizers import Adam\nfrom keras.callbacks import Callback\n\nimport nltk\nfrom nltk.tokenize import word_tokenize\nfrom nltk import FreqDist\nfrom nltk.corpus import stopwords\nfrom nltk.stem import SnowballStemmer,WordNetLemmatizer\nstemmer=SnowballStemmer('english')\nlemma=WordNetLemmatizer()\nfrom string import punctuation\n\nimport re\nimport os\nimport gc\n\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4b996b78d77bbe2cbb0d6262fbaffe93be81e5dd"},"cell_type":"markdown","source":"# Directory content"},{"metadata":{"trusted":true,"_uuid":"d3f3e3ed388fee21578bef830967fa0d4c1ac920"},"cell_type":"code","source":"print(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8524fc43b0562ddbb5c7907face7240bf82a9383"},"cell_type":"markdown","source":"# Loading data"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"df = pd.read_csv('../input/train.csv')\ndf_test = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bb190e50a0521c47e4089e91abcc49d0a014ed34"},"cell_type":"markdown","source":"The dataset contains only 3 variables : \n1. an id to identify the question\n2. a question\n3. a label telling whether the question is insincere or not."},{"metadata":{"trusted":true,"_uuid":"9a9718f85279f18d32809e123f52981a2e0e0b27"},"cell_type":"code","source":"print(\"Columns :\", df.columns)\nprint(\"Row 0 :\")\nprint(\"qid :\", df.iloc[0]['qid'])\nprint(\"question_text : \", df.iloc[0]['question_text'])\nprint(\"target :\", df.iloc[0]['target'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ef39791aa9203fe3936e330c94aa5ac4256a8e86"},"cell_type":"markdown","source":"# Features Insights"},{"metadata":{"_uuid":"f40c0c91ede2c7e97e96cfead7cc9bfa8fa7f196"},"cell_type":"markdown","source":"**Good news** the dataset is not so imbalanced"},{"metadata":{"trusted":true,"_uuid":"81797dbd03e6d3b46caf3440abe9ef2527f4d7e2"},"cell_type":"code","source":"n_rows = len(df)\nn_insincere = sum(df['target'])\nprint(n_rows)\nprint(n_insincere)\n\nlabel_repart = pd.DataFrame(data={\"\" :[n_rows - n_insincere, n_insincere]}, index = [str(n_rows - n_insincere) + ' sincere questions', str(n_insincere) + ' insincere question'])\nlabel_repart.plot(kind='pie', title='Labels repartition, ratio ' + str(round(n_insincere / n_rows,2)*100) + \"%\", subplots=True, figsize=(8,8))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"de7138c054e220fcfe8f60c8d7c56d304798ade3"},"cell_type":"markdown","source":"## Questions exploration"},{"metadata":{"trusted":true,"_uuid":"37373443bc596af6ec0f8f3c0b935c5c66ba8477"},"cell_type":"code","source":"insincere_question = df[df['target'] == 1]['question_text'].values\nfor i in range(10):\n    print(insincere_question[i])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7fada691290b61557a7087c331bd70721884f7f9"},"cell_type":"markdown","source":"## Text Preprocessing"},{"metadata":{"trusted":true,"_uuid":"d47adce40e5af3d6d2d1abdbccce7408be21ebd4"},"cell_type":"code","source":"def clean_review(review_col):\n    review_corpus=[]\n    stops = set(stopwords.words(\"english\"))\n    for i in range(0,len(review_col)):\n        review=str(review_col[i])\n        review=re.sub('[^a-zA-Z]',' ',review)\n        word_token = word_tokenize(str(review).lower())\n        #review = [word for word in word_token if word not in stops]\n        #review=' '.join(review)\n        review=[lemma.lemmatize(w) for w in word_token if w not in stops]\n        review=' '.join(review)\n        review_corpus.append(review)\n        #if i % 1000 == 0:\n           #print(i/len(review_col)) \n    return review_corpus","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b427fbe9ced65e55d3d2c4d71d681ca95c723c02"},"cell_type":"code","source":"df['clean_question']=clean_review(df['question_text'].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6b490803b9e3374a7ab5524e8fedd40adced81f0"},"cell_type":"code","source":"df_test['clean_question']=clean_review(df_test['question_text'].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"33f086830069ba223b64e3ac04d15075e3e541fd"},"cell_type":"code","source":"#all_words=' '.join(df['clean_question'].values)\n#all_words=word_tokenize(all_words)\n#dist=FreqDist(all_words)\n#num_unique_word=len(dist)\nnum_unique_word = 166289","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fcae01431b6eaef109f76a7f3bd016ba462bc51c","scrolled":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2a5298cfee83bcceaf31c511c8478a68621b14a9"},"cell_type":"code","source":"#r_len=[]\n#for text in df['clean_question'].values:\n#    word=word_tokenize(text)\n#    l=len(word)\n#    r_len.append(l)    \n#MAX_QUESTION_LEN=np.max(r_len)\nMAX_QUESTION_LEN=125","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3d5eaef521eb0bf457c0b3ee6e789f72be1612a8"},"cell_type":"markdown","source":"## Preparing input data"},{"metadata":{"trusted":true,"_uuid":"c8d112e0d80b27c6a9f7dc30af013693585b9a19"},"cell_type":"code","source":"MAX_FEATURES = num_unique_word\nMAX_WORDS = MAX_QUESTION_LEN","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b4197b18af593f400fd69169ffd6e570ac2b8931"},"cell_type":"code","source":"y_train = df['target'].values\nX_train_text = df['clean_question'].values\nX_test_text = df_test['clean_question'].values\nprint(X_train_text.shape,y_train.shape)\nprint(X_test_text.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ad33c89af2cc56f15fd63da7a288da91de5f24dc"},"cell_type":"code","source":"X_train_text, X_val_text, y_train, y_val = train_test_split(X_train_text, y_train, test_size=0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7a237a50e03886adf804adaa6d659f6283a71b5a"},"cell_type":"code","source":"tokenizer = Tokenizer(num_words=MAX_FEATURES)\ntokenizer.fit_on_texts(list(X_train_text))\nX_train = tokenizer.texts_to_sequences(X_train_text)\nX_val = tokenizer.texts_to_sequences(X_val_text)\nX_test = tokenizer.texts_to_sequences(X_test_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b1a626de04a6be5f213b5e8fa392543cdd37a7d7"},"cell_type":"code","source":"X_train = sequence.pad_sequences(X_train, maxlen=MAX_WORDS)\nX_val = sequence.pad_sequences(X_val, maxlen=MAX_WORDS)\nX_test = sequence.pad_sequences(X_test, maxlen=MAX_WORDS)\nprint(X_train.shape,X_val.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab1329e31bd183b9894056d266f1bf6e74ab336a"},"cell_type":"markdown","source":"## GloVe embedding layers\n\nGloVe is an algorithm that enables to represent word as vector of semantic features. Refer to https://nlp.stanford.edu/projects/glove/ for more information."},{"metadata":{"trusted":true,"_uuid":"fc64f8852870c17f23b99f2d31331cd1ed940cd2"},"cell_type":"code","source":"def get_coefs(word, *arr):\n    return word, np.asarray(arr, dtype='float32')\n    \ndef get_embed_mat(EMBEDDING_FILE, max_features,embed_dim):\n    # word vectors\n    embeddings_index = dict(get_coefs(*o.rstrip().rsplit(' ')) for o in open(EMBEDDING_FILE, encoding='utf8'))\n    print('Found %s word vectors.' % len(embeddings_index))\n\n    # embedding matrix\n    word_index = tokenizer.word_index\n    num_words = min(max_features, len(word_index) + 1)\n    all_embs = np.stack(embeddings_index.values()) #for random init\n    embedding_matrix = np.zeros((len(word_index) + 1, embed_dim))\n    for word, i in word_index.items():\n        if i >= max_features:\n            continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[i] = embedding_vector\n    max_features = embedding_matrix.shape[0]\n    \n    return embedding_matrix\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f6466fc131cd7ff503d96007ed8c4d01b93cb290"},"cell_type":"code","source":"gloveEmbed = get_embed_mat('../input/embeddings/glove.840B.300d/glove.840B.300d.txt', MAX_FEATURES, 300)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e7056abc28077c1057a16709c22de845f7d1dc51"},"cell_type":"code","source":"word_index = tokenizer.word_index\nembedding_layer = Embedding(len(word_index) + 1,\n                            300,\n                            weights=[gloveEmbed],\n                            input_length=MAX_WORDS,\n                            trainable=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4a40f5ce22af31e613e6d6cd24e5e675aff899f7"},"cell_type":"markdown","source":"## Model : LSTM"},{"metadata":{"trusted":true,"_uuid":"66afd74f2813e246f2c4f9d498da4c935e7c8f01"},"cell_type":"code","source":"def line_search_f1_score(y_score, y_test):\n    max_f1_score = 0\n    opt_threshold = 0\n    for threshold in [i*0.01 for i in range(100)]:\n        y_preds = y_score > threshold\n        score = f1_score(y_preds, y_test)\n        if max_f1_score < score:\n            max_f1_score = score\n            opt_threshold = threshold\n    return max_f1_score, opt_threshold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c6bb8445a1fd70788bb0c72cca686108be445bc"},"cell_type":"code","source":"class Metrics(Callback):\n    def __init__(self):\n        self.best_threshold = 0.5\n        self.best_f1_score = 0\n    def on_train_begin(self, logs={}):\n        self.val_f1s = []\n        self.val_recalls = []\n        self.val_precisions = []\n        self.best_f1_score = 0\n    def on_epoch_end(self, epoch, logs={}):\n         idx = np.random.randint(0,self.validation_data[0].shape[0],1000)\n         val_predict = (np.asarray(self.model.predict(self.validation_data[0][idx], verbose=1))).round()\n         val_targ = self.validation_data[1][idx]\n         #_val_f1 = f1_score(val_targ, val_predict)\n         _val_f1, threshold = line_search_f1_score(val_targ, val_predict)\n         if _val_f1 > self.best_f1_score:\n                self.best_f1_score = _val_f1\n         self.best_threshold = threshold\n         _val_recall = recall_score(val_targ, val_predict)\n         _val_precision = precision_score(val_targ, val_predict)\n         self.val_f1s.append(_val_f1)\n         self.val_recalls.append(_val_recall)\n         self.val_precisions.append(_val_precision)\n         print(\" — val_f1: %f — val_precision: %f — val_recall %f\" %(_val_f1, _val_precision, _val_recall))\n         return\n \nmetric = Metrics()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7d55370156b5ffbc4ad0c7c38e9167407d7f2af2"},"cell_type":"code","source":"lstm_out = 200\n\nmodel = Sequential()\nmodel.add(embedding_layer)\nmodel.add(LSTM(lstm_out, dropout_U = 0.2, dropout_W = 0.2))\nmodel.add(Dense(1,activation='sigmoid'))\nmodel.compile(loss = 'binary_crossentropy', optimizer='adam',metrics = ['accuracy'])\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"36747d25ec13d5e8e22426e744521c3fd4db38a8"},"cell_type":"code","source":"model.fit(X_train, y_train, validation_data=(X_val, y_val),\n          epochs=2, batch_size=1024, verbose=1,callbacks=[metric])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d23c249ccde50f1204e9d14347210fc12a0cf02e"},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true,"_uuid":"3edc2f20b69476464e0ee0066761679826ca7921"},"cell_type":"code","source":"y_score_test = model.predict(X_val, verbose=1)\nmax_f1_score, threshold = line_search_f1_score(y_score_test, y_val)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"419764c3e889cf494101f8d9452ad7e4f6b1ac91"},"cell_type":"code","source":"y_sub = model.predict(X_test, verbose = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3b79d8e2630f315e95db54299543c9be0df937a8"},"cell_type":"code","source":"sub = pd.read_csv('../input/sample_submission.csv')\nsub.prediction = np.array(y_sub > threshold, dtype=int) \nsub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"289e1cd7facc38d6f52f1c74e6f561696219592a"},"cell_type":"code","source":"# Best f1_score on validation dataset :\nprint(threshold)\nprint(max_f1_score)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}