{"cells":[{"metadata":{},"cell_type":"markdown","source":"# LSTM Text classification using Tensorflow 2.0 Alpha"},{"metadata":{},"cell_type":"markdown","source":"# Tensorflow 2.0 Alpha important information:\n- [Official TF2 Alpha](https://www.tensorflow.org/alpha)\n- [New Features in TF 2](https://medium.com/tensorflow/whats-coming-in-tensorflow-2-0-d3663832e9b8)\n- [Standardizing on Keras for TF2](https://medium.com/tensorflow/standardizing-on-keras-guidance-on-high-level-apis-in-tensorflow-2-0-bad2b04c819a)\n- [TF2 GPU installation guide](https://www.tensorflow.org/install/gpu)"},{"metadata":{},"cell_type":"markdown","source":"# Importing Libraries"},{"metadata":{},"cell_type":"markdown","source":"### Installing TF-gpu 2.0 Alpha"},{"metadata":{"trusted":true},"cell_type":"code","source":"# !pip install tensorflow-cpu==2.1.0rc1\n!pip install tensorflow-gpu==2.1.0rc1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport math\nimport operator\nfrom collections import Counter\nfrom wordcloud import WordCloud, STOPWORDS\n\nfrom tqdm import tqdm\ntqdm.pandas()\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\nfrom sklearn.feature_extraction.text import CountVectorizer\n\nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation\nfrom tensorflow.keras.layers import Bidirectional, GlobalMaxPool1D\nfrom tensorflow.keras.models import Model, Sequential\nfrom tensorflow.compat.v1.keras.layers import CuDNNLSTM\nfrom tensorflow.keras import layers\n\n%matplotlib inline\nimport matplotlib.pyplot as plt\n\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.max_colwidth', 1000)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Environment checks"},{"metadata":{},"cell_type":"markdown","source":"#### Check Tensorflow Version"},{"metadata":{"scrolled":false,"trusted":true},"cell_type":"code","source":"print(tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Check GPU detection for tensorflow"},{"metadata":{"trusted":true},"cell_type":"code","source":"x = tf.random.uniform([3, 3])\n\nprint(\"Is there a GPU available: \"),\nprint(tf.test.is_gpu_available())\n\nprint(\"Is the Tensor on GPU #0:  \"),\nprint(x.device.endswith('GPU:0'))\n\nprint(\"Device name: {}\".format((x.device)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Check if Eager execution is running\n- Eager execution enables a more interactive frontend to TensorFlow, the details of which we will discuss much later.\n- [Eager basics official guide](https://www.tensorflow.org/tutorials/eager/eager_basics)"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(tf.executing_eagerly())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(tf.keras.__version__)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Importing data"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results\n\n# df = pd.read_csv(\"train.csv\")\ndf  = pd.read_csv(\"/kaggle/input/quora-insincere-questions-classification/train.csv\")\ndf_test = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/test.csv')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df[df.target==1].head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Number of questions: \", df.shape[0])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# EDA on the Question Text corpus"},{"metadata":{},"cell_type":"markdown","source":"## Target variable response rate\n- __6.18 %__ questions are insincere in the provided dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"df.target.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Percentage of insincere questions: {}\".format(sum(df.target == 1)*100/len(df.target)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" df.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Wordclouds for the question text corpus\n- This will show us the most frequently used words for sincere and insincere labeled questions"},{"metadata":{},"cell_type":"markdown","source":"### Defining function for generating wordcloud"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Thanks : https://www.kaggle.com/aashita/word-clouds-of-various-shapes ##\ndef plot_wordcloud(text, mask=None, max_words=200, max_font_size=100, figure_size=(24.0,16.0), \n                   title = None, title_size=40, image_color=False):\n    stopwords = set(STOPWORDS)\n    more_stopwords = {'one', 'br', 'Po', 'th', 'sayi', 'fo', 'Unknown'}\n    stopwords = stopwords.union(more_stopwords)\n\n    wordcloud = WordCloud(background_color='black',\n                    stopwords = stopwords,\n                    max_words = max_words,\n                    max_font_size = max_font_size, \n                    random_state = 42,\n                    width=800, \n                    height=400,\n                    mask = mask)\n    wordcloud.generate(str(text))\n    \n    plt.figure(figsize=figure_size)\n    if image_color:\n        image_colors = ImageColorGenerator(mask);\n        plt.imshow(wordcloud.recolor(color_func=image_colors), interpolation=\"bilinear\");\n        plt.title(title, fontdict={'size': title_size,  \n                                  'verticalalignment': 'bottom'})\n    else:\n        plt.imshow(wordcloud);\n        plt.title(title, fontdict={'size': title_size, 'color': 'black', \n                                  'verticalalignment': 'bottom'})\n    plt.axis('off');\n    plt.tight_layout()  \n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Wordcloud for sincere questions"},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_wordcloud(df[df.target == 0][\"question_text\"], title=\"Word Cloud of Sincere Questions\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Wordcloud for insincere questions\n- We observe that insincere questions contain highly debated and emotion based topics of Trump, Muslims, America, Russia, Obama, Liberal etc."},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_wordcloud(df[df.target == 1][\"question_text\"], title=\"Word Cloud of Insincere Questions\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Word frequency and n-gram frequency\n- Frequency of words can show the most used words for sincere and insincere questions\n- N-grams can be useful in distinguishing sincere and insincere questions by identifying patterns in topics"},{"metadata":{"trusted":true},"cell_type":"code","source":"stopwords = set(STOPWORDS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sincere_words = df[df.target==0].question_text.apply(lambda x: x.lower().split()).tolist()\ninsincere_words = df[df.target==1].question_text.apply(lambda x: x.lower().split()).tolist()\n\nsincere_words = [item for sublist in sincere_words for item in sublist if item not in stopwords]\ninsincere_words = [item for sublist in insincere_words  for item in sublist if item not in stopwords ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Number of sincere words',len(sincere_words))\nprint('Number of insincere words',len(insincere_words))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sincere_words_counter = Counter(sincere_words)\ninsincere_words_counter = Counter(insincere_words)\nprint(sincere_words_counter,insincere_words_counter)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Most common sincere words"},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"most_common_sincere_words = sincere_words_counter.most_common()[:10]\nmost_common_sincere_words = pd.DataFrame(most_common_sincere_words)\nmost_common_sincere_words.columns = ['word', 'freq']\nmost_common_sincere_words['percentage'] = most_common_sincere_words.freq *100 / sum(most_common_sincere_words.freq)\nmost_common_sincere_words","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Most common insincere words"},{"metadata":{"trusted":true},"cell_type":"code","source":"most_common_insincere_words = insincere_words_counter.most_common()[:10]\nmost_common_insincere_words = pd.DataFrame(most_common_insincere_words)\nmost_common_insincere_words.columns = ['word', 'freq']\nmost_common_insincere_words['percentage'] = most_common_insincere_words.freq *100 / sum(most_common_insincere_words.freq)\nmost_common_insincere_words","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Function for generating n-grams"},{"metadata":{"trusted":true},"cell_type":"code","source":"def generate_ngrams(words, n):\n    \n    # Use the zip function to help us generate n-grams\n    # Concatentate the tokens into ngrams and return\n    ngrams = zip(*[words[i:] for i in range(n)])\n    return [\" \".join(ngram) for ngram in ngrams]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n = 3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sincere_ngram_counter = Counter(generate_ngrams(sincere_words, n))\ninsincere_ngram_counter = Counter(generate_ngrams(insincere_words, n))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Most common sincere n-grams"},{"metadata":{"trusted":true},"cell_type":"code","source":"most_common_sincere_ngram = sincere_ngram_counter.most_common()[:10]\nmost_common_sincere_ngram = pd.DataFrame(most_common_sincere_ngram)\nmost_common_sincere_ngram.columns = ['word', 'freq']\nmost_common_sincere_ngram['percentage'] = most_common_sincere_ngram.freq *100 / sum(most_common_sincere_ngram.freq)\nmost_common_sincere_ngram","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Most common insincere n-grams\n- We observe a pattern of suggestive/controversial n-grams amongst the insincere questions"},{"metadata":{"trusted":true},"cell_type":"code","source":"most_common_insincere_ngram = insincere_ngram_counter.most_common()[:10]\nmost_common_insincere_ngram = pd.DataFrame(most_common_insincere_ngram)\nmost_common_insincere_ngram.columns = ['word', 'freq']\nmost_common_insincere_ngram['percentage'] = most_common_insincere_ngram.freq *100 / sum(most_common_insincere_ngram.freq)\nmost_common_insincere_ngram","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Parameters for preprocessing and algorithms"},{"metadata":{"trusted":true},"cell_type":"code","source":"# config values\nembed_size = 300 # how big is each word vector\nmax_features = 50000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 100 # max number of words in a question to use","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Preparation"},{"metadata":{},"cell_type":"markdown","source":"## Training and Test data split"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test  = train_test_split(df, test_size=0.1, random_state=2019)\ny_train, y_test = X_train['target'].values, X_test['target'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Missing value treatment\n- Fill missing questions with a string placeholder \"\\_NA\\_\""},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = X_train['question_text'].fillna('_NA_').values\nX_test = X_test['question_text'].fillna('_NA_').values\nX_submission = df_test['question_text'].fillna('_NA_').values","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"X_train.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Tokenizing the sentences to words"},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(X_train))\nX_train = tokenizer.texts_to_sequences(X_train)\nX_test = tokenizer.texts_to_sequences(X_test)\nX_submission = tokenizer.texts_to_sequences(X_submission)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Padding sequences for uniform dimensions"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = pad_sequences(X_train, maxlen=maxlen)\nX_test = pad_sequences(X_test, maxlen=maxlen)\nX_submission = pad_sequences(X_submission, maxlen=maxlen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Convert to TF-IDF data\nfrom sklearn.feature_extraction.text import TfidfTransformer\ntf_transformer= TfidfTransformer().fit(X_train)    \nx_train = tf_transformer.transform(X_train)\ntf_transformer.fit(X_test)\nx_test = tf_transformer.transform(X_test)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(x_test[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfTransformer \nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.pipeline import Pipeline\nimport numpy as np\ncorpus = ['this is the first document',\n           'this document is the second document',\n          'and this is the third one',\n           'is this the first document']\nvocabulary = ['this', 'document', 'first', 'is', 'second', 'the',\n              'and', 'one']\npipe = Pipeline([\n                 ('tfid', TfidfTransformer())]).fit(X_train)\n# pipe['count'].transform(corpus).toarray()\npipe['tfid'].idf_\nprint(len(pipe['tfid'].idf_))\n# print(X_train)\n\npipe1 = Pipeline([\n                 ('tfid', TfidfTransformer())]).fit(X_test)\n# pipe['count'].transform(corpus).toarray()\npipe1['tfid'].idf_\nprint(len(pipe1['tfid'].idf_))\n# print(X_test)\nprint(maxlen)\n\n\n# x_train = pad_sequences(pipe['tfid'].idf_, maxlen=maxlen)\n# x_test = pad_sequences(pipe1['tfid'].idf_, maxlen=maxlen)\n# print(x_test.shape)\n# print(x_train.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data prep function for future use\n- Creating a function including all of the above data prep steps, to use for quick future data processing"},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_prep(df):\n    print(\"Splitting dataframe with shape {} into training and test datasets\".format(df.shape))\n    X_train, X_test  = train_test_split(df, test_size=0.1, random_state=2019)\n    y_train, y_test = X_train['target'].values, X_test['target'].values\n    \n    print(\"Filling missing values\")\n    X_train = X_train['question_text'].fillna('_NA_').values\n    X_test = X_test['question_text'].fillna('_NA_').values\n    X_submission = df_test['question_text'].fillna('_NA_').values\n    \n    print(\"Tokenizing {} questions into words\".format(df.shape[0]))\n    tokenizer = Tokenizer(num_words=max_features)\n    tokenizer.fit_on_texts(list(X_train))\n    X_train = tokenizer.texts_to_sequences(X_train)\n    X_test = tokenizer.texts_to_sequences(X_test)\n    X_submission = tokenizer.texts_to_sequences(X_submission)\n    \n    print(\"Padding sequences for uniform dimensions\")\n    X_train = pad_sequences(X_train, maxlen=maxlen)\n    X_test = pad_sequences(X_test, maxlen=maxlen)\n    X_submission = pad_sequences(X_submission, maxlen=maxlen)\n    \n    print(\"Completed data preparation, returning training, test and submission datasets, split as dependent(X) and independent(Y) variables\")\n    \n    return X_train, X_test, y_train, y_test, X_submission","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Define LSTM layers and parameters\n- Bidirectional LSTM using CUDA GPU processing\n- No embeddings used currently\n- Two fully connected layers with dropout layers for reducing chances of overfit"},{"metadata":{"scrolled":false,"trusted":true},"cell_type":"code","source":"model1 = Sequential()\nmodel1.add(Embedding(max_features, embed_size, input_length=maxlen))\n# model1.add(Bidirectional(CuDNNLSTM(128, return_sequences=True)))\nmodel1.add(GlobalMaxPool1D())\nmodel1.add(Dropout(0.2))\nmodel1.add(Dense(64, activation='relu'))\nmodel1.add(Dropout(0.2))\nmodel1.add(Dense(32, activation='relu'))\nmodel1.add(Dropout(0.2))\nmodel1.add(Dense(16, activation='relu'))\nmodel1.add(Dropout(0.2))\nmodel1.add(Dense(1, activation='sigmoid'))\nmodel1.compile(loss='mse', optimizer='adam', metrics=['accuracy'])\nmodel1.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(y_train[y_train==0]),len(y_train[y_train==1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ninsincere_x = X_train[y_train==1]\nsincere_x = X_train[y_train==0]\ninsincere_y = y_train[y_train==1]\nsincere_y = y_train[y_train==0]\ninsincere_x.shape,sincere_x.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import random\nfrom sklearn.neighbors import NearestNeighbors\nimport numpy as np\nclass Smote:\n    def __init__(self,samples,N=10,k=5):\n        self.n_samples,self.n_attrs=samples.shape\n        self.N=N\n        self.k=k\n        self.samples=samples\n        self.newindex=0\n\n    def over_sampling(self):\n        N=int(self.N/100)\n        self.synthetic = np.zeros((self.n_samples * N, self.n_attrs))\n        neighbors=NearestNeighbors(n_neighbors=self.k).fit(self.samples)  \n#         print ('neighbors',neighbors)\n        for i in range(len(self.samples)):\n#             print('samples',self.samples[i])\n            nnarray=neighbors.kneighbors(self.samples[i].reshape((1,-1)),return_distance=False)[0]  #Finds the K-neighbors of a point.\n#             print ('nna',nnarray)\n            self._populate(N,i,nnarray)\n        return self.synthetic\n\n\n    # for each minority class sample i ,choose N of the k nearest neighbors and generate N synthetic samples.\n    def _populate(self,N,i,nnarray):\n        for j in range(N):\n#             print('j',j)\n            nn=random.randint(0,self.k-1)  #包括end\n            dif=self.samples[nnarray[nn]]-self.samples[i]\n            gap=random.random()\n            self.synthetic[self.newindex]=self.samples[i]+gap*dif\n            self.newindex+=1\n#             print(self.synthetic)\n            \na=np.array([[1,2,3],[4,5,6],[2,3,1],[2,1,2],[2,3,4],[2,3,4]])\ns=Smote(a,N=200)\nn= s.over_sampling()\nprint(np.shape(n))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def balance_data(sin_i,sin_t, in_sin_i,in_sin_t,method ='downsampling', test_sin_sample= 0,test_insin_sample= 0,sample_size=140000):\n    \"\"\"\n\n    :param aep_i:\n    :param aep_t:\n    :param non_aep_i:\n    :param non_aep_t:\n    :param method:\n    :param test_aep_sample:\n    :param test_nonaep_sample:\n    :return:\n    \"\"\"\n\n    # split test set and train set    \n    sin_test_i = sin_i[0:test_sin_sample,:]\n    sin_test_t = sin_t[0:test_sin_sample]\n    test_insin_sample = int(0.3*len(in_sin_i))\n    in_sin_test_i = in_sin_i[0:test_insin_sample,:]\n    in_sin_test_t = in_sin_t[0:test_sin_sample]\n\n    # test set\n    test_i = np.concatenate((sin_test_i,in_sin_test_i),axis=0)\n    test_t = []\n    test_t.extend(sin_test_t)\n    test_t.extend( in_sin_test_t)\n\n    # train set\n    sin_i =sin_i[test_sin_sample:,:]\n    sin_t = np.array(sin_t[test_sin_sample:])\n    in_sin_i = in_sin_i[test_insin_sample:, :]\n    in_sin_t = in_sin_t[test_insin_sample:]\n\n \n    # balance training set here\n    if method == 'downsampling':\n        sample_range = len(sin_i)\n        indices = np.random.randint(sample_range, size=len(in_sin_i))\n        new_sin_i = sin_i[indices,:]\n        new_sin_t = sin_t[indices]\n        \n        train_i = np.concatenate((new_sin_i, in_sin_i),axis=0)\n        train_t =[]\n        train_t.extend(new_sin_t)\n        train_t.extend(in_sin_t)\n\n\n    elif method == 'oversampling':\n        sample_range = len(in_sin_i)\n        indices = np.random.randint(sample_range, size=len(sin_i))\n        new_insin_i = in_sin_i[indices,:]\n        new_insin_t = in_sin_t[indices]\n        \n        train_i = np.concatenate((sin_i,new_insin_i),axis=0)\n        train_t =[]\n        train_t.extend(sin_t)\n        train_t.extend(new_insin_t)\n        pass\n    elif method == 'SMOTE':\n        \n#         s= Smote(sin_i,N=10*int(5000/len(insin_i)))\n        s= Smote(sin_i,N=200)\n        new_insin_i = s.over_sampling()\n        new_insin_t = [1.0]*len(new_sin_i)\n        indices = np.random.randint(len(sin_i), size=len(new_insin_i))\n        new_sin_i = sin_i[indices,:]\n        new_sin_t = sin_t[indices]\n        \n        train_i = np.concatenate((new_sin_i,new_insin_i),axis=0)\n        train_t =[]\n        train_t.extend(new_sin_t)\n        train_t.extend(new_insin_t)\n    else:\n#         sample_size = 5000\n        indices = np.random.randint(len(in_sin_i), size=sample_size)\n        new_insin_i = in_sin_i[indices,:]\n        new_insin_t = in_sin_t[indices]\n        indices = np.random.randint(len(sin_i), size=sample_size)\n        new_sin_i = sin_i[indices,:]\n        new_sin_t = sin_t[indices]\n        \n        train_i = np.concatenate((new_sin_i,new_insin_i),axis=0)\n        train_t =[]\n        train_t.extend(new_sin_t)\n        train_t.extend(new_insin_t)\n        pass\n\n\n    return train_i,train_t\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#method ='',或‘undersampling’, 'oversampling'.  这不要用SMOTE，主要是SMOTE只能用于continuous data, 这里的数据类型不对\n#如果用 method='' default,  insincere 和sincere data 各自生成sample_size 的量\n\nnew_x,new_y = balance_data(sincere_x,sincere_y, insincere_x,insincere_y,method ='', test_sin_sample= 0,test_insin_sample= 0,sample_size=140000)\nlen(new_x), len(new_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_sincere_y = np.array(sincere_y)\nnew_sincere_y[new_sincere_y==0].shape, new_sincere_y[new_sincere_y==1].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = np.array(new_x)\ny_train = np.array(new_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Test 3 classical Classifier"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_fscore_matrix(fitted_clf, model_name):\n    print(model_name, ' :')\n    \n    # get classes predictions for the classification report \n    y_train_pred, y_pred = fitted_clf.predict(X_train), fitted_clf.predict(X_test)\n    print(classification_report(y_test, y_pred), '\\n') # target_names=y\n    \n    # computes probabilities keep the ones for the positive outcome only      \n    print(f'F1-score = {f1_score(y_test, y_pred):.2f}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import f1_score, classification_report\nfrom sklearn.linear_model import LogisticRegression\nlrmodel = LogisticRegression(class_weight={0:y_train.sum(), 1:len(y_train) - y_train.sum()}, C=0.5, max_iter=100, n_jobs=-1)\nlrmodel.fit(X_train, y_train)\nget_fscore_matrix(lrmodel, 'LogisticRegression')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.tree import DecisionTreeClassifier\ndtc = DecisionTreeClassifier(random_state=0)\ndtc.fit(X_train, y_train)\nget_fscore_matrix(dtc, 'DecisionTreeClassifier')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.naive_bayes import GaussianNB\ngnb = GaussianNB()\ngnb.fit(X_train, y_train)\nget_fscore_matrix(gnb, 'Gaussian Naive Bayes Classifier')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Fit LSTM model "},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"%time model1.fit(X_train, y_train, batch_size=512, epochs=2, validation_data=(X_test, y_test), verbose = 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predict using trained model"},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_test_y = model1.predict([X_test], batch_size=1024, verbose=1)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Calculate optimal probability threshold for classification\n- Calculating best probability cut-off giving the highest F1 - Score"},{"metadata":{"trusted":true},"cell_type":"code","source":"opt_prob = None\nf1_max = 0\n\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    f1 = metrics.f1_score(y_test, (pred_test_y > thresh).astype(int))\n    print('F1 score at threshold {} is {}'.format(thresh, f1))\n    \n    if f1 > f1_max:\n        f1_max = f1\n        opt_prob = thresh\n        \n#print('Optimal probabilty threshold is {} for maximum F1 score {}'.format(opt_prob, f1_max))\nprint('The F1 score is {}'.format(f1_max))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model2 = Sequential()\nmodel2.add(Embedding(max_features, embed_size, input_length=maxlen))\nmodel2.add(Bidirectional(LSTM(128, return_sequences=True)))\nmodel2.add(GlobalMaxPool1D())\nmodel2.add(Dropout(0.2))\n# model2.add(Dense(64, activation='relu'))\n# model2.add(Dropout(0.2))\nmodel2.add(Dense(32, activation='relu'))\nmodel2.add(Dropout(0.2))\n# model2.add(Dense(16, activation='relu'))\n# model2.add(Dropout(0.2))\nmodel2.add(Dense(1, activation='sigmoid'))\nmodel2.compile(loss='mse', optimizer='adam', metrics=['accuracy'])\nmodel2.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%time model2.fit(X_train, y_train, batch_size=1024, epochs=1, validation_data=(X_test, y_test), verbose = 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_submission_y = model1.predict([X_submission], batch_size=1024, verbose=1)\npred_submission_y = (pred_submission_y > opt_prob).astype(int)\n\ndf_submission = pd.DataFrame({'qid': df_test['qid'].values})\ndf_submission['prediction'] = pred_submission_y\n#df_submission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loading embeddings"},{"metadata":{},"cell_type":"markdown","source":"## Function to load embeddings from file"},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_embed(file):\n    def get_coefs(word,*arr): \n        return word, np.asarray(arr, dtype='float32')\n    \n    if file == '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec':\n        embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file, encoding=\"utf8\") if len(o)>100)\n    else:\n        embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file, encoding='latin'))\n        \n    return embeddings_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"glove = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\nparagram =  '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\nwiki_news = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Extracting GloVe embedding\")\nembed_glove = load_embed(glove)\n#print(\"Extracting Paragram embedding\")\n#embed_paragram = load_embed(paragram)\n#print(\"Extracting FastText embedding\")\n#embed_fasttext = load_embed(wiki_news)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Building Vocabulary and calculating coverage"},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_vocab(texts):\n    sentences = texts.apply(lambda x: x.split()).values\n    vocab = {}\n    for sentence in sentences:\n        for word in sentence:\n            try:\n                vocab[word] += 1\n            except KeyError:\n                vocab[word] = 1\n    return vocab","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def check_coverage(vocab, embeddings_index):\n    known_words = {}\n    unknown_words = {}\n    nb_known_words = 0\n    nb_unknown_words = 0\n    for word in vocab.keys():\n        try:\n            known_words[word] = embeddings_index[word]\n            nb_known_words += vocab[word]\n        except:\n            unknown_words[word] = vocab[word]\n            nb_unknown_words += vocab[word]\n            pass\n\n    print('Found embeddings for {:.2%} of vocab'.format(len(known_words) / len(vocab)))\n    print('Found embeddings for  {:.2%} of all text'.format(nb_known_words / (nb_known_words + nb_unknown_words)))\n    unknown_words = sorted(unknown_words.items(), key=operator.itemgetter(1))[::-1]\n\n    return unknown_words","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Bulding dataset vocabulary"},{"metadata":{"trusted":true},"cell_type":"code","source":"vocab = build_vocab(df['question_text'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Calculating coverage for each embedding"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Glove : \")\noov_glove = check_coverage(vocab, embed_glove)\n#print(\"Paragram : \")\n#oov_paragram = check_coverage(vocab, embed_paragram)\n#print(\"FastText : \")\n#oov_fasttext = check_coverage(vocab, embed_fasttext)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"type(embed_glove)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dict(list(embed_glove.items())[20:22])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Text pre-processing to improve coverage of embeddings"},{"metadata":{},"cell_type":"markdown","source":"## Lower casing questions for uniform matching"},{"metadata":{"trusted":true},"cell_type":"code","source":"df['processed_question'] = df['question_text'].apply(lambda x: x.lower())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vocab_low = build_vocab(df['processed_question'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Glove : \")\noov_glove = check_coverage(vocab_low, embed_glove)\n#print(\"Paragram : \")\n#oov_paragram = check_coverage(vocab_low, embed_paragram)\n#print(\"FastText : \")\n#oov_fasttext = check_coverage(vocab_low, embed_fasttext)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov_glove[1:20]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Adding lower case words to embeddings if missing"},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_lower(embedding, vocab):\n    count = 0\n    for word in vocab:\n        if word in embedding and word.lower() not in embedding:  \n            embedding[word.lower()] = embedding[word]\n            count += 1\n    print(f\"Added {count} words to embedding\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Glove : \")\nadd_lower(embed_glove, vocab)\n#print(\"Paragram : \")\n#add_lower(embed_paragram, vocab)\n#print(\"FastText : \")\n#add_lower(embed_fasttext, vocab)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Glove : \")\noov_glove = check_coverage(vocab_low, embed_glove)\n#print(\"Paragram : \")\n#oov_paragram = check_coverage(vocab_low, embed_paragram)\n#print(\"FastText : \")\n#oov_fasttext = check_coverage(vocab_low, embed_fasttext)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov_glove[1:20]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Removing special characters appropriately\n- This ensures better a match to embeddings"},{"metadata":{"trusted":true},"cell_type":"code","source":"punctuations = \"/-'?!.,#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~\" + '\"\"“”’' + '∞θ÷α•à−β∅³π‘₹´°£€\\×™√²—–&'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_text(x):\n\n    x = str(x)\n    for punct in \"/-'\":\n        x = x.replace(punct, ' ')\n    for punct in punctuations:\n        x = x.replace(punct, '')\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df[\"processed_question\"] = df[\"processed_question\"].progress_apply(lambda x: clean_text(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vocab_low = build_vocab(df['processed_question'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Glove : \")\noov_glove = check_coverage(vocab_low, embed_glove)\n#print(\"Paragram : \")\n#oov_paragram = check_coverage(vocab_low, embed_paragram)\n#print(\"FastText : \")\n#oov_fasttext = check_coverage(vocab_low, embed_fasttext)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['question_text'] = df['processed_question']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test, X_submission = data_prep(df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Utilizing embeddings in LSTM classifier\n- Following a similar model network structure as previous for comparable results"},{"metadata":{"scrolled":false,"trusted":true},"cell_type":"code","source":"model1 = Sequential()\nmodel1.add(Embedding(max_features, embed_size, input_length=maxlen, weights = [embed_glove]))\nmodel1.add(Bidirectional(CuDNNLSTM(128, return_sequences=True)))\nmodel1.add(GlobalMaxPool1D())\nmodel1.add(Dropout(0.2))\nmodel1.add(Dense(64, activation='relu'))\nmodel1.add(Dropout(0.2))\nmodel1.add(Dense(32, activation='relu'))\nmodel1.add(Dropout(0.2))\nmodel1.add(Dense(1, activation='sigmoid'))\nmodel1.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n\nmodel1.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pipeline = Pipeline([\n    ('tfidf', TfidfVectorizer(max_df=0.5, min_df=20, stop_words = 'english')),\n    ('clf', model1(batch_size=512, epochs=5, validation_data=(X_test, y_test), verbose = 1))\n])\nX_train, X_test, y_train, y_test = train_test_split(df_train.text, df_train.target, test_size=0.1, random_state=1)\n\npipeline.fit(X_train, y_train)\nprediction = pipeline.predict(X_test)\nprint(metrics.accuracy_score(y_test, prediction))\nprint(metrics.precision_score(y_test, prediction))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Fit LSTM model "},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"%time model1.fit(X_train, y_train, batch_size=512, epochs=5, validation_data=(X_test, y_test), verbose = 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predict using trained model"},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_test_y = model1.predict([X_test], batch_size=1024, verbose=1)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Calculate optimal probability threshold for classification"},{"metadata":{"trusted":true},"cell_type":"code","source":"opt_prob = None\nf1_max = 0\n\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    f1 = metrics.f1_score(y_test, (pred_test_y > thresh).astype(int))\n    print('F1 score at threshold {} is {}'.format(thresh, f1))\n    \n    if f1 > f1_max:\n        f1_max = f1\n        opt_prob = thresh\n        \nprint('Optimal probabilty threshold is {} for maximum F1 score {}'.format(opt_prob, f1_max))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"pred_submission_y = model1.predict([X_submission], batch_size=1024, verbose=1)\npred_submission_y = (pred_submission_y > opt_prob).astype(int)\n\ndf_submission = pd.DataFrame({'qid': df_test['qid'].values})\ndf_submission['prediction'] = pred_submission_y\ndf_submission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Further Improvements:\n- Optimizing LSTM hyperparameters\n- Optimizing LSTM network structure (adding LSTM, dense, maxpooling etc. layers)\n- Text processing to further improve embeddings coverage\n- Using all 3 embeddings together/combining the weighted output of 3 LSTM models using each embedding"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.3"}},"nbformat":4,"nbformat_minor":4}