{"cells":[{"metadata":{},"cell_type":"markdown","source":"Most work in this notebook has been picked up Sudalai Rajkumar's kernels on [embeddings](https://www.kaggle.com/sudalairajkumar/a-look-at-different-embeddings) and [exploration](https://www.kaggle.com/sudalairajkumar/simple-exploration-notebook-qiqc). I have just tried to make them more beginner friendly. I'll be putting more detailed kernels around preprocessing on the same problem, picking up ideas from some other kernels on this problem and some of my own. \n\nHappy Learning!!"},{"metadata":{},"cell_type":"markdown","source":"#### This notebook is a work in progress!"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nsns.set()\nimport matplotlib\nimport cufflinks as cf\nimport plotly\nimport plotly.offline as py\nimport plotly.graph_objs as go\nfrom tqdm import tqdm\ntqdm.pandas()\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Wrangling and Exploration"},{"metadata":{"trusted":true},"cell_type":"code","source":"!unzip ../input/quora-insincere-questions-classification/embeddings.zip","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/train.csv')\ntest = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Train set shape : \",train.shape)\nprint(\"Test set shape : \",test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# no. of observations with distinct targets\ncount_targets = train['target'].value_counts()\n\n# setting up the above results in form of a bar chart using python graph objects module\ntrace = go.Bar(x = count_targets.index, y = count_targets, marker = dict(color = count_targets.values))\n# setting up parameters for layout of the bar chart \nlayout = go.Layout(title = 'Target counts', font = dict(size=12))\n\ndata = [trace] \nfig = go.Figure(data = data, layout = layout) # inserting defined traces and layout as parameters of the plotly figure method\npy.iplot(fig, filename = \"TargetCount\") # Plotting the bar chart\n\n\n# Further, plotting the observations for each class in form of a pie chart\n\nlabels = (np.array(count_targets.index)) # defining the targets of the dataset in the labels object\n# defining the proportions of count of each target out of total count\nproportions = (np.array((count_targets/count_targets.sum())*100)) \n\n# setting up our results as parameters in the trace object i.e. the data to plot\ntrace = go.Pie(labels = labels, values = proportions)\nlayout = go.Layout(                       \n    title = \"Target proportion pie\",     # pie chart layout specifications \n    font = dict(size = 12),\n    width = 600,\n    height = 600)\n\ndata = [trace]\nfig = go.Figure(data = data, layout = layout) \npy.iplot(fig, filename = \"usertype\")  # Plotting the pie chart","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Our dataset is highly imbalanced."},{"metadata":{"trusted":true},"cell_type":"code","source":"from nltk.corpus import stopwords\nstop_words = set(stopwords.words('english'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Number of words in sentences of the text \ntrain['num_words'] = train['question_text'].apply(lambda x: len(str(x).split()))\ntest['num_words'] = test['question_text'].apply(lambda x: len(str(x).split()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"maximum,mean = train['num_words'].max(),train['num_words'].mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Maximum no. of words : \",maximum)\nprint(\"Average no. of words : {:.2f}\".format(mean))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['num_words'].quantile([0.25,0.5,0.75,0.99])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Number of unique words in sentences of the text \ntrain['num_unique_words'] = train['question_text'].apply(lambda x: len(set(str(x).split()))) #set only holds unique\ntest['num_unique_words'] = test['question_text'].apply(lambda x: len(set(str(x).split())))   # values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"maximum_unique,mean_unique = train['num_unique_words'].max(),train['num_unique_words'].mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Maximum no. of words : \",maximum_unique)\nprint(\"Average no. of words : {:.2f}\".format(mean_unique))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['num_unique_words'].quantile([0.25,0.5,0.75,0.99])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Truncating the no. of words since that large a number is very rare in the dataset\ntrain['num_words'].loc[train['num_words']>50] = 50 \ntrain['num_unique_words'].loc[train['num_unique_words']>50] = 50","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f,axes = plt.subplots(2,1,figsize = (10,10))\n\nsns.boxplot(x = 'target', y = 'num_words', data = train, ax = axes[0])\naxes[0].set_xlabel('Target', fontsize = 10)\naxes[0].set_title('Number of words in each class', fontsize = 12)\n\nsns.boxplot(x = 'target', y = 'num_unique_words', data = train, ax = axes[1])\naxes[1].set_xlabel('Target', fontsize = 10)\naxes[1].set_title('Number of unique words in each class', fontsize = 12)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Ideas around Visualization using plotly have been taken from this amazing [kernel](https://www.kaggle.com/sudalairajkumar/simple-exploration-notebook-qiqc) by Sudalai Rajakumar(@srk)"},{"metadata":{},"cell_type":"markdown","source":"# Model Building"},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm\nimport math\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Input, Embedding, Dense, Activation, GRU, Conv1D, Activation, Dropout, Bidirectional, GlobalMaxPool1D\nfrom keras.models import Model\nfrom keras import initializers, regularizers, constraints, optimizers, layers","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 85-15 train, validation split\ntrain_df, val_df = train_test_split(train, test_size=0.15, random_state=1000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# size of every word vector\nembed_size = 300\n\n# number of unique words to use \nmax_features = 50000\n\n# maximum number of words in a question. This will be our input size\nmaxlen = 100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Filling missing values in the text columns if any\ntrain_X = train_df['question_text'].fillna(\"_na_\").values\nval_X = val_df['question_text'].fillna(\"_na_\").values\ntest_X = test['question_text'].fillna(\"_na_\").values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Tokenizing words in our sentences using keras tokenizer\ntokenizer = Tokenizer(num_words = max_features)\ntokenizer.fit_on_texts(list(train_X))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# converting each text in the dataset to a sequence of integers\ntrain_X = tokenizer.texts_to_sequences(train_X)\nval_X = tokenizer.texts_to_sequences(val_X)\ntest_X = tokenizer.texts_to_sequences(test_X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Padding sequences \n# Rememeber : The maximum length of our input can not be greater than 100 so we need to pad the incoming sequences at 100\n\ntrain_X = pad_sequences(train_X, maxlen = maxlen)\nval_X = pad_sequences(val_X, maxlen = maxlen)\ntest_X = pad_sequences(test_X, maxlen = maxlen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Target values\ntrain_y = train_df['target'].values\nval_y = val_df['target'].values","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### GloVe "},{"metadata":{"trusted":true},"cell_type":"code","source":"embeddings = 'glove.840B.300d/glove.840B.300d.txt'\n\ndef get_coefs(word,*arr):\n    return word, np.asarray(arr, dtype = 'float32')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(embeddings))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embs = np.stack(embeddings_index.values())\nembs_mean,embs_std = embs.mean(),embs.std()\nembs_shape = embs.shape[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"word_index = tokenizer.word_index #dictionary of tokens of words in our input text\nnb_words = min(max_features,len(word_index)) #number of words\nembedding_matrix = np.random.normal(embs_mean, embs_std, (nb_words,embed_size)) #emedding matrix of shape(50000,300)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for word,i in word_index.items():\n    if i >= max_features:\n        continue\n        embeddings_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[i] = embeddings_vector","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Building the Network"},{"metadata":{"trusted":true},"cell_type":"code","source":"inp = Input(shape = (maxlen,)) #input layer\nx = Embedding(max_features,embed_size,weights = [embedding_matrix])(inp) #embedding layer\nx = Bidirectional(GRU(64, return_sequences = True))(x) #bidirectional GRU layer\nx = GlobalMaxPool1D()(x) #layer that yields the maximum input as the output \nx = Dense(16, activation = 'relu')(x) #dense layer with relu activation\nx = Dropout(0.1)(x)\nx = Dense(1, activation = 'sigmoid')(x) #dense layer with sigmoid activation\nmodel = Model(inputs = inp, outputs = x)\nmodel.compile(loss = 'binary_crossentropy', optimizer = 'adam', metrics = 'accuracy')\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Details on the binary cross entropy and thresholds in this [paper](https://arxiv.org/abs/1402.1892)."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Training the model in batch sizes of 1024 for 5 iterations\nmodel.fit(train_X, train_y, batch_size=1024, epochs=5, validation_data=(val_X, val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, metrics.f1_score(val_y, (pred_glove_val_y>thresh).astype(int))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}