{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#import zipfile\n#\n#with zipfile.ZipFile(\"../input/quora-insincere-questions-classification/embeddings.zip\") as zf:\n    #print(zf.namelist())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Glove embedding extraction","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/test.csv')\ntrain = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/train.csv')\n\n\nprint(len(test),len(train))\nprint(train.isna().sum())\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#import io\n\n#import tqdm\n\n#embedding_index = {}\n\n#with zipfile.ZipFile(\"../input/quora-insincere-questions-classification/embeddings.zip\") as zf:\n    #with io.TextIOWrapper(zf.open(\"glove.840B.300d/glove.840B.300d.txt\"), encoding=\"utf-8\") as f:\n       # for line in tqdm.tqdm(f):\n          #  values = line.split(' ')# \".split(' ')\" only for glove-840b-300d; for all other files, \".split()\" works\n          #  word = values[0]\n           # coefs = np.asarray(values[1:],dtype = 'float32')\n           # embedding_index[word] = coefs\n\n#print('Loaded %s word vectors.' % len(embedding_index))\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer,TfidfVectorizer\nfrom sklearn.model_selection import train_test_split, cross_val_score\n\n\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\n\nfrom tensorflow.keras.callbacks import EarlyStopping\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef add_num_feat(df):\n    \n    df['len']               = df['question_text'].apply(lambda x: len(x))\n    df['question_marks']    =  df['question_text'].str.count('\\?')\n    df['exclimat_marks']    =  df['question_text'].str.count('\\!')\n    df['dot_marks']         =  df['question_text'].str.count('\\.')\n    df['comma_marks']       =  df['question_text'].str.count('\\,')\n    df['capital_latters']   =  df['question_text'].str.count('[A-Z]')\n    df['abbrivi_count']     =  df['question_text'].str.count('[A-Z]{2,}')\n    \n    return df\n\n\ntrain = add_num_feat(train)\ntest  = add_num_feat(test)\n\ntrain['len'].hist(bins = 100)\nprint('Mean len: {}\\nSTD len: {}\\nMin len: {}\\nMax len: {}'.format(    np.mean(train['len']),\n                                                                       np.std(train['len']),\n                                                                       min(train['len']),\n                                                                       max(train['len'])))\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer = Tokenizer()\ntokenizer.fit_on_texts(list(train['question_text'])+list(test['question_text']))\n\nvocab_size = len(tokenizer.word_index)+1\nembedd_size = 300\nmax_len = 100\n\nprint('Vocab Size: {}\\nEmbedding size: {}\\nMax words: {}'.format(vocab_size,\n                                                                 embedd_size, \n                                                                 max_len))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create random embedding matrix based on mean and std of glove\n\n#all_embs = np.stack(list(embedding_index.values()))\n#emb_mean,emb_std = all_embs.mean(), all_embs.std()\n#nb_words = len(tokenizer.word_index)\n#embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embedd_size))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# replace random values with known values from glove\n\n#for word, idx in tokenizer.word_index.items():\n   # idx-=1\n    #embedding_vector = embedding_index.get(word)\n    #if embedding_vector is not None: \n                #embedding_matrix[idx] = embedding_vector\n            \n#print(len(embedding_index),len(embedding_matrix),len( tokenizer.word_index))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = train['question_text'].values\ny = train['target'].values\n\nX_train,X_val,y_train,y_val = train_test_split(X,y,test_size=0.3,\n                                               shuffle = True,\n                                               random_state = 42)\n\n\n\nX_train_tok = tokenizer.texts_to_sequences(X_train)\nX_val_tok = tokenizer.texts_to_sequences(X_val)\n\nX_train = pad_sequences(X_train_tok,maxlen=max_len)\nX_val = pad_sequences(X_val_tok,maxlen=max_len)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Sequential API","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model_seq = tf.keras.Sequential([\n    # when add embedding remember to add embedding_size.shape[1],weight,trainable\n    tf.keras.layers.Embedding(vocab_size,\n                              64,\n                              input_length = max_len),    \n    tf.keras.layers.Bidirectional(tf.compat.v1.keras.layers.CuDNNLSTM(64, return_sequences=True),input_shape=(max_len, 64)),\n    tf.keras.layers.Bidirectional(tf.compat.v1.keras.layers.CuDNNLSTM(64)),\n    tf.keras.layers.Dense(64,activation = 'relu'),\n    tf.keras.layers.Dense(1,activation = 'sigmoid')\n        \n])\n\n\nmodel_seq.compile(loss = 'binary_crossentropy',\n                  optimizer = 'adam',\n                  metrics = ['accuracy'])\n\nmodel_seq.fit(X_train,y_train,\n                 batch_size=1024,\n                 epochs=1,\n                 validation_data=(X_val,y_val),\n                 verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import f1_score \n\ndef threshold_adj(y_true,y_pred):\n    f1 = 0\n    for i in np.arange(0.05,0.501,0.01):\n        thresh = np.round(i, 2)\n        f1_next = f1_score(y_true, np.where(y_pred > thresh,1,0))\n        print('With threshold ',thresh,' f1 is ',f1_next )\n        if f1_next > f1:\n            f1 = f1_next\n    \n    return f1\n\npred_val = model_seq.predict(X_val,verbose=1,batch_size=1024)\nthreshold = threshold_adj(y_val,pred_val)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\n\nconfusion_matrix(y_val,np.where(pred_val > threshold,1,0))\n\n\n\nX_test = test['question_text'].values\n\nX_test = tokenizer.texts_to_sequences(X_test)\nX_test = pad_sequences(X_test,maxlen = max_len)\n\npred = model_seq.predict(X_test,verbose = 1,batch_size = 1024)\n\npred = np.where(pred > threshold,1,0)\n\ntest_qid = test['qid']\nsub = pd.DataFrame({'qid':test_qid,'prediction':pred.reshape(pred.shape[0],)})\nsub.to_csv('submission.csv',index = False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Functional API","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#X = train.drop('target',axis = 1)\n#y = train['target']\n\n#X_train,X_val,y_train,y_val = train_test_split(X,y,test_size=0.3,\n                                               #shuffle = True,\n                                               #random_state = 42)\n\n#X_train_input_1 = X_train['question_text'].values\n#X_train_input_2 = X_train.loc[:,'len':].values\n\n\n#X_val_input_1 = X_val['question_text'].values\n#X_val_input_2 = X_val.loc[:,'len':].values\n\n\n#X_train_tok = tokenizer.texts_to_sequences(X_train_input_1)\n#X_val_tok = tokenizer.texts_to_sequences(X_val_input_1)\n\n#X_train = pad_sequences(X_train_tok,maxlen=max_len)\n#X_val = pad_sequences(X_val_tok,maxlen=max_len)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#input1 = tf.keras.layers.Input(shape =(max_len,))#\n#input2 = tf.keras.layers.Input(shape = (7,))\n\n# when add embedding remember to add embedding_size.shape[1],weight,trainable\n#embed       = tf.keras.layers.Embedding(len(tokenizer.word_index)+1,\n                                        #8,\n                                        #embedding_matrix.shape[1],\n                                        #input_length = max_len,\n                                        #weights = [embedding_matrix],\n                                        #trainable    = True)(input1)\n\n#blstm       = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64))(embed)\n#blstm_dense = tf.keras.layers.Dense(64,activation  = 'relu')(blstm)\n\n#dense_num   = tf.keras.layers.Dense(128,activation = 'relu')(input2)\n#dense2_num  = tf.keras.layers.Dense(64,activation  = 'relu')(dense_num)\n\n#concat      = tf.keras.layers.concatenate([blstm_dense,dense2_num])\n\n#dense3      = tf.keras.layers.Dense(32,activation = 'relu')(concat)\n#dropout      = tf.keras.layers.Dropout(0.1)(dense3)\n\n#final       = tf.keras.layers.Dense(1,activation  = 'sigmoid')(dropout)\n\n\n#model = tf.keras.Model(inputs = [input1,input2],outputs = final)\n\n\n#model.compile(loss      = 'binary_crossentropy',\n              #optimizer = 'adam',\n              #metrics   = ['accuracy'])\n\n\n#tf.keras.utils.plot_model( \n    #model, to_file='model.png',\n    #show_shapes=True, show_layer_names=True,\n    #rankdir='TB', expand_nested=False, dpi=96\n#)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#model.fit([X_train,X_train_input_2],y_train,\n          #batch_size=1024,\n          #epochs=1,\n          #validation_data=([X_val,X_val_input_2]),\n          #verbose=1)\n\n#pred_val = model.predict([X_val,X_val_input_2],verbose=1,batch_size=1024)\n#threshold = threshold_adj(y_val,pred_val)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Predict test file and write submission file","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}