import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import matplotlib.pyplot as plt
import seaborn as sns
from nltk.corpus import stopwords
from nltk import word_tokenize, ngrams
from sklearn import ensemble
from sklearn.model_selection import KFold
from sklearn.metrics import log_loss
import xgboost as xgb
from gensim.models import word2vec

data_train = pd.read_csv('../input/train.csv')
data_test = pd.read_csv('../input/test.csv')

from nltk.corpus import stopwords

stops = set(stopwords.words("english"))

def word_match_share(row):
    q1words = {}
    q2words = {}
    for word in str(row['question1']).lower().split():
        if word not in stops:
            q1words[word] = 1
    for word in str(row['question2']).lower().split():
        if word not in stops:
            q2words[word] = 1
    if len(q1words) == 0 or len(q2words) == 0:
        # The computer-generated chaff includes a few questions that are nothing but stopwords
        return 0
    shared_words_in_q1 = [w for w in q1words.keys() if w in q2words]
    shared_words_in_q2 = [w for w in q2words.keys() if w in q1words]
    R = (len(shared_words_in_q1) + len(shared_words_in_q2))/(len(q1words) + len(q2words))
    return R

data_train['word_match'] = data_train.apply(word_match_share, axis=1, raw=True)
data_test['word_match'] = data_test.apply(word_match_share, axis=1, raw=True)


import string
import re
def remove_punctuation_from_sentence(val):
    chars = re.escape(string.punctuation)
    sentence = re.sub(r'['+chars+']', '',val.lower())
    sentence=sentence.split(' ')
    sentence = [x for x in sentence if x]
    sentence = ' '.join(sentence)
    return sentence

def remove_punctuation():
    "drop nans, then apply 'clean_sentence' function to question1 and 2"
    global data_train,data_test
    
    data_train = data_train.dropna(how="any")
    data_test = data_test.dropna(how="any")
    for col in ['question1', 'question2']:
        data_train[col] = data_train[col].apply(remove_punctuation_from_sentence)
        data_test[col] = data_test[col].apply(remove_punctuation_from_sentence)

remove_punctuation()

def build_data(data_train,data_test):
    "Creates a list of lists containing words from each sentence"
    corpus = []
    for col in ['question1', 'question2']:
        for sentence in data_train[col].iteritems():
            word_list = sentence[1].split(" ")
            corpus.append(word_list)
    for col in ['question1', 'question2']:
        for sentence in data_test[col].iteritems():
            word_list = sentence[1].split(" ")
            corpus.append(word_list)
    return corpus

corpus = build_data(data_train,data_test)        
model = word2vec.Word2Vec(corpus, size=100, window=10,min_count=1, workers=4)

def vector_sentences(val):
    "remove chars that are not letters or numbers, downcase, then remove stop words"
    words = val.split(' ')
    v=model.wv[words[0]]
    for i in range(1,len(words)):
        try:
            v = np.add(v,model.wv[words[i]])
        except:
            continue
    return v

def vector():
    global data_train,data_test
    data_train['vectorq1'] = ''
    data_train['vectorq2'] = ''
    data_test['vectorq1'] = ''
    data_test['vectorq2'] = ''
    for col in ['question1', 'question2']:
        if col == 'question1':
            data_train['vectorq1'] = data_train[col].apply(vector_sentences)
            data_test['vectorq1'] = data_test[col].apply(vector_sentences)
        else:
            data_train['vectorq2'] = data_train[col].apply(vector_sentences)
            data_test['vectorq2'] = data_test[col].apply(vector_sentences)

vector()
data_train.head(5)
'''Deleting the unused columns to free memory'''
del data_train['qid1']
del data_train['qid2']
del data_train['question1']
del data_train['question2']
del data_test['question1']
del data_test['question2']

def unit_vector(vector):
    """ Returns the unit vector of the vector.  """
    return vector / np.linalg.norm(vector)
def angle_between(v1, v2):
    v1_u = unit_vector(v1)
    v2_u = unit_vector(v2)
    return np.arccos(np.clip(np.dot(v1_u, v2_u), -1.0, 1.0))

def cos(val):
    a = val['vectorq1']
    b = val['vectorq2']
    return angle_between(a,b)

def cosine():
    global data_train,data_test
    data_train['cosine'] = ''
    data_train['cosine'] = data_train.apply (lambda row: cos (row),axis=1)
    data_test['cosine'] = ''
    data_test['cosine'] = data_test.apply (lambda row: cos (row),axis=1)
cosine()

x_train = data_train[['word_match','cosine']]
y_train = data_train[['is_duplicate']]

# Finally, we split some of the data off for validation
from sklearn.cross_validation import train_test_split

x_train, x_validation, y_train, y_validation = train_test_split(x_train, y_train, test_size=0.2, random_state=1234)

import xgboost as xgb

# Set our parameters for xgboost
params = {}
params['objective'] = 'binary:logistic'
params['eval_metric'] = 'logloss'
params['eta'] = 0.02
params['max_depth'] = 6

train = xgb.DMatrix(x_train, label=y_train)
validation = xgb.DMatrix(x_validation, label=y_validation)

watchlist = [(train, 'train'), (validation, 'valid')]

bst = xgb.train(params, train, 400, watchlist, early_stopping_rounds=50, verbose_eval=10)

x_test = data_test[['word_match','cosine']]
test = xgb.DMatrix(x_test)
predictions = bst.predict(test)

sub = pd.DataFrame()
sub['test_id'] = data_test['test_id']
sub['is_duplicate'] = predictions
sub.to_csv('simple_xgb.csv', index=False)