# coding=utf-8

import pandas as pd
import numpy as np
from sklearn.metrics import log_loss
from scipy.optimize import minimize

train = pd.read_csv("../input/train.csv")
raw_corpus = [i.split() for i in train['question1']] + [i.split() for i in train['question2']]
dictionary = corpora.Dictionary(raw_corpus)
corpus = [dictionary.doc2bow(t) for t in raw_corpus]
tfidf = models.TfidfModel(corpus) # 训练一个tf-idf模型

q_1, q_2 = train['question1'], train['question2']
q_tfidf1, q_tfidf2 = tfidf[q_1], tfidf[q_2]
print(q_tfidf1[0])


# train[ 'R' ] = train.apply( word_match_share, axis=1, raw=True )
# print( train.head() ) 

# test = pd.read_csv("../input/test.csv", index_col=False )
# test['R'] = test.apply( word_match_share, axis=1, raw=True )
# print( test.head() )

# #Mean target
# GLOBAL_MEAN = np.mean( train['is_duplicate'] ) 
# print( 'Mean is_duplicated', GLOBAL_MEAN )

# #OPTIMIZE FUNCTIONS
# def minimize_train_log_loss( W ):
#     train["prediction"] = GLOBAL_MEAN + train["R"] * W[0] + W[1]
#     score = log_loss( train['is_duplicate'], train['prediction'] )
#     print(  score , W )
#     return( score )

# res = minimize(minimize_train_log_loss, [0.00,  0.00], method='Nelder-Mead', tol=1e-4, options={'maxiter': 400})
# W = res.x
# print( 'Best weights: ',W )


# #APPLY TO TESTSET
# test["is_duplicate"] = GLOBAL_MEAN + test["R"] * W[0] + W[1]
# test[ ['test_id','is_duplicate'] ].to_csv("count_words_benchmark.csv", header=True, index=False)