{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.\n# 3000条训练数据,F1_Score:0.235,Threshold:0.5,(句向量由question中各个词的词向量取平均得到)\n# 300000条训练数据,F1_Score:0.492,Threshold:0.5,(句向量由question中各个词的词向量取平均得到)\n# 400000条训练数据,F1_Score:0.490,Threshold:0.5,(句向量由question中各个词的词向量取平均得到)\n# 300000条训练数据,F1_Score:0.326,Threshold:0.5,(句向量由question中各个词的词向量乘以其对应的TFIDF值，再取平均得到)\n# 400000条训练数据,F1_Score:0.373,Threshold:0.5,(句向量由question中各个词的词向量乘以其对应的TFIDF值，再取平均得到)\n# 此版本用600000条训练数据,句向量由question中各个词的词向量取平均得到,threshold:0.5\n\n# 之前的版本中都未将单词统一为小写，我们基于以下事实：\n# 在保留单词原有形式时：\n# The number of all words of train.csv and test.csv is : 172995 , The number of Unknown words is : 62346 ,The number of words with Google-vector is : 110649 \n# 可以看到未在Google_vector找到词向量的单词 占 所有单词的比例高达 1/3 ，当然这是基于600000条数据做的判断；\n\n# 在将单词统一为小写时：\n# The number of all words of train.csv and test.csv is : 144678 , The number of Unknown words is : 80684 ,The number of words with Google-vector is : 63994\n# 可以看到总单词量下降了3万左右，未知单词的数量增加了2万左右 ；\n\n# 只将question_text首单词变为小写时：\n# The number of all words of train.csv and test.csv is : 172917 , The number of Unknown words is : 62546 ,The number of words with Google-vector is : 110371 \n# 可以看到总单词量相比保留原有单词形式时，没有太大变化；\n\n# 此版本做如下改进：\n# 1. 将question首单词变为小写； \n# 2. 句向量的构成：用各词向量各维度累加之和来表示","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Data_stage_1.py\n# -*- coding: utf-8 -*-\n# @Time    : 2019/1/7 14:56\n# @Author  : Weiyang\n\n'''\n目标：读取train.csv和test.csv数据，去除标点符号等特殊符号，将question变为word列表，单词转为小写、\n输入：train.csv\n输出1：questions = [[word1,word2,word3,..],...]\n输出2：labels = [1,0,1,...]\n输出3：all_words = [word1,word2,...] train.csv 和 test.csv 的词表,用以查找词向量以及随机生成词向量\n输出4：test_questions = [[word1,word2,word3,..],...]\n输出5：test_qids = [001256,05891,...]\n注意:若去除特殊符号后,question为空,则将question直接删除,其不参与训练；而如果test.csv中出现此情况,则question类别确定为1,也无需参与预测;\n'''\n\nimport pandas as pd\nimport re\nimport time\nimport numpy as np\n\nstart = time.time()\n# read train.csv\ntrain_data = pd.read_csv('../input/train.csv',encoding='utf-8-sig')\n\n# 去除标点符号等特殊符号，将question变为词列表,将单词转为小写\nquestions = [] # [[word1,word2,..],..] question文本且分成一个一个词\nlabels = [] # labels\nall_words = [] # [word1,word2,...]\nsymbols = ['\\''] # don't, I'm\nrandint = np.random.randint(0,len(train_data['question_text']),100) # 产生100个随机整数\ncount = 0\nfor question,label in zip(train_data['question_text'],train_data['target']):\n    if count > 600000:\n        break\n    if count in randint:\n        print('train.csv: ',question)\n    question = re.sub(\"[\\s+\\.\\!\\/_,\\\\:;><{}\\-$%^*()+\\\"\\[\\]]+|[+——！，。：；》《？?、~@#￥%……&*（）]+\",' ',question)\n    question = [str(word) for word in question.split() if len(word.strip()) !=0 and word not in symbols]\n    question = [word.rstrip('\\'').lstrip('\\'') for word in question] # 去除左右两侧的单引号\n    question = [word.rstrip('\\\\').lstrip('\\\\') for word in question]  # 去除左右两侧的\\\\\n    question = [word.rstrip('’').lstrip('’') for word in question]  # 去除左右两侧的’\n    question = [word.rstrip('‘').lstrip('‘') for word in question]  # 去除左右两侧的‘\n    question = [word.rstrip('”').lstrip('”') for word in question]  # 去除左右两侧的”\n    question = [word.rstrip('“').lstrip('“') for word in question]  # 去除左右两侧的“\n    question = [word.rstrip('`').lstrip('`') for word in question]  # 去除左右两侧的` 保留单词原有的形式\n    #question = [word.lower() for word in question]  # 将单词统一为小写\n    \n    # 将question首个单词变为小写，其余单词形式不变\n    temp = []\n    for i in range(len(question)):\n        if i == 0:\n            temp.append(question[i].lower())\n        else:\n            temp.append(question[i])\n    question = temp[:]\n    \n    # 判断question是否为空\n    if len(question) == 0:\n        continue\n    all_words.extend(question)\n    questions.append(question)\n    labels.append(label)\n    if count in randint:\n        print('train.csv: ',question)\n    count += 1\n\ndel train_data\n\n# read test.csv\ntest_data = pd.read_csv('../input/test.csv',encoding='utf-8-sig')\n\n# 去除标点符号等特殊符号，将question变为词列表,将单词转为小写\ntest_questions = [] # [[word1,word2,..],..] question文本且分成一个一个词\ntest_qids = [] #qids\nempty_question_qids = [] # 如果去除特殊符号后,question为空,则question类别为1,即是insincere question;其不再参与预测\nsymbols = ['\\''] # don't,I'm,...\nrandint = np.random.randint(0,len(test_data['question_text']),100) # 产生100个随机整数\ncount = 0\nfor question,qid in zip(test_data['question_text'],test_data['qid']):\n    #if count > 3000:\n        #break\n    if count in randint:\n        print('test.csv: ',question)\n    question = re.sub(\"[\\s+\\.\\!\\/_,\\\\:;><{}\\-$%^*()+\\\"\\[\\]]+|[+——！，。：；》《？?、~@#￥%……&*（）]+\",' ',question)\n    question = [str(word) for word in question.split() if len(word.strip()) !=0 and word not in symbols]\n    question = [word.rstrip('\\'').lstrip('\\'') for word in question] # 去除左右两侧的单引号\n    question = [word.rstrip('\\\\').lstrip('\\\\') for word in question]  # 去除左右两侧的\\\\\n    question = [word.rstrip('’').lstrip('’') for word in question]  # 去除左右两侧的’\n    question = [word.rstrip('‘').lstrip('‘') for word in question]  # 去除左右两侧的‘\n    question = [word.rstrip('”').lstrip('”') for word in question]  # 去除左右两侧的”\n    question = [word.rstrip('“').lstrip('“') for word in question]  # 去除左右两侧的“\n    question = [word.rstrip('`').lstrip('`') for word in question]  # 去除左右两侧的` 且保留单词原有的形式不变\n    #question = [word.lower() for word in question]  # 将单词统一为小写\n    \n    # 将question首个单词变为小写，其余单词形式不变\n    temp = []\n    for i in range(len(question)):\n        if i == 0:\n            temp.append(question[i].lower())\n        else:\n            temp.append(question[i])\n    question = temp[:]\n    \n    # 判断question是否为空\n    if len(question) == 0:\n        empty_question_qids.append(qid)\n        continue\n    test_questions.append(question)\n    test_qids.append(qid)\n    all_words.extend(question)\n    if count in randint:\n        print('test.csv: ',question)\n    count += 1\n\ndel test_data\nall_words = set(all_words)\n\nprint('The total time of Data_stage_1.py program is : %d s' %(time.time()-start))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4550866a38bcdd94e33de95c5d851d2e506a599d"},"cell_type":"code","source":"# Data_stage_2.py\n# -*- coding: utf-8 -*- \n# @Time    : 2019/1/7 16:59 \n# @Author  : Weiyang\n\n'''\n数据探索\n目标：探索sincere question 和 insincere question 中特征词的分布\n输入：questions\n#输出1：different_words = [word1,word2,...]# sincere question 和 insincere question 各自独有的词\n输出2：words_idf = {word1:idf,word2:idf,...} 即找出questions中每个word的idf值，此字典可用于train.csv和test.csv\n注：这里利用Gensim自动去除停用词\n'''\nimport warnings\nwarnings.filterwarnings(action=\"ignore\")\nfrom collections import Counter\nfrom gensim import corpora ,models\nimport math\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport time\n\nstart = time.time()\n\n##################################################  探索数据 ###########################################################\n#统计question长度分布\nquestion_len = [len(question) for question in questions]\nprint('The max length of question is : %d'% max(question_len))\nprint('The min length of question is : %d'% min(question_len))\nprint('The average length of question is : %d'% (sum(question_len)/float(len(question_len))))\nc = Counter(question_len).most_common(100)\nprint('The top 100 questions length with the largest number of length are : ')\nprint(c)\n\n'''\n# 分别以word 的 dfs 倒序输出 insincere question 和 sincere question 中 的 1/idf ，找出与两个类别最相关的词\ninsincere_questions = [questions[i] for i in range(len(questions)) if labels[i] == 1]\nsincere_questions = [questions[i] for i in range(len(questions)) if labels[i] == 0]\n\ninsincere_dictionary = corpora.Dictionary(insincere_questions)\nid2token = {} # {id:token,..} # id:word\nfor token,id in insincere_dictionary.token2id.items():\n    id2token[id] = token\ninsincere_token_frequency = {} # {word:1/idf,...},value是文档频数,指该词出现在不同question的次数/总文档的数量\nfor id,dfs in insincere_dictionary.dfs.items():\n    insincere_token_frequency[id2token[id]] = dfs/float(len(insincere_questions))\n\nsincere_dictionary = corpora.Dictionary(sincere_questions)\nid2token = {} # {id:token,..} # id:word\nfor token,id in sincere_dictionary.token2id.items():\n    id2token[id] = token\nsincere_token_frequency = {} # {word:1/idf,...},value是文档频数指该词出现在不同question的次数/总文档的数量\nfor id,dfs in sincere_dictionary.dfs.items():\n    sincere_token_frequency[id2token[id]] = dfs/float(len(sincere_questions))\n\ninsincere_words = [] # 存储出现在insincere question中的word ,word 以 文档频率 为准做倒排序，文档频数指该词出现在不同question的次数/总文档的数量\nsincere_words = [] # 存储出现在sincere question中的word ,word 以 文档频率 为准做倒排序，文档频数指该词出现在不同question的次数/总文档的数量\nins = sorted(insincere_token_frequency.items(),key=lambda x : x[1],reverse=True)\nsin = sorted(sincere_token_frequency.items(),key=lambda x : x[1],reverse=True)\n#print(ins)\n#print(sin)\nplt.figure()\ninsincere_X = [] # 单词本身，用作标签\ninsincere_Y = [] # 单词文档频率\nfor item in ins:\n    insincere_X.append(item[0])\n    insincere_Y.append(item[1])\ninsincere_X = np.array(insincere_X)\ninsincere_Y = np.array(insincere_Y)\nplt.subplot(2,1,1)\nplt.bar(insincere_X[:30],insincere_Y[:30])\n\n\nsincere_X = [] # 单词本身，用作标签\nsincere_Y = [] # 单词文档频率\nfor item in sin:\n    sincere_X.append(item[0])\n    sincere_Y.append(item[1])\nsincere_X = np.array(sincere_X)\nsincere_Y = np.array(sincere_Y)\nplt.subplot(2,1,2)\nplt.bar(sincere_X[:30],sincere_Y[:30])\nprint('Print the top 30 words of insincere question and sincere question with its 1/idf...')\n#plt.show()\n\n# insincere questions 和 sincere questions 都存在的单词\nshare_words = set(list(insincere_X)) & set(list(sincere_X))\nprint(share_words)\n# insincere questions 和 sincere questions 各自特殊的单词\ndifferent_words = set(list(insincere_X) + list(sincere_X)) - share_words\nprint(different_words)\n\n'''\n################################################# 输出word 的tfidf idf #################################################\n#输出每个question中每个单词的TF-IDF值\ndictionary = corpora.Dictionary(questions) # 生成字典语料\nid2token = {} # {id:token,..} # id:word\nfor token,id in dictionary.token2id.items():\n    id2token[id] = token\n\nid_dfs = {} # {id:dfs,...} dfs是 文档频数,即有多少文档包含该单词\nfor id,dfs in dictionary.dfs.items():\n    id_dfs[id] = dfs\n\ncorpus = [dictionary.doc2bow(question) for question in questions] # 得到每个question对应的稀疏向量(BOW向量)\nmodel = models.TfidfModel(corpus) # 对每个question中每个单词统计其TFIDF值\nvectors = model[corpus] # 将question转为BOW向量，格式为[(id,tfidf),...]\nprint('Starting output words_idf...')\nquestion_num = len(questions)\nwords_idf = {} # {word:idf,...}\ncount = 0\nfor question_vector in vectors:\n    tfidfs = {}\n    idfs = {}\n    for id,tfidf in question_vector:\n        tfidfs[id2token[id]] = tfidf\n        idf = math.log(float(question_num)/(id_dfs[id]+1),2)\n        idfs[id2token[id]] = idf\n        words_idf[id2token[id]] = words_idf.get(id2token[id],None)\n        words_idf[id2token[id]] = idf\n    # 如果question_vector没有相应word的tfidf，则其值取0(Gensim会自动去除停用词，以及删掉某些单个字符)\n    # Gensim并不能对所以word都计算tfidf\n    for word in questions[count]:\n        if word not in tfidfs and word not in idfs:\n            words_idf[word] = 10000\n    count += 1\n# 遍历all_words，如果该词在words_idf中没有idf值，则赋值为10000,若为0,则word代表着很强的特征意义\nfor word in all_words:\n    if word not in words_idf:\n        words_idf[word] = 10000\ndel dictionary,id2token,corpus,model,vectors,question_num,count,c,question_len\n\nprint('Finishing output words_idf...')\nprint('The total time of Data_stage_2.py program is : %d s' %(time.time()-start))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e71eb74e11b33b9c43dece2694572b6d95f955e"},"cell_type":"code","source":"# Data_stage_3.py\n# -*- coding: utf-8 -*- \n# @Time    : 2019/1/8 10:03 \n# @Author  : Weiyang\n\n'''\nread GoogleNews-vectors-negative300\n目标：加载GoogleNews-vectors-negative300.bin词向量，如果对应词不存在词向量，则利用均匀分布随机生成一个300维的向量作词向量\n输入：GoogleNews-vectors-negative300.bin(Google预训练的词向量),all_words:train.csv 和 test.csv中出现的词\n输出：words_vector,通过word_vector_model[word]可以得到每个词对应的词向量,如果单词没有对应的词向量,则随机赋予一个300维的词向量\n'''\n\nimport warnings\nwarnings.filterwarnings(action=\"ignore\")\nimport numpy as np\nfrom gensim.models.keyedvectors import KeyedVectors\nimport time\nimport sys\n\n# loads 300x1 word vectors from file.\ndef load_bin_vec(fname, vocab):\n    words_vector = KeyedVectors.load_word2vec_format(fname, binary=True) # 通过model来取对应词的词向量\n    return words_vector\n\n# add random vectors of unknown words which are not in pre-trained vector file.\n# if pre-trained vectors are not used, then initialize all words in vocab with random value.\ndef add_unknown_words(words_vector, vocab, k=300):\n    unknown_words_vector = {}\n    for word in vocab:\n        if word not in words_vector:\n            unknown_words_vector[word] = np.random.uniform(-0.25, 0.25, k)\n    return unknown_words_vector\n\nstart = time.time()\n# 预训练的词向量的路径\nvectors_file =  '../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\nprint('Loading all_words ....')\nprint('Loading googleNew vector....')\nwords_vector = load_bin_vec(vectors_file, all_words)  # pre-trained vectors\nunknown_words_vector = add_unknown_words(words_vector, all_words)\nprint('The number of all words of train.csv and test.csv is : %d , The number of Unknown words is : %d ,The number of words with Google-vector is : %d '%(len(all_words),len(unknown_words_vector),(len(all_words)-len(unknown_words_vector))))\ndel all_words,vectors_file,unknown_words_vector\nprint('The size of words_vector is : %d MB'%((sys.getsizeof(words_vector)/1024)))\n#print('The size of unknown_words_vector is : %d MB'%((sys.getsizeof(unknown_words_vector)/1024)))\nprint('The total time of Data_stage_3.py program is : %d s' %(time.time()-start))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ca4e35fac1d119587eb872e203b4935b2a9e79e7"},"cell_type":"code","source":"# Gen_feature.py\n# -*- coding: utf-8 -*- \n# @Time    : 2019/1/8 9:18 \n# @Author  : Weiyang\n\n'''\n目标：生成句向量\n输入：words_vector,questions,test_questions,questions_tfidf,words_idf\n输出1：questions_vector = [[value1,value2,..],..]  question sentence vector # 将各个单词的向量累加取平均作为当前question的句向量\n#输出2：questions_vector_tfidf = [[value1*tfidf,value2*tfidf,..],..] question sentence vector * tfidf  # 将各个单词的向量乘以其对应的tfidf值，再取平均作为当前question的句向量\n#输出3：questions_vector_idf = [[value1*idf,value2*idf,...],..] question sentence vector * idf  # 将各个单词的向量乘以其对应的idf值，再取平均作为当前question的句向量\n输出4：test_questions_vector = [[value1,value2,..],..]  question sentence vector # 将各个单词的向量累加取平均作为当前question的句向量\n'''\nimport warnings\nwarnings.filterwarnings(action=\"ignore\")\nimport numpy as np\nfrom collections import Counter\nimport sys\nimport time\n\nstart = time.time()\n\n\n# output question sentence vectors\nprint('Outputing question sentence vectors...')\nquestions_vector = [] \ntest_questions_vector = []\nfor i in range(len(questions)):\n    words = [word for word in questions[i]]\n    vector = []\n    for word in questions[i]:\n        if word in words_vector:\n            vector.append(words_vector[word]) # [[value1,value2,..],...]\n        else:\n            #vector.append(unknown_words_vector[word]) # [[value1,value2,..],...]\n            vector.append(np.random.uniform(-0.25, 0.25, 300)) # 如果词向量不存在则随机赋予一个300维的词向量\n    vector_lst = []\n    for j in range(len(vector)):\n        if words_idf[words[j]] != 10000: # 如果word的idf值存在，则word不用去除\n            vector_lst.append((np.array(vector[j]) * 1).tolist())\n        else:# 否则word的idf值不存在，则word是停用词，应去除\n            vector_lst.append((np.array(vector[j]) * 0).tolist())\n    vector = np.mat(vector_lst)\n    #vector = np.mean(vector,axis=0) # 将词向量相加取平均\n    vector = np.sum(vector,axis=0) # 将词向量累加\n    questions_vector.append([float(value) for value in vector.tolist()[0]])\n\ndel questions\nprint('Del questions...')\n    \nfor i in range(len(test_questions)):\n    words = [word for word in test_questions[i]]\n    vector = []\n    for word in test_questions[i]:\n        if word in words_vector:\n            vector.append(words_vector[word]) # [[value1,value2,..],...]\n        else:\n            #vector.append(unknown_words_vector[word]) # [[value1,value2,..],...]\n            vector.append(np.random.uniform(-0.25, 0.25, 300)) # 如果词向量不存在则随机赋予一个300维的词向量\n    vector_lst = []\n    for j in range(len(vector)):\n        if words_idf[words[j]] != 10000: # 如果word的idf值存在，则word不用去除\n            vector_lst.append((np.array(vector[j]) * 1).tolist())\n        else:# 否则word的idf值不存在，则word是停用词，应去除\n            vector_lst.append((np.array(vector[j]) * 0).tolist())\n    vector = np.mat(vector_lst)\n    #vector = np.mean(vector,axis=0) # 将词向量相加取平均\n    vector = np.sum(vector,axis=0) # 将词向量累加\n    test_questions_vector.append([float(value) for value in vector.tolist()[0]])\n\n\n''' \n# output question sentence vectors * tfidf\nprint('Output question sentence vector * tfidf .... ')\nquestions_vector_tfidf = [] \ntest_questions_vector_tfidf = []\nfor i in range(len(questions)):\n    c = Counter(questions[i])\n    words = [word for word in questions[i]]\n    word_frequency = [c[word]/float(len(questions[i])) for word in questions[i]] # 计算word的词频/len(test_question)\n    vector = [] # [[value1,value2,..],...]\n    for word in questions[i]:\n        if word in words_vector:\n            vector.append(words_vector[word]) # [[value1,value2,..],...]\n        else:\n            #vector.append(unknown_words_vector[word]) # [[value1,value2,..],...]\n            vector.append(np.random.uniform(-0.25, 0.25, 300)) # 如果词向量不存在则随机赋予一个300维的词向量\n    vector = np.mat([(np.array(vector[j])*words_idf[words[j]]*float(word_frequency[j])).tolist() for j in range(len(vector))]) # 对应词乘以其tfidf值\n    vector = np.mean(vector,axis=0) # 将词向量相加取平均\n    #vector = np.sum(vector,axis=0) # 将词向量累加\n    questions_vector_tfidf.append([float(value) for value in vector.tolist()[0]])\n\ndel questions\nprint('Del questions...')\n\nfor i in range(len(test_questions)):\n    c = Counter(test_questions[i])\n    words = [word for word in test_questions[i]]\n    word_frequency = [c[word]/float(len(test_questions[i])) for word in test_questions[i]] # 计算word的词频/len(test_question)\n    vector = [] # [[value1,value2,..],...]\n    for word in test_questions[i]:\n        if word in words_vector:\n            vector.append(words_vector[word]) # [[value1,value2,..],...]\n        else:\n            #vector.append(unknown_words_vector[word]) # [[value1,value2,..],...]\n            vector.append(np.random.uniform(-0.25, 0.25, 300)) # 如果词向量不存在则随机赋予一个300维的词向量\n    vector = np.mat([(np.array(vector[j])*words_idf[words[j]]*float(word_frequency[j])).tolist() for j in range(len(vector))]) # 对应词乘以其tfidf值\n    vector = np.mean(vector,axis=0) # 将词向量相加取平均\n    #vector = np.sum(vector,axis=0) # 将词向量累加\n    test_questions_vector_tfidf.append([float(value) for value in vector.tolist()[0]])\n\n'''\n'''    \n# output question sentence vectors * idf\nprint('Output question sentence vector * idf .... ')\nquestions_vector_idf = [] \ntest_questions_vector_idf = []\nfor i in range(len(questions)):\n    words = [word for word in questions[i]]\n    vector = [] # [[value1,value2,..],...]\n    for word in questions[i]:\n        if word in words_vector:\n            vector.append(words_vector[word]) # [[value1,value2,..],...]\n        else:\n            #vector.append(unknown_words_vector[word]) # [[value1,value2,..],...]\n            vector.append(np.random.uniform(-0.25, 0.25, 300)) # 如果词向量不存在则随机赋予一个300维的词向量\n    vector = np.mat([(np.array(vector[j])*words_idf[words[j]]).tolist() for j in range(len(vector))]) # 对应词乘以其idf值\n    vector = np.mean(vector,axis=0) # 将词向量相加取平均\n    #vector = np.sum(vector,axis=0) # 将词向量累加\n    questions_vector_idf.append([float(value) for value in vector.tolist()[0]])\n\ndel questions\nprint('Del questions...')\n\nfor i in range(len(test_questions)):\n    words = [word for word in test_questions[i]]\n    vector = [] # [[value1,value2,..],...]\n    for word in test_questions[i]:\n        if word in words_vector:\n            vector.append(words_vector[word]) # [[value1,value2,..],...]\n        else:\n            #vector.append(unknown_words_vector[word]) # [[value1,value2,..],...]\n            vector.append(np.random.uniform(-0.25, 0.25, 300)) # 如果词向量不存在则随机赋予一个300维的词向量\n    vector = np.mat([(np.array(vector[j])*words_idf[words[j]]).tolist() for j in range(len(vector))]) # 对应词乘以其idf值\n    vector = np.mean(vector,axis=0) # 将词向量相加取平均\n    #vector = np.sum(vector,axis=0) # 将词向量累加\n    test_questions_vector_idf.append([float(value) for value in vector.tolist()[0]])\n'''\ndel words_vector,test_questions,words_idf\nprint('The total time of Gen_feature.py program is : %d s' %(time.time()-start))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d2952b6e0840b8c93b513c1168c9d6bdb639523"},"cell_type":"code","source":"# xgb_model.py\n# -*- coding: utf-8 -*- \n# @Time    : 2019/1/8 21:52 \n# @Author  : Weiyang\n\n'''\n模块目标：用xgboost实现分类\n输入1：questions_vector,test_questions_vector,labels,test_qid\n输入2：questions_vector_tfidf,test_questions_vector_tfidf,labels,test_qid\n输入3：questions_vector_idf,test_questions_vector_idf,labels,test_qid\n'''\n\nimport xgboost as xgb\nfrom xgboost import plot_importance\nfrom sklearn.metrics import precision_score\nfrom sklearn.metrics import recall_score\nfrom sklearn.metrics import f1_score\nfrom matplotlib import pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport time\n\nclass xgb_model:\n\n    def __init__(self):\n        self.data=[] # 存储vector\n        self.label_true=[] # 存储label\n        self.rate=0.9 #训练数据和预测数据的比例\n        self.model=None #保存训练好的模型\n\n    def load_data(self,train_data,label_data):\n        \"\"\"读取训练数据,格式为:value1 \\t value2 \\t value3 \\t...\\t label\"\"\"\n        self.data = train_data\n        self.label_true = label_data\n\n    def train(self,is_incremental=False,batch_size=300000):\n        \"\"\"训练模型：由于采用增量学习的方式来训练模型,is_incremental指是否需要增量学习,batch_size指每份训练集的大小\"\"\"\n        split_num=int(self.rate*len(self.data))\n        x_train=self.data[:split_num]\n        y_train=self.label_true[:split_num]\n        num_rounds=100 #训练迭代次数\n        params={\n            'objective':'binary:logistic',\n            'max_depth':2,\n            'silent':1,\n            'eta':1,\n            'nthread':4\n        }\n        if is_incremental == True:\n            # 将训练集切割成 n 份,采用增量学习的方式来训练模型\n            # batch_size : 每份训练集的大小\n            for start in range(0,len(x_train),batch_size):\n                self.model = xgb.train(params,dtrain=xgb.DMatrix(x_train[start:start+batch_size],y_train[start:start+batch_size]),xgb_model=self.model)\n        else:\n            dtrain=xgb.DMatrix(x_train,y_train)\n            self.model=xgb.train(params,dtrain,num_rounds,xgb_model=self.model)\n        #显示重要特征,值越大说明该特征越重要\n        #plot_importance(self.model)\n        #plt.show()\n        # 输出在测试集上的结果\n        x_test = self.data[split_num:]\n        y_test = self.label_true[split_num:]\n        dtest = xgb.DMatrix(x_test)\n        label_pre = self.model.predict(dtest)\n        label_pre = list(label_pre)\n        #寻找最佳阈值\n        thresholds_f1 = []\n        thresholds_precision = []\n        thresholds_recall = []\n        for thresh in np.arange(0.1,0.501,0.01):\n            thresh = np.round(thresh,2)\n            res_f1 = f1_score(y_test,label_pre >= thresh).astype(np.float32)\n            res_precision = precision_score(y_test,label_pre >= thresh).astype(np.float32)\n            res_recall = recall_score(y_test,label_pre >= thresh).astype(np.float32)\n            thresholds_f1.append([thresh,res_f1])\n            thresholds_precision.append([thresh, res_precision])\n            thresholds_recall.append([thresh, res_recall])\n            print(\"Threshold:%.4f , Precision:%.4f , Recall:%.4f ,F1:%.4f \" % (thresh,precision_score(y_test, label_pre >= thresh), recall_score(y_test, label_pre >= thresh),f1_score(y_test, label_pre >= thresh)))   \n        print('*'*100)\n        thresholds_f1.sort(key=lambda x : x[1],reverse=True)\n        best_thresh = thresholds_f1[0][0]\n        print('Best F1 Score at threshold {0} is {1}'.format(best_thresh,thresholds_f1[0][1]))\n\n        thresholds_precision.sort(key=lambda x: x[1], reverse=True)\n        best_thresh = thresholds_precision[0][0]\n        print('Best Precision at threshold {0} is {1}'.format(best_thresh, thresholds_precision[0][1]))\n\n        thresholds_recall.sort(key=lambda x: x[1], reverse=True)\n        best_thresh = thresholds_recall[0][0]\n        print('Best Recall at threshold {0} is {1}'.format(best_thresh, thresholds_recall[0][1]))\n\n    def cross_validation(self):\n        \"\"\"交叉验证训练模型\"\"\"\n        x_train = self.data\n        y_train = self.label_true\n        dtrain = xgb.DMatrix(x_train, y_train)\n        num_rounds = 500  # 训练迭代次数\n        params={\n            'objective':'binary:logistic',\n            'max_depth':2,\n            'silent':0,\n            'eta':1,\n            'nthread':4\n        }\n        result=xgb.cv(params,dtrain,num_rounds,nfold=10)#10折交叉验证,全部数据\n        for line in result:\n            print(line)\n\n    def predict(self,test_data,test_qid,threshold):\n        \"\"\"模型预测:test_data_path:测试数据路径,model_path:加载训练好的模型,result_path:测试结果保存路径,threshold阈值，大于等于则为label 1 小于则为 label 0\"\"\"\n        #读取训练数据,格式为:qid \\t value1 \\t value2 \\t....\n        test_data = test_data # 存储question的vector\n        test_qid = test_qid # 存储question的qid\n        dtest=xgb.DMatrix(test_data)\n        # load trained model\n        label_pre=self.model.predict(dtest)\n        label_pre = [int(prob >= threshold) for prob in list(label_pre)]\n        # 输出预测结果\n        result = pd.DataFrame(columns=['qid','prediction'])\n        result['qid'] = test_qid\n        result['prediction'] = label_pre\n        #print(label_pre)\n        return result\n\nstart = time.time()\nx = xgb_model()\n# 训练模型\nprint(len(questions_vector),len(labels))\nx.load_data(questions_vector,labels)\nx.train(is_incremental=True,batch_size=600000) #增量学习,每份训练集包含30万条数据\n#x.cross_validation()\n#del questions_vector,labels\n# 模型预测\nsub = x.predict(test_questions_vector,test_qids,0.26)\n# 将question为空的qid的类别添加进去并且设置为1\nif len(empty_question_qids) !=0:\n    empty_questions = pd.DataFrame(columns=['qid','prediction'])\n    empty_questions['qid'] = empty_question_qids\n    empty_questions['prediction'] = [1]*len(empty_question_qids)\n    sub = pd.concat([sub,empty_questions],axis=0)\nsub.to_csv('submission.csv',index=False)\nprint(empty_question_qids)\n\nprint('The total time of xgb_model.py program is : %d min' % ((time.time() - start)/60))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}