{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split\n\nimport warnings\nwarnings.simplefilter(\"ignore\")\n\ntrain_df = pd.read_csv(\"../input/quora-insincere-questions-classification/train.csv\")\n# 只保留50万的数据作为训练集\ndf_train = train_df.loc[:500000, :]\ndf_train, df_valid = train_test_split(df_train, test_size=0.1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(df_train.target.values[:10])\nprint(df_train.head(n=2))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 数据分析部分\n\n1. 首先构建一个词典，这个词典会把单词对应成索引\n2. 通过使用keras内置的分析工具取出训练集中最为常用的20000个单词作为词典\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"# print(df_train.head(n=10))\nprint(df_train.columns)\n\n# create a Vocabulary using the question_text\n\nfrom keras.preprocessing.text import Tokenizer\ndef get_vocab(df, num_words=20000):\n    \"\"\"\n    get the dictionary using the df\n    \"\"\"\n    tokenizer = Tokenizer(num_words=num_words)\n    texts = df.question_text.tolist()\n    tokenizer.fit_on_texts([item.lower() for item in texts])\n    return tokenizer\n# 使用训练数据集构建一个关于训练集的tokenizer\ntokenizer = get_vocab(df_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(tokenizer.texts_to_sequences([\"I Love you\"]))\nprint(tokenizer.texts_to_sequences([\"To be a better man.\"]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"使用序列化方法sequence来实现对句子的padding"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfrom keras.preprocessing.sequence import pad_sequences\n\nMAX_LENGTH = 40\ntrain_X = np.array(pad_sequences(tokenizer.texts_to_sequences(df_train.question_text.tolist()), maxlen=MAX_LENGTH, padding = 'post'))\nvalid_X = np.array(pad_sequences(tokenizer.texts_to_sequences(df_valid.question_text.tolist()), maxlen=MAX_LENGTH, padding = 'post'))\nprint(train_X.shape)\n\ntrain_y, valid_y = np.array(df_train.target.values), np.array(df_valid.target.values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(df_train.question_text[0], train_X[0])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 使用预训练的词向量，并且构建和之前词典中的一个映射关系出来。"},{"metadata":{},"cell_type":"markdown","source":"* 首先建立一个关于所有单词的embedding矩阵，，把这个矩阵保存起来，之后使用这个矩阵来初始化模型后面的embedding-layer就行了。"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# 这步是把txt文档转成向量 搜的别人的代码\n# using embedding here to get the numpy array for later useage\nembeddings_index = {}\nfile = open('../input/quora-insincere-questions-classification/embeddings/glove.840B.300d/glove.840B.300d.txt')\nfor line in tqdm(file):\n    values = line.split(\" \")\n    word = values[0]\n    coefs = np.asarray(values[1:], dtype='float32')\n    embeddings_index[word] = coefs\nfile.close()\n\nprint('Found %s word vectors.' % len(embeddings_index))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(tokenizer.num_words)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**构建一个matrix，这个matrix和之前建立的词典 tokenizer.word_index 要一一对应其起来，之后使用这个matrix来作为embedding的初始化**"},{"metadata":{"trusted":true},"cell_type":"code","source":"vocab_size = len(tokenizer.word_index.items())\n# create a weight matrix for words in training docs\nembedding_matrix = np.random.normal(loc=0, scale=1.0, size=(vocab_size+1, 300))\nfor word, i in tokenizer.word_index.items():\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 使用keras来构建一个模型"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import RNN, LSTM, Dropout, Flatten, Embedding, SpatialDropout1D, Dense","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# define model\n\nmodel = Sequential()\nmodel.add(Embedding(vocab_size+1, 300, input_length=MAX_LENGTH, weights=[embedding_matrix]))\nmodel.add(SpatialDropout1D(0.2))\nmodel.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2))\nmodel.add(Dense(128, activation='relu'))\nmodel.add(Dense(1, activation='sigmoid'))\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 直接进行训练"},{"metadata":{},"cell_type":"markdown","source":"# 训练两轮，每一次batch_size 是128"},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(train_X, train_y, epochs=2, verbose=1, batch_size=256)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}