{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nimport tensorflow as tf\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### 解压数据"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Extract a zip file\nimport zipfile\nzip_ref = zipfile.ZipFile('/kaggle/input/quora-insincere-questions-classification/embeddings.zip', 'r')\nprint(zip_ref.namelist())\nembeddings = zip_ref.open('glove.840B.300d/glove.840B.300d.txt', 'r')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### 读取语料库"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_coefs(word,*arr): \n    return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.decode().split(\" \")) for o in embeddings)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### 读取数据"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/train.csv')\ntest_data = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/test.csv')\nprint('训练集维度：\\n',train_data.shape)\nprint('测试集维度：\\n',test_data.shape)\ntrain_data.sample(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#将数据转化为列表\ntrain_input = list(train_data['question_text'])\ntrain_label = list(train_data['target'])\n\ntest_input = list(test_data['question_text'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### 数据预处理-用空格代替语气等无用词"},{"metadata":{"trusted":true},"cell_type":"code","source":"from nltk.corpus import stopwords\nstop=stopwords.words('english') #调用英语语气词库\n\ndef remove_stop_words(x):\n    for word in stop:\n        token = \" \" + word + \" \"\n        if (x.find(token) != -1): #没找到 返回-1，\n            x = x.replace(token, \" \")\n    return x\n\ntrain_input_rsw = list(map(remove_stop_words, train_input))\ntest_input_rsw = list(map(remove_stop_words, test_input))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"max_features=100000\nembed_size = 300 # 词向量维度\nmax_length = 60 #最长句子长度（即RNN中隐层的长度）","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### 分词，并通过语料库提取嵌入矩阵"},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer=Tokenizer(num_words=max_features)\n#num_words:None或整数,处理的最大单词数量。少于此数的单词丢掉\ntokenizer.fit_on_texts(train_input_rsw)\n#使用一系列文档来生成token词典，texts为list类，每个元素为一个文档。\nword_index = tokenizer.word_index\nn_words=min(max_features,len(word_index))\n\nembedding_matrix = np.zeros((n_words+1, 300))\n\nfor word, i in word_index.items():\n    if i >= max_features: \n        continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: \n        embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### 将文本转换为数字，并对其进行填充处理"},{"metadata":{"trusted":true},"cell_type":"code","source":"sequences = tokenizer.texts_to_sequences(train_input_rsw)\n#将多个文档转换为word下标的向量形式,shape为[len(texts)，len(text)] -- (文档数，每条文档的长度)\ntrain_input_padded = pad_sequences(sequences, maxlen=max_length, padding='post', truncating='post')\nprint(train_input_padded.shape)\n\nsequences = tokenizer.texts_to_sequences(test_input_rsw)\ntest_input_padded = pad_sequences(sequences, maxlen=max_length, padding='post', truncating='post')\nprint(test_input_padded.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_input_padded[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" #shuffling the data\n#np.random.seed(2)\n#trn_idx = np.random.permutation(len(train_data))\n\n#train_X = train_input_padded[trn_idx]\n#train_y = train_data['target'][trn_idx] \n\ntrain_text, cv_text, train_target, cv_target = train_test_split(train_input_padded, train_label, test_size = 0.1, random_state=2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"####  Keras 建模"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Embedding,Bidirectional,LSTM,Dropout,Conv1D,MaxPooling1D,Dense","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lstm=Sequential()\nlstm.add(Embedding(n_words+1,300,input_length=max_length,weights=[embedding_matrix], trainable=False))\nlstm.add(Bidirectional(LSTM(256,return_sequences=True)))\nlstm.add(Dropout(0.2))\nlstm.add(Conv1D(100,5,activation='relu'))\nlstm.add(MaxPooling1D(pool_size=4))\nlstm.add(LSTM(128))\nlstm.add(Dropout(0.4))\nlstm.add(Dense(1,activation='sigmoid'))\n\nlstm.summary()\n\nlstm.compile(loss='binary_crossentropy',optimizer='adam', metrics=['acc'])\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del embeddings_index\nimport gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history=lstm.fit(np.array(train_text), np.array(train_target), epochs = 5, validation_data=(np.array(cv_text),np.array(cv_target)), batch_size=1024,verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n\nacc=history.history['acc']\nval_acc=history.history['val_acc']\nloss=history.history['loss']\nval_loss=history.history['val_loss']\n\nepochs=range(5)\n##精确率图像\nplt.figure()\nplt.plot(epochs, acc, 'r')\nplt.plot(epochs, val_acc, 'b')\nplt.title('Training and validation accuracy')\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"Accuracy\")\nplt.legend([\"Accuracy\", \"Validation Accuracy\"])\nplt.show()\n##损失图像\nplt.figure()\nplt.plot(epochs, loss, 'r')\nplt.plot(epochs, val_loss, 'b')\nplt.title('Training and validation loss')\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"Loss\")\nplt.legend([\"Loss\", \"Validation Loss\"])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### 预测"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import f1_score\ncv_predictions = lstm.predict(cv_text, batch_size=512)\n\nthresholds = []\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    result = f1_score(cv_target, (cv_predictions>thresh).astype(int))\n    thresholds.append([thresh, result])\n    print(\"F1 score at threshold {} is {}\".format(thresh, result))\n\nthresholds.sort(key=lambda x: x[1], reverse=True)\nprint(\"Best value {}\".format(thresholds[0]))\nbest_thresh = thresholds[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = lstm.predict(cv_text)\npredictions = np.around(predictions).astype(int)\ndf = pd.DataFrame({'pred': predictions.flatten(), 'actual': cv_target})\ndf.head()\npd.crosstab(df['pred'], df['actual'], margins=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = lstm.predict(test_input_padded)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions1 = (predictions>best_thresh).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = (predictions>best_thresh).astype(int)\n\nsubmission = pd.DataFrame({'qid': test_data.qid, 'prediction': predictions1[:,0]})\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}