{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split\n\nimport warnings\nwarnings.simplefilter(\"ignore\")\n\ntrain_df = pd.read_csv(\"../input/quora-insincere-questions-classification/train.csv\")\ntrain_df.info()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# 保留65万的数据作为训练集\ndf_train = train_df.loc[:650000, :]\ndf_train, df_valid = train_test_split(df_train, test_size=0.1)\n\nprint(df_train.target.values[:10])\n# print(df_train.head(n=10))\nprint(df_train.head(n=3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# backend\nprint(df_train.columns)\n\n# create a Vocabulary using the question_text\nfrom keras.preprocessing.text import Tokenizer\ndef get_vocab(df, num_words=20000):\n    \"\"\"\n    get the dictionary using the df\n    \"\"\"\n    tokenizer = Tokenizer(num_words=num_words)\n    texts = df.question_text.tolist()\n    tokenizer.fit_on_texts([item.lower() for item in texts])\n    return tokenizer\n# Tokenzier by using dictionary on training data\ntokenizer = get_vocab(df_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfrom keras.preprocessing.sequence import pad_sequences\n\nMAX_LENGTH = 40\ntrain_X = np.array(pad_sequences(tokenizer.texts_to_sequences(df_train.question_text.tolist()), maxlen=MAX_LENGTH, padding = 'post'))\nvalid_X = np.array(pad_sequences(tokenizer.texts_to_sequences(df_valid.question_text.tolist()), maxlen=MAX_LENGTH, padding = 'post'))\nprint(train_X.shape)\n\ntrain_y, valid_y = np.array(df_train.target.values), np.array(df_valid.target.values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# using embedding here to get the numpy array for later useage\nembeddings_index = {}\nfile = open('../input/quora-insincere-questions-classification/embeddings/glove.840B.300d/glove.840B.300d.txt')\nfor line in tqdm(file):\n    values = line.split(\" \")\n    word = values[0]\n    coefs = np.asarray(values[1:], dtype='float32')\n    embeddings_index[word] = coefs\nfile.close()\n\nprint('Found %s word vectors.' % len(embeddings_index))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vocab_size = len(tokenizer.word_index.items())\n# Create a weight matrix for words in training docs\nembedding_matrix = np.random.normal(loc=0, scale=1.0, size=(vocab_size+1, 300))\nfor word, i in tokenizer.word_index.items():\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Building a model with using Keras\nfrom keras.models import Sequential\nfrom keras.layers import RNN, LSTM, Dropout, Flatten, Embedding, SpatialDropout1D, Dense\n\n# define model\nmodel = Sequential()\nmodel.add(Embedding(vocab_size+1, 300, input_length=MAX_LENGTH, weights=[embedding_matrix]))\nmodel.add(SpatialDropout1D(0.2))\nmodel.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2))\nmodel.add(Dense(128, activation='relu'))\nmodel.add(Dense(1, activation='sigmoid'))\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(train_X, train_y, epochs=2, verbose=1, batch_size=256)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Prediction\ndf_test = pd.read_csv(\"../input/quora-insincere-questions-classification/test.csv\")\ntest_tokenizer = get_vocab(df_test)\ntest_X = np.array(pad_sequences(tokenizer.texts_to_sequences(df_test.question_text.tolist()), maxlen=MAX_LENGTH, padding = 'post'))\n\npred_y = model.predict([test_X], batch_size=256, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_y = (pred_y > 0.35).astype(int)\nsubmit_pd = pd.DataFrame({\"qid\":df_test[\"qid\"].values})\nsubmit_pd['prediction'] = pred_y\nsubmit_pd.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}