{"cells":[{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import re\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\n\nfrom nltk.corpus import stopwords\nfrom nltk.stem import SnowballStemmer\nfrom gensim.models import KeyedVectors\nfrom sklearn.preprocessing import StandardScaler\n\nimport keras\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, Embedding, Dropout, Reshape, Flatten, LSTM, Bidirectional\nfrom keras.layers.merge import concatenate\nfrom keras.models import Model\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# -----------------------------------\n# functions for pre-processing texts \n# -----------------------------------\n\ndef text_to_wordlist(text, remove_stopwords=False, stem_words=False):\n    # Clean the text, with the option to remove stopwords and to stem words.\n    \n    # Convert words to lower case and split them\n    text = text.lower().split()\n\n    # Optionally, remove stop words\n    if remove_stopwords:\n        stops = set(stopwords.words(\"english\"))\n        text = [w for w in text if not w in stops]\n    \n    text = \" \".join(text)\n\n    # Clean the text\n    text = re.sub(r\"[^A-Za-z0-9^,!.\\/'+-=]\", \" \", text)\n    text = re.sub(r\"what's\", \"what is \", text)\n    text = re.sub(r\"\\'s\", \" \", text)\n    text = re.sub(r\"\\'ve\", \" have \", text)\n    text = re.sub(r\"can't\", \"cannot \", text)\n    text = re.sub(r\"n't\", \" not \", text)\n    text = re.sub(r\"i'm\", \"i am \", text)\n    text = re.sub(r\"\\'re\", \" are \", text)\n    text = re.sub(r\"\\'d\", \" would \", text)\n    text = re.sub(r\"\\'ll\", \" will \", text)\n    text = re.sub(r\",\", \" \", text)\n    text = re.sub(r\"\\.\", \" \", text)\n    text = re.sub(r\"!\", \" ! \", text)\n    text = re.sub(r\"\\/\", \" \", text)\n    text = re.sub(r\"\\^\", \" ^ \", text)\n    text = re.sub(r\"\\+\", \" + \", text)\n    text = re.sub(r\"\\-\", \" - \", text)\n    text = re.sub(r\"\\=\", \" = \", text)\n    text = re.sub(r\"'\", \" \", text)\n    text = re.sub(r\"(\\d+)(k)\", r\"\\g<1>000\", text)\n    text = re.sub(r\":\", \" : \", text)\n    text = re.sub(r\" e g \", \" eg \", text)\n    text = re.sub(r\" b g \", \" bg \", text)\n    text = re.sub(r\" u s \", \" american \", text)\n    text = re.sub(r\"\\0s\", \"0\", text)\n    text = re.sub(r\" 9 11 \", \"911\", text)\n    text = re.sub(r\"e - mail\", \"email\", text)\n    text = re.sub(r\"j k\", \"jk\", text)\n    text = re.sub(r\"\\s{2,}\", \" \", text)\n    \n    # Optionally, shorten words to their stems\n    if stem_words:\n        text = text.split()\n        stemmer = SnowballStemmer('english')\n        stemmed_words = [stemmer.stem(word) for word in text]\n        text = \" \".join(stemmed_words)\n    \n    # Return a list of words\n    return(text)   \n\n# ----\n\ndef read_train_data(file):\n        texts = [] \n        labels = []\n        df_train = pd.read_csv(file)  \n        line_num = 0\n        for idx in range(len(df_train)):\n          #if line_num < 8000: # for test purpose\n            texts.append(text_to_wordlist(df_train['question_text'][idx]))\n            labels.append(df_train['target'][idx])\n            line_num += 1\n        return texts, labels\n    \ndef read_test_data(file):\n        texts = [] \n        ids = []\n        df_test = pd.read_csv(file)\n        line_num = 0\n        for idx in range(len(df_test)):\n          #if line_num < 200: # for test purpose\n            texts.append(text_to_wordlist(df_test['question_text'][idx]))\n            ids.append(df_test['qid'][idx])\n            line_num += 1\n        return texts, ids\n\n# ---\n\ndef preprocess_data(train_data_file, test_data_file, max_seq_len, split_ratio):\n\n        # 1) load train and test datasets\n        texts, labels= read_train_data(train_data_file)  \n        print('Finished loading train.csv: %s samples' % len(texts))\n        \n        test_texts, test_ids = read_test_data(test_data_file)\n        print('Finished loading test.csv: %s samples' % len(test_texts))\n                      \n        # 2) train the tokenizer\n        tokenizer = Tokenizer(num_words=200000)\n        tokenizer.fit_on_texts(texts + test_texts)        \n        word_index = tokenizer.word_index\n        print('%s tokens in total' % len(word_index))\n\n        # 3) sentences to sequences\n        train_sequences = tokenizer.texts_to_sequences(texts)\n        test_sequences = tokenizer.texts_to_sequences(test_texts)\n        x = pad_sequences(train_sequences, maxlen=max_seq_len, padding='post', truncating='post')\n        test_x = pad_sequences(test_sequences, maxlen=max_seq_len, padding='post', truncating='post')\n    \n        # 4) final step\n        num_samples = len(x)\n        perm = np.random.permutation(num_samples)\n        idx = int(num_samples*split_ratio)\n        idx_train = perm[:idx]\n        idx_val = perm[idx:]\n        \n        train_x = x[idx_train]\n        val_x = x[idx_val]\n        \n        y = np.array(labels)\n        train_y = y[idx_train]\n        val_y = y[idx_val]\n            \n        print('Shape of training data: {}'.format(train_x.shape))\n        print('Shape of training label: {}'.format(train_y.shape))\n        print('Shape of val data: {}'.format(val_x.shape))\n        print('Shape of val label: {}'.format(val_y.shape))\n        print('Shape of test data: {}'.format(test_x.shape))\n        \n        return train_x, train_y, val_x, val_y, test_x, test_ids, word_index\n\n# ------------------------------------------\n# functions for generating embedding matrix\n# ------------------------------------------\n\ndef load_embeddings_index(file, embedding_dim):\n  embeddings_index = {} # dict \n  f=open(file)\n  for line in tqdm(f):\n      values = line.split(\" \")\n      word = values[0]\n      coefs = np.asarray(values[1:], dtype='float32')\n      embeddings_index[word] = coefs                     # embeddings for all words in glove are contained here\n  print('Found %s word vectors.' % len(embeddings_index))\n  return embeddings_index\n\ndef generate_embedding_layer(word_index, embeddings_index):\n  nb_words = len(word_index) + 1\n  embeddings_matrix = np.zeros((nb_words, embedding_dim)) # embedding matrix for all words\n\n  word_out_network = []\n  for word, i in word_index.items():\n      embedding_vector = embeddings_index.get(word)       # get embedding vector for a given word\n      if embedding_vector is not None:\n          embeddings_matrix[i] = embedding_vector\n      else:\n          word_out_network.append(word)\n  percent = round(100*len(word_out_network)/len(word_index),1)\n  print('%s percent of words out of network' % percent)\n  #print('Here they are:', word_out_network)\n  return embeddings_matrix\n\n# ------------------------------------------\n# function for building the model\n# ------------------------------------------\n\ndef build_model(max_seq_len, word_index, embedding_dim, embedding_matrix):\n    \n    # 1) Embedding layer\n    inp = Input(shape=(max_seq_len,), dtype='int32')\n\n    x = Embedding(len(word_index)+1,\n                  embedding_dim,\n                  input_length=max_seq_len,\n                  weights=[embedding_matrix],\n                  trainable=False)(inp)\n\n    # 2) LSTM Layer\n    x = LSTM(64,dropout=0.2, recurrent_dropout=0.2)(x)\n    x = Dropout(0.2)(x)\n    x = BatchNormalization()(x)\n\n    # 3) Dense Layer\n    x = Dense(32, activation='relu')(x)\n    x = Dropout(0.2)(x)\n    x = BatchNormalization()(x)\n\n    # 4) Output Layer\n    preds = Dense(1, activation='sigmoid')(x)\n\n    model = Model(inputs=inp, outputs=preds)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78578eab64a477d0a5ad6b1c917ae154868a44df"},"cell_type":"code","source":"# --- Step 1 Preprocessing texts (texts to numerical values)\n\nmax_seq_len = 30\nsplit_ratio = 0.8\ntrain_file = '../input/quora-insincere-questions-classification/train.csv'\ntest_file = '../input/quora-insincere-questions-classification/test.csv'\ntrain_x, train_y, val_x, val_y, test_x, test_ids, word_index = \\\npreprocess_data(train_file, test_file, max_seq_len, split_ratio)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# --- Step 2 Prepare embedding matrix\n\nembedding_dim = 300\nembedding_matrix = np.zeros((max(list(word_index.values())) + 1, embedding_dim), dtype = 'float32')\nembedding_file='../input/quora-insincere-questions-classification/embeddings/glove.840B.300d/glove.840B.300d.txt'\nf = open(embedding_file)\nfor line in tqdm(f):\n    values = line.split(\" \")\n    word = values[0]\n    if word not in word_index:\n       continue\n    embedding_matrix[word_index[word]] = np.asarray(values[1:], dtype='float32')\nf.close()\n\n\n# Slower method below\n#embedding_dim=300\n#embedding_file='../input/quora-insincere-questions-classification/embeddings/glove.840B.300d/glove.840B.300d.txt'\n#embeddings_index = load_embeddings_index(embedding_file, embedding_dim)\n#embeddings_matrix = generate_embedding_layer(word_index, embeddings_index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7a16003f33ff26675d1a9c5f4feafc5357c7b087"},"cell_type":"code","source":"# --- Step 3 Build and train model\n\n#keras.backend.clear_session()\nmodel = build_model(max_seq_len, word_index, embedding_dim, embedding_matrix)\nmodel.compile(loss='binary_crossentropy',\n              optimizer='adam',\n              metrics=['acc'])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nb_epoches = 200\n\nearly_stopping = EarlyStopping(monitor='val_loss', patience=5)\nmodel_name = 'model_best.h5'\nmodel_checkpoint = ModelCheckpoint(model_name, save_best_only=True)\n\nhist = model.fit(train_x, train_y, \\\n                 validation_data=(val_x, val_y), \\\n                 epochs=nb_epoches, batch_size=2048, shuffle=True, verbose=2, \\\n                 callbacks=[early_stopping, model_checkpoint])\n\nmodel.load_weights(model_name)\nbest_val_score = min(hist.history['val_loss']) \nprint('Min val loss is', best_val_score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b326f67d6f1ab028c37699cbb2acccc5b88dd53"},"cell_type":"code","source":"# --- Final step: submission\n\npreds = model.predict(test_x, batch_size=1024, verbose=1)\npreds = (preds > 0.35).astype(int)\n\nsub = pd.DataFrame({'qid':test_ids, 'prediction':preds.ravel()})\nsub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}