{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nfrom tqdm import tqdm\nimport re\nimport numpy as np\n\nfrom nltk.corpus import stopwords\nfrom nltk.stem import SnowballStemmer\nfrom gensim.models import KeyedVectors\nfrom sklearn.preprocessing import StandardScaler # Randomize Data\n\nimport keras\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense,Input,Embedding,Dropout,Reshape,Flatten,LSTM,Bidirectional\nfrom keras.layers.merge import concatenate\nfrom keras.models import Model\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.callbacks import EarlyStopping,ModelCheckpoint","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def text_to_wordlist(text , remove_stopwords = False , stem_words = False):\n    \n    # Clean The text , with the option to remove stopwords and to stem words.\n    \n    # Convert words to lower case and split them\n    text = text.lower().split()\n    \n    # Optionally , remove stop words\n    \n    if remove_stopwords:\n        stops = set(stopwords.words('english'))\n        text= [w for w in text if not w in stops]\n        \n    text = ' '.join(text)\n    \n    # Clean the text\n    \n    text = re.sub(r\"[^A-Za-z0-9^,!.\\/'+-=]\", \" \", text)\n    text = re.sub(r\"what's\", \"what is \", text)\n    text = re.sub(r\"\\'s\", \" \", text)\n    text = re.sub(r\"\\'ve\", \" have \", text)\n    text = re.sub(r\"can't\", \"cannot \", text)\n    text = re.sub(r\"n't\", \" not \", text)\n    text = re.sub(r\"i'm\", \"i am \", text)\n    text = re.sub(r\"\\'re\", \" are \", text)\n    text = re.sub(r\"\\'d\", \" would \", text)\n    text = re.sub(r\"\\'ll\", \" will \", text)\n    text = re.sub(r\",\", \" \", text)\n    text = re.sub(r\"\\.\", \" \", text)\n    text = re.sub(r\"!\", \" ! \", text)\n    text = re.sub(r\"\\/\", \" \", text)\n    text = re.sub(r\"\\^\", \" ^ \", text)\n    text = re.sub(r\"\\+\", \" + \", text)\n    text = re.sub(r\"\\-\", \" - \", text)\n    text = re.sub(r\"\\=\", \" = \", text)\n    text = re.sub(r\"'\", \" \", text)\n    text = re.sub(r\"(\\d+)(k)\", r\"\\g<1>000\", text)\n    text = re.sub(r\":\", \" : \", text)\n    text = re.sub(r\" e g \", \" eg \", text)\n    text = re.sub(r\" b g \", \" bg \", text)\n    text = re.sub(r\" u s \", \" american \", text)\n    text = re.sub(r\"\\0s\", \"0\", text)\n    text = re.sub(r\" 9 11 \", \"911\", text)\n    text = re.sub(r\"e - mail\", \"email\", text)\n    text = re.sub(r\"j k\", \"jk\", text)\n    text = re.sub(r\"\\s{2,}\", \" \", text)\n    \n    \n    # Optionally , shorten words to their stems\n    \n    if stem_words:\n        text = text.split()\n        stemmer = SnowballStemmer('english')\n        stemmed_words = [stemmer.stem(word) for word in text]\n        text = ' '.join(stemmed_words)\n        \n    # Return a list of words\n    return(text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_train_data(file):\n    texts = []\n    labels = []\n    df_train = pd.read_csv(file)\n    line_num = 0\n    for idx in range(len(df_train)):\n        \n        # if Line_num < 8000: # For test purpose\n        \n        texts.append(text_to_wordlist(df_train['question_text'][idx]))\n        labels.append(df_train['target'][idx])\n        line_num += 1\n    return texts,labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_test_data(file):\n    texts = []\n    ids = []\n    df_test = pd.read_csv(file)\n    line_num  = 0\n    for idx in range(len(df_test)):\n        # if line_num < 200 : # for test purpose\n        texts.append(text_to_wordlist(df_test['question_text'][idx]))\n        ids.append(df_test['qid'][idx])\n        line_num += 1 \n    return texts , ids","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocess_data(train_data_file , test_data_file , max_seq_len , split_ratio):\n    \n    # 1) load train and test datasets\n    texts , labels = read_train_data(train_data_file)\n    \n    print('Finished loading train.csv: %s samples '%len(texts))\n    \n    test_texts , test_ids = read_test_data(test_data_file)\n    \n    print('Finished loading test.csv: %s samples '%len(test_texts))\n    \n    # 2) train the tokenizer\n    \n    tokenizer = Tokenizer(num_words = 200000)\n    tokenizer.fit_on_texts(texts + test_texts)\n    word_index = tokenizer.word_index\n    print('%s tokens in total' % len(word_index))\n    \n    # 3) sentences to sequences\n    \n    train_sequences = tokenizer.texts_to_sequences(texts)\n    test_sequence = tokenizer.texts_to_sequences(test_texts)\n    x = pad_sequences(train_sequences , maxlen = max_seq_len , padding='post',truncating='post')\n    test_x = pad_sequences(test_sequence , maxlen=max_seq_len , padding='post',truncating = 'post')\n    \n    # 4) final step\n    \n    num_samples = len(x)\n    perm = np.random.permutation(num_samples)\n    idx = int(num_samples * split_ratio)\n    idx_train = perm[:idx]\n    idx_val = perm[idx:]\n    \n    train_x = x[idx_train]\n    val_x = x[idx_val]\n    \n    y = np.array(labels)\n    train_y = y[idx_train]\n    val_y = y[idx_val]\n    \n    print('shape of taining data: {}'.format(train_x.shape))\n    print('shape of training label: {}'.format(train_y.shape))\n    print('shape of val data: {}'.format(val_x.shape))\n#     print('shape of test data : {}'.format(test_x.shape))\n    \n    return train_x , train_y,val_x , val_y , test_x , test_ids , word_index\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Generating embedding matrix\n\ndef load_embeddings_index(file , embedding_dim):\n    embeddings_index = {} # dict\n    f = open(file)\n    for line in tqdm(f):\n        values = line.split(' ')\n        word = values[0]\n        coefs = np.asarray(values[1:],dtype = 'float32')\n        # Ex ~> word : 123 233 133 232\n        embeddings_index[word] = coefs\n        \n        # embeddings for all words in glove are contained here\n    \n    print('Found %s word vectors' % len(embeddings_index))\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def generate_embedding_layer(word_index , embeddings_index):\n    \n        nb_words = len(word_index) + 1\n        embeddings_matrix = np.zeros((nb_words , embedding_dim)) #embedding matrix for all words\n        \n        word_out_netword = []\n        \n        for word , i in word_index.items():\n            embedding_vector = embeddings_index.get(word) # Get embedding vector for a given word\n            \n            if embedding_vector is not None:\n                embeddings_matrix[i] = embedding_vector\n                \n            else:\n                word_out_netword.append(word)\n        \n        percent = round(100 * len(word_out_network) / len(word_index),1)\n        print('%s precent of words out of network' % percent)\n        \n        return embeddings_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Function for building the model\n\ndef build_model(max_seq_len , word_index , embedding_dim , embedding_matrix):\n    \n    # 1) Embedding Layer\n    \n    inp = Input(shape=(max_seq_len,),dtype = 'int32')\n    \n    x = Embedding(len(word_index) + 1,\n                 embedding_dim,\n                 input_length= max_seq_len,\n                 weights = [embedding_matrix],\n                 trainable = False)(inp)\n    \n    # 2) LSTM Layer\n    \n    x = LSTM(64 , dropout = 0.2 , recurrent_dropout=0.2)(x)\n    x = Dropout(0,2)(x)\n    x = BatchNormalization()(x)\n    \n    # 3) Dense Layer\n    \n    x = Dense(32 , activation='relu')(x)\n    x = Dropout(0.2)(x)\n    x = BatchNormalization()(x)\n    \n    # 4) Output layer\n    \n    preds = Dense(1,activation='sigmoid')(x)\n    \n    model = Model(inputs = inp , outputs = preds)\n    return model\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# --- step 1) preprocessing texts (texts to numberical values)\n\nmax_seq_len = 30\nsplit_ratio = 0.8\ntrain_file = '../input/quora-insincere-questions-classification/train.csv'\ntest_file = '../input/quora-insincere-questions-classification/test.csv'\n\ntrain_x , train_y , val_x , val_y , test_x , test_idx , word_index = \\\npreprocess_data(train_file , test_file , max_seq_len , split_ratio)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# --- Step 2) Prepare embedding matrix\n\n\nembedding_dim = 300\nembedding_matrix = np.zeros((max(list(word_index.values())) + 1 , embedding_dim),dtype = 'float')\nembedding_file = '../input/quora-insincere-questions-classification/embeddings/glove.840B.300d/glove.840B.300d.txt'\nf = open(embedding_file)\nfor line in tqdm(f):\n    values = line.split(' ')\n    word = values[0]\n    if word not in word_index:\n        continue\n    embedding_matrix[word_index[word]] = np.asarray(values[1:],dtype = 'float32')\n    f.close","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# -- Step 3) Build The model\n\n# keras.backend_clear_session()\n\nmodel = build_model(max_seq_len , word_index , embedding_dim , embedding_matrix)\nmodel.compile(loss = 'binary_crossentropy',\n             optimizer = 'adam',\n             metrics = ['acc'])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#-- Step 4) Train The Model\n\nnb_epoches = 200\n\nearly_stopping = EarlyStopping(monitor = 'val_loss',patience = 5)\nmodel_name = 'model_best.h5'\nmodel_checkpoint = ModelCheckpoint(model_name , save_best_only = True)\n\nhist = model.fit(train_x , train_y ,\\\n                validation_data = (val_x , val_y),\\\n                epochs = nb_epoches , batch_size = 2048,\n                shuffle = True , verbose = 2 ,\\\n                callbacks = [early_stopping , model_checkpoint])\n\n# Load Model : \nmodel.load_weights(model_name)\nbest_val_score = min(hist.history['val_loss'])\nprint('min val loss is' , best_val_score)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# -- Final Step: Submission\n\npreds = model.predict(test_x, batch_size=1024, verbose=1)\npreds = (preds > 0.35).astype(int)\n\nsub = pd.DataFrame({'qid':test_idx, 'prediction':preds.ravel()})\nsub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}