{"cells":[{"metadata":{"_uuid":"34760f73b636159435d6c35bf6b996d89b25adbc"},"cell_type":"markdown","source":"The final model is a logistic regression classifier using predictions from a logistic regression using countvectorizer bag of words and a bag of embeddings neural model."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.base import BaseEstimator, ClassifierMixin\nfrom sklearn.metrics import f1_score\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import train_test_split\nimport lightgbm as lgbm\nfrom nltk.tokenize import sent_tokenize, word_tokenize\nimport math\nimport string\n\nDATA_FOLDER = \"../input\"\nSEED = 9\n\ndef read_data(train_data_perc=0.8):\n    train_data_file = DATA_FOLDER + \"/\" + \"train.csv\"\n\n    all_data = pd.read_csv(train_data_file)\n    X = all_data[[\"qid\", \"question_text\"]]\n    y = all_data[[\"target\"]]\n    \n    X[\"num_words\"] = X[\"question_text\"].apply(lambda x: len(str(x).split()))\n    X[\"num_unique_words\"] = X[\"question_text\"].apply(lambda x: len(set(str(x).split())))\n    X[\"num_chars\"] = X[\"question_text\"].apply(lambda x: len(str(x)))\n    X[\"num_punctuations\"] = X[\"num_unique_words\"].apply(lambda x: len([c for c in str(x)\n                                                                       if c in string.punctuation]) )    \n\n    X_train, X_test, y_train, y_test = train_test_split(X, y,\n                                                        test_size=1 - train_data_perc,\n                                                        random_state=SEED)\n\n    return X_train, y_train, X_test, y_test\n\ndef read_test_data():\n    test_data_file = DATA_FOLDER + \"/\" + \"test.csv\"\n\n    all_data = pd.read_csv(test_data_file)\n    X = all_data[[\"qid\", \"question_text\"]]\n\n    X[\"num_words\"] = X[\"question_text\"].apply(lambda x: math.log(len(str(x).split())))\n    X[\"num_chars\"] = X[\"question_text\"].apply(lambda x: math.log(len(str(x))))\n\n    return X\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"48f814d37e7728fc233aa5df04288d8882538ed0"},"cell_type":"markdown","source":"Check word, character and punctuation counts for sincere and insincere questions\n"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"X_train, y_train, X_test, y_test = read_data(train_data_perc=0.6)\n\nprint(\"Number of training samples:\", len(X_train))\nprint(\"Number of test samples:\", len(X_test))\n\nnumber_of_ones = (y_train[\"target\"] == 1).sum()\nnumber_of_zeros = len(y_train) - number_of_ones\n\nprint(\"Number of zeros in train set (sincere questions)\", number_of_zeros)\nprint(\"Number of ones in train set (insincere questions)\", number_of_ones)\n\"\"\"\ninsincere_stats = {\"num_words\": 0, \n                  \"num_unique_words\": 0,\n                  \"num_chars\": 0,\n                  \"num_punctuations\": 0}\nsincere_stats = {\"num_words\": 0, \n                  \"num_unique_words\": 0,\n                  \"num_chars\": 0,\n                  \"num_punctuations\": 0}\nfor i, X_val in X_train.iterrows():\n    if y_train[\"target\"][i] == 0:\n        sincere_stats[\"num_words\"] += X_val[\"num_words\"]\n        sincere_stats[\"num_unique_words\"] += X_val[\"num_unique_words\"] \n        sincere_stats[\"num_chars\"] += X_val[\"num_chars\"]\n        sincere_stats[\"num_punctuations\"] += X_val[\"num_punctuations\"]   \n    else:\n        insincere_stats[\"num_words\"] += X_val[\"num_words\"]\n        insincere_stats[\"num_unique_words\"] += X_val[\"num_unique_words\"] \n        insincere_stats[\"num_chars\"] += X_val[\"num_chars\"]\n        insincere_stats[\"num_punctuations\"] += X_val[\"num_punctuations\"]           \n\n# calculate averages instead of sums\nfor key, value in sincere_stats.items():\n    sincere_stats[key] = sincere_stats[key] / number_of_zeros\n    \nfor key, value in insincere_stats.items():\n    insincere_stats[key] = insincere_stats[key] / number_of_ones\n    \nprint(\"insincere_stats\", insincere_stats)\nprint(\"sincere_stats\", sincere_stats)\n\"\"\"\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"27e1e4ccb1e6dcef99702e0310cf0e9209d6b2ed"},"cell_type":"markdown","source":"Do some cleanup on text. For instance remove punctuation and make words lower case."},{"metadata":{"trusted":true,"_uuid":"68116d0d37abdc6849af7a8bcea861d05c0625fd"},"cell_type":"code","source":"\"\"\"\nimport re, string, unicodedata\nimport nltk\nfrom nltk import word_tokenize, sent_tokenize\nfrom nltk.corpus import stopwords\nfrom nltk.stem import LancasterStemmer, WordNetLemmatizer\n\n\nprint(X_train[\"question_text\"].head())\n\nstemmer = LancasterStemmer()\nlemmatizer = WordNetLemmatizer()\n\ndef remove_non_ascii(word):\n    # Remove non-ASCII characters from list of tokenized words\n    new_word = unicodedata.normalize('NFKD', word).encode('ascii', 'ignore').decode('utf-8', 'ignore')\n    return new_word\n\ndef to_lowercase(word):\n    # Convert all characters to lowercase from list of tokenized words\n    return word.lower()\n\ndef remove_punctuation(word):\n    # Remove punctuation from list of tokenized words\n    new_word = re.sub(r'[^\\w\\s]', '', word)\n    return new_word\n\ndef remove_stopwords(word):\n    # Remove stop words from list of tokenized words\n    if word not in stopwords.words('english'):\n        return word\n    return ''\n\ndef stem_words(word):\n    # Stem words in list of tokenized words\n    return stemmer.stem(word)\n\ndef lemmatize_verbs(word):\n    # Lemmatize verbs in list of tokenized words\n    return lemmatizer.lemmatize(word, pos='v')\n\ndef normalize(word):\n    word = remove_non_ascii(word)\n    word = to_lowercase(word)\n    word = remove_punctuation(word)\n    # word = remove_stopwords(word)\n    #word = lemmatize_verbs(word)\n    return word\n\ndef get_processed_text(string):\n    words = nltk.word_tokenize(string)\n    new_words = []\n    for word in words:\n        new_word = normalize(word)\n        if new_word != '':\n            new_words.append(new_word)\n    return ' '.join(new_words)\n\nX_train[\"question_text\"] = X_train[\"question_text\"].apply(lambda x: get_processed_text(x))\nprint(X_train[\"question_text\"].head())\nX_test[\"question_text\"] = X_test[\"question_text\"].apply(lambda x: get_processed_text(x))\n\"\"\"\n\nvectorizer = CountVectorizer(ngram_range=(1, 1),\n                               min_df=1)\n\n\nX_final_test = read_test_data()\n\nprepared_text = vectorizer.fit_transform(X_train[\"question_text\"].values.tolist() +\n                                          X_test[\"question_text\"].values.tolist() +\n                                        X_final_test[\"question_text\"].values.tolist())\nunique_words = set(vectorizer.get_feature_names())\nprint(\"Number of unique words: \", len(unique_words))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7ee0869788276011e6b0e5dda65e30a00f4f0970"},"cell_type":"markdown","source":"Load GloVe vectors for bag of embeddings model."},{"metadata":{"trusted":true,"_uuid":"580bde1fc135a8965d46bbb3915c0ffe42778b9d"},"cell_type":"code","source":"def load_glove_with_vocabulary(vocabulary_map):\n    gloveFile = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n    f = open(gloveFile,'r')\n    emb_list = [None] * len(vocabulary_map)\n    print(\"Length of vocabulary: \", len(vocabulary_map))\n    found_words = 0\n    for i, line in enumerate(f):\n        splitLine = line.split(\" \")\n        word = splitLine[0]\n        if word in vocabulary_map:\n            found_words += 1\n            embedding = np.array([float(val) for val in splitLine[1:]], dtype=np.float32)\n            emb_list[vocabulary_map[word]] = embedding\n    print(\"Loaded GloVe vectors for %i words. Generating random vectors for the rest.\" % found_words)\n    full_emb_list = []\n    glove_mean = -0.00584\n    glove_std = 0.452\n    for emb in emb_list:\n        if emb is None:\n            full_emb_list.append(np.random.normal(glove_mean, glove_std, (1, 300)))\n        else:\n            full_emb_list.append(emb)\n    print(\"Done. Vectors loaded :\", len(full_emb_list))\n    embs = np.vstack(full_emb_list)\n    return embs\n\n\nimport gc\ngc.collect()\n#unique_words = load_unique_words(X_train, X_test)\nprint(\"len(unique_words)\", len(unique_words))\nembs = load_glove_with_vocabulary(vectorizer.vocabulary_)\nprint(\"embs.shape\", embs.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"74c6050294138cba29edaa812307d2335eaf41de"},"cell_type":"markdown","source":"Create tensor where each row contains the indices of the words in that sentence. Duplicates of the same word are ignored (for simplicity)."},{"metadata":{"trusted":true,"_uuid":"875d4e914bd4bce59287e873f869d032e728856b"},"cell_type":"code","source":"import torch\nimport torch.nn as nn\n\ntrain_prepared_data = vectorizer.transform(X_train[\"question_text\"])\ntest_prepared_data = vectorizer.transform(X_test[\"question_text\"])\nfinal_test_prepared_data = vectorizer.transform(X_final_test[\"question_text\"])\n\nprint(\"Vectorized train and test data\")\n\n\ntrain_targets = torch.from_numpy(np.array(y_train[\"target\"])).float()\nprint(\"Train targets\", train_targets)\ntest_targets = torch.from_numpy(np.array(y_test[\"target\"])).float()\nprint(\"Test targets\", test_targets)\n\ntrain_word_indices_list = []\ntrain_offsets_list = []\n\noffset = 0\nfor i in range(0, train_prepared_data.shape[0]):\n    train_offsets_list.append(offset)\n    row, col = train_prepared_data.getrow(i).nonzero()\n    train_word_indices_list.append(torch.tensor(torch.from_numpy(col), dtype=torch.int64))\n    offset += len(row)\n    \ntrain_words = torch.cat(train_word_indices_list)\ntrain_offsets = torch.tensor(train_offsets_list, dtype=torch.int64)\n\nprint(\"Created words and offsets for train data\")\n\ntest_word_indices_list = []\ntest_offsets_list = []\n\noffset = 0\nfor i in range(0, test_prepared_data.shape[0]):\n    test_offsets_list.append(offset)\n    row, col = test_prepared_data.getrow(i).nonzero()\n    test_word_indices_list.append(torch.tensor(torch.from_numpy(col), dtype=torch.int64))\n    offset += len(row)\n    \ntest_words = torch.cat(test_word_indices_list)\ntest_offsets = torch.tensor(test_offsets_list, dtype=torch.int64)\n\n\nprint(\"Created words and offsets for test data\")\n\nfinal_test_word_indices_list = []\nfinal_test_offsets_list = []\n\noffset = 0\nfor i in range(0, final_test_prepared_data.shape[0]):\n    final_test_offsets_list.append(offset)\n    row, col = final_test_prepared_data.getrow(i).nonzero()\n    final_test_word_indices_list.append(torch.tensor(torch.from_numpy(col), dtype=torch.int64))\n    offset += len(row)\n    \nfinal_test_words = torch.cat(final_test_word_indices_list)\nfinal_test_offsets = torch.tensor(final_test_offsets_list, dtype=torch.int64)\nprint(\"final_test_words.shape\", final_test_words.shape)\nprint(\"final_test_offsets.shape\", final_test_offsets.shape)\n\nprint(\"Created words and offsets for final test data\")\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"de5a6bf4955dae330c31f53149cf4ccf654cbea5"},"cell_type":"markdown","source":"Create bag of embeddings model in PyTorch. Just average all embeddings for each sentence and pass them through a neural net with a single hidden layer."},{"metadata":{"trusted":true,"_uuid":"c321308de5a0cbb9ab7d4ff1a1c77244a8e1565e"},"cell_type":"code","source":"class BagOfEmbeddings(nn.Module):\n    def __init__(self, embedding_weights, hidden_dim=100, dropout=0.5, embedding_mode='mean'):\n        super(BagOfEmbeddings, self).__init__()\n        self.hidden_dim = hidden_dim\n        self.dropout = nn.Dropout(p=dropout)\n        embedding_size = embedding_weights.shape[1]\n        self.embedding = nn.EmbeddingBag(embedding_weights.shape[0], embedding_size, mode=embedding_mode)\n        self.embedding.weight = nn.Parameter(torch.from_numpy(embedding_weights).float())\n        self.embedding.weight.requires_grad = False\n        self.final_layer = nn.Sequential(\n            nn.Dropout(p=dropout),\n            nn.Linear(embedding_size, hidden_dim),\n            nn.ReLU(),\n            nn.Dropout(p=dropout),\n            nn.Linear(hidden_dim, 1)\n        )\n    \n    def forward(self, words, offsets):\n        x = self.embedding(words, offsets)\n        return self.final_layer(x)\n\n    \nmodel = BagOfEmbeddings(embs, dropout=0.1, hidden_dim=75, embedding_mode='mean')\nprint(\"model\", model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"77f8be47226c4e1f3ff3ad6f6479b31f3eae265b"},"cell_type":"code","source":"Run bag of embeddings model training","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f617bc31d331973526b33d3e0f6d7e1a8b6b233f"},"cell_type":"code","source":"loss = nn.BCEWithLogitsLoss()\noptimizer = torch.optim.Adam(model.parameters(), lr = 0.001)\ntorch.manual_seed(SEED)\n\ndef get_batch(words, offsets, targets, start_index, size):\n    first_word_index = offsets[start_index]\n    offsets_end_index = start_index + size\n    if offsets_end_index > offsets.shape[0]:\n        offsets_end_index = offsets.shape[0]\n        last_word_index = words.shape[0]\n    else:\n        last_word_index = offsets[offsets_end_index]\n    if targets is not None:\n        return words[first_word_index:last_word_index], offsets[start_index:offsets_end_index] - offsets[start_index], targets[start_index:offsets_end_index]\n    else:\n        return words[first_word_index:last_word_index], offsets[start_index:offsets_end_index] - offsets[start_index], None\n    \ndef run_training(epochs, model, optimizer, loss_fn, \n                 all_words, all_offsets, \n                 all_targets, batch_size=32):\n    \n    print(\"Training samples: \", all_offsets.shape[0])\n    batch_losses = []\n    for e in range(epochs):\n        model.train()\n        start_index = 0\n        batch_nr = 0\n        print(\"Starting epoch %i\" % (e + 1))\n        while start_index < all_offsets.shape[0]:\n            batch_nr += 1\n            words, offsets, target = get_batch(all_words, all_offsets, all_targets, start_index, batch_size)            \n            start_index += batch_size\n            optimizer.zero_grad()\n            output = model(words, offsets)\n            loss = loss_fn(output.squeeze(), target)\n            loss.backward()\n            optimizer.step()            \n            if batch_nr % 1000 == 0:\n                batch_losses.append(loss.item())\n                print(\"Epoch %i, batch: %i, loss: %.5f\" % (e + 1, batch_nr, loss.item()))\n    return batch_losses\n            \n\nlosses = run_training(epochs=1, model=model, optimizer=optimizer, loss_fn=loss,\n                     all_words=train_words, all_offsets=train_offsets, all_targets=train_targets,\n                     batch_size=32)\nprint(\"Training avg model complete!\")\n\n                ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"acc37cd38a31222f5a09a50e32a4441d4ca14f7c"},"cell_type":"code","source":"def run_test(model, loss_fn, \n                 all_words, all_offsets, \n                 all_targets, batch_size=256):\n    \n    print(\"Test samples: \", all_offsets.shape[0])\n    batch_losses = []\n    outputs = []\n    model.eval()\n    start_index = 0\n    batch_nr = 0\n    print(\"Starting testing\")\n    while start_index < all_offsets.shape[0]:\n        batch_nr += 1\n        words, offsets, target = get_batch(all_words, all_offsets, all_targets, start_index, batch_size)            \n        start_index += batch_size\n        output = model(words, offsets)\n        outputs.append(torch.sigmoid(output))\n        if loss_fn:\n            loss = loss_fn(output.squeeze(), target)        \n            if batch_nr % 100 == 0:\n                batch_losses.append(loss.item())\n                print(\"Batch: %i, loss: %.5f\" % (batch_nr, loss.item()))\n    return batch_losses, torch.cat(outputs)\n\nprint(\"Evaluating test set\")\nbatch_losses, outputs = run_test(model=model, loss_fn=loss,\n                     all_words=test_words, all_offsets=test_offsets, all_targets=train_targets,\n                     batch_size=256)\n\nprint(\"outputs.shape\", outputs.shape)\n\ndef calculate_best_threshold(y_test, predictions):\n    best_threshold = -1\n    best_score = -1\n    for threshold in np.arange(0.01, 0.801, 0.01):\n        threshold = np.round(threshold, 2)\n        model_f1_score = f1_score(y_true=y_test[\"target\"],\n                                  y_pred=(predictions > threshold).astype(int))\n        if model_f1_score > best_score:\n            best_score = model_f1_score\n            best_threshold = threshold\n        print(\"F1 score at threshold %s: %s\" % (threshold, model_f1_score))\n    print(\"F1 score at best threshold %s: %s\" % (best_threshold, best_score))\n    return best_threshold, best_score\n\nboe_pred = outputs.detach().numpy()\nbest_threshold_boe, best_score_boe = calculate_best_threshold(y_test[:400000], boe_pred[:400000])\n\nprint(\"boe_pred\", boe_pred)\nprint(\"Evaluating final test outputs\")\n_, final_test_outputs = run_test(model=model, loss_fn=None,\n                     all_words=final_test_words, all_offsets=final_test_offsets, all_targets=None,\n                     batch_size=256)\n\nboe_final_pred = final_test_outputs.detach().numpy()\nprint(\"boe_final_pred\", boe_final_pred)\nprint(\"Final test outputs done\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"3b2db635a87885a492475725fb55147cc5cb9c2b"},"cell_type":"code","source":"logistic_regression_model = LogisticRegression(solver='lbfgs', max_iter=1000, n_jobs=-1, random_state=SEED)\nlogistic_regression_model.fit(train_prepared_data, y_train)\nlogreg_predictions = logistic_regression_model.predict_proba(test_prepared_data)[:, 1]\nbest_threshold_logreg, best_score_logreg = calculate_best_threshold(y_test[:400000], logreg_predictions[:400000])\n\nlogreg_final_predictions = logistic_regression_model.predict_proba(final_test_prepared_data)[:, 1]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"96d998b05981334f725b66cac0535ed8a31ba693"},"cell_type":"markdown","source":"Train a tree classifier on the predictions of the logistic regression and bag of embedding models. Also give number of words and number of characters as features."},{"metadata":{"trusted":true,"_uuid":"3c686c5d837dcf1820afc0a1d2144b1cfbea1cbd"},"cell_type":"code","source":"#final_classifier = lgbm.sklearn.LGBMClassifier(learning_rate=0.05,\n#                                           n_estimators=25,\n#                                           n_jobs=-1,\n #                                          random_state=SEED)\n\nfinal_classifier = LogisticRegression(solver='lbfgs', max_iter=1000, n_jobs=-1, random_state=SEED)\n\nX_final_augmented_train = pd.DataFrame({\"logreg_pred\": logreg_predictions[:400000], \"boe_pred\": boe_pred[:400000].squeeze(), \n                             \"num_words\": X_test[\"num_words\"].values[:400000], \"num_chars\": X_test[\"num_chars\"].values[:400000]})\n\ny_final_augmented_train = y_test[:400000]\n\nX_final_augmented_test = pd.DataFrame({\"logreg_pred\": logreg_predictions[400000:], \"boe_pred\": boe_pred[400000:].squeeze(), \n                             \"num_words\": X_test[\"num_words\"].values[400000:], \"num_chars\": X_test[\"num_chars\"].values[400000:]})\ny_final_augmented_test = y_test[400000:]\n\nfinal_classifier.fit(X_final_augmented_train, y_final_augmented_train)\n\nfinal_predictions = final_classifier.predict_proba(X_final_augmented_test)[:,1]\nbest_threshold_final, best_score_final = calculate_best_threshold(y_final_augmented_test, final_predictions)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ae01a14e32850b6534962d3eb2afa5dda9d3ccb2"},"cell_type":"markdown","source":"Output predictions on test data"},{"metadata":{"trusted":true,"_uuid":"7b7defb90c6d0ca09411c4911aeb7739e8553af5"},"cell_type":"code","source":"\n\nthreshold = best_threshold_final\nprint(X_final_test[\"qid\"].head())\nprint(X_final_test[\"question_text\"].head())\nX_final_classifier_test = pd.DataFrame({\"logreg_pred\": logreg_final_predictions, \"boe_pred\": boe_final_pred.squeeze(), \n                             \"num_words\": X_final_test[\"num_words\"].values, \"num_chars\": X_final_test[\"num_chars\"].values})\nprint(X_final_classifier_test[:10])\n\npredictions_proba = final_classifier.predict_proba(X_final_classifier_test)[:,1]\nprint(\"predictions_proba[:10]\", predictions_proba[:10])\nlabels = (predictions_proba > threshold).astype(int)\nprint(\"labels[:10]\", labels[:10])\npreds = pd.DataFrame({\n    \"qid\": X_final_test[\"qid\"],\n    \"prediction\": labels\n})\npreds.to_csv(\"submission.csv\", index=False)\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}