{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":false,"_kg_hide-output":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport zipfile #unzip embedddings\nimport re\nimport difflib\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"INPUT_PATH = \"../input\"\nOUTPUT_PATH = \"../output\"\nTRAIN_FILE = \"train.csv\"\nTEST_FILE = \"test.csv\"\nCORRECTED_TEST_FILE = \"corrected_test.csv\"\nCORRECTED_TRAIN_FILE = \"corrected_train.csv\"\nGLOVE_EMBEDDING = \"embeddings/glove.840B.300d/glove.840B.300d.txt\"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d6d9368934e4ac804976360f8e7b246182110b54"},"cell_type":"markdown","source":"# Toolbox\n(in-house)"},{"metadata":{"trusted":true,"_uuid":"0f288aa3ad0b0bf2305013da8643d0f87e541116"},"cell_type":"code","source":"#extract data from a csv file in a DataFrame\ndef load_data(csv_path = os.path.join(INPUT_PATH,TRAIN_FILE)):\n    return pd.read_csv(csv_path)\n\n#nested function returning True if len(X) is in [r_min, r_max] window, False otherwise\ndef len_in_range(r_min, r_max):\n    def in_range(X):\n        if len(X)>=r_min and len(X)<=r_max:\n            return True\n        else:\n            return False\n    return in_range\n\n#nested function returning True if the prefix of oov (out of vocabulary) word \n#and voc word are the same, False otherwise\ndef prefix_comparator(oov, prefix_length = 1):\n    def prefix_checker(voc):\n        if voc[:prefix_length].lower() == oov[:prefix_length].lower():\n            return True\n        else:\n            return False\n    return prefix_checker\n\n#compares oov_word to the vocabulary. If the similarity is over the ratio_treshold, the function will\n#return the corresponding token in the vocabulary, otherwise the unknown_token value\n#other parameters: \n#    hard_check: If hard_check is set to \"False\", function will replace ratio_treshold by \n#                min((len(oov_word)-1)/len(oov_word), ratio_threshold)\n\ndef oov_checker(oov_word, vocab_list, unknown_token = 'UNK', ratio_threshold = 0.9, hard_check = False):\n    best_ratio = 0.\n    best_voc = unknown_token    \n    if hard_check:\n        best_treshold = ratio_threshold\n    else:\n        best_treshold = min((len(oov_word)-1)/len(oov_word), ratio_threshold)\n        \n    for voc in vocab_list:\n        ratio = difflib.SequenceMatcher(None, oov_word, voc).ratio()\n        if ratio >= best_treshold and ratio > best_ratio:\n            best_ratio = ratio\n            best_voc = voc \n\n    return best_voc, best_ratio\n\n#Generates a dict of tupples in which the keys are words in oov_list and the tupple contains\n#the closest vocabulary word to the oov word and the associated score\n#Only the words in vocabulary which 1.length is in [len(oov_word)-len_window, len(oov_word)+len_window]\n#and 2. share the same prefix as oov_word are considered during the screening\ndef correction_score_generator(oov_list, vocabulary, unknown_token = 'UNK', len_window = 1):\n    correction_list = {}\n    if type(vocabulary) == list:\n        vocab_list = vocabulary\n    else:\n        vocab_list = [*vocabulary]\n        \n    sorted_vocab_list = sorted(vocab_list, key=len)\n    sorted_oov_list = sorted(oov_list, key=len)\n    length = 0 \n    for oov in sorted_oov_list:\n        if length != len(oov):\n            length = len(oov)\n            min_len = length - len_window\n            max_len = length + len_window\n            vocab_window = len_in_range(min_len, max_len)\n            filtered_vocab_list = list(filter(lambda X: vocab_window(X), sorted_vocab_list))\n        prefix_comp = prefix_comparator(oov)\n        filtered_vocab = list(filter(lambda X: prefix_comp(X), filtered_vocab_list))\n        correction_list[oov] = oov_checker(oov, filtered_vocab, unknown_token, ratio_threshold = 0.)\n    return correction_list\n\n#Takes sentences list and return a corrected version of it based on a correction_dict and a treshold:\n#1. scans each word of each sentence\n#2. checks if the word is present in the embedding dict\n#3. if not, checks if the word is present in the correction_dict and compares treshold with\n#   the score of the proposed correction, if it exists\n#4. if 2. and 3. are not positive, replace the unknown word by the \"UNK\" token\n#5. returns the corrected sentence\ndef sentence_correcter(sentences, embeddings, correction_dict, threshold = 0.9):\n    corrected_sentences = []\n    unknown = \"UNK\"\n    for sentence in sentences:\n        corrected_sentence = []\n        for word in sentence:\n            if word in embeddings: \n                corrected_sentence.append(word)\n            else:\n                try:\n                    if correction_dict[word][1] >= threshold:\n                        corrected_sentence.append(correction_dict[word][0])\n                    else:\n                        corrected_sentence.append(unknown)\n                except KeyError:\n                    corrected_sentence.append(unknown)\n        corrected_sentences.append(corrected_sentence)\n    \n    return corrected_sentences\n\ndef de_tokenize(sentences_list):\n    de_sentences_list = []\n    for sentence in sentences_list:\n        de_sentence = \"\"\n        for word in sentence:\n            de_sentence += word + \" \"\n        de_sentences_list.append(de_sentence)\n        \n    return de_sentences_list","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8a4d426c7116f3be09c9ca9bdb9e9f73ecb811f3"},"cell_type":"markdown","source":"(inspired by https://www.kaggle.com/alhalimi/tokenization-and-word-embedding-compatibility/notebook)"},{"metadata":{"trusted":true,"_uuid":"1b844dadfae904e1ca359a9634725b17261d53e5"},"cell_type":"code","source":"#Only extract GloVe embedddings as a first approach\ndef glove_embeddings(gloveFile = os.path.join(INPUT_PATH, GLOVE_EMBEDDING), extract = -1):\n\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    \n    embeddings_index = {}\n    f = open(gloveFile,'r', encoding=\"utf8\")\n    increment = 0\n    for line in f:\n        word, vect = get_coefs(*line.split(\" \"))\n        embeddings_index[word] = vect\n        if increment == extract - 1:\n            break\n        elif extract != -1:\n            increment += 1           \n    return embeddings_index\n\n#Returns a list of lists containing the tokenized version of the sentences contained in the sentences_list\ndef tokenize(sentences_list):\n    return [re.findall(r\"[\\w]+|[']|[.,!?;]\", x) for x in sentences_list]\n\n#Return a dict containing as keys all unique words from a tokenized sentences list, and as value the \n#number of times these words appears in the sentences corpus\ndef get_vocab(sentences):\n    \"\"\"\n    :param sentences: a list of list of words\n    :return: a dictionary of words and their frequency \n    \"\"\"\n    vocab={}\n    for sentence in sentences:\n        for word in sentence:\n            try:\n                vocab[word] +=1\n            except KeyError:\n                vocab[word] = 1\n    return vocab\n\n#Finds words in common between a given embedding and the vocabulary\ndef compare_vocab_and_embeddings(vocab, embeddings_index):\n    \"\"\"\n    :params vocab: our corpus vocabulary (a dictionary of word frquencies)\n            embeddings_index: a genim object containing loaded embeddings.\n    :returns in_common: words in common,\n             in_common_freq: total frequency in the corpus vocabulary of \n                             all words in common\n             oov: out of vocabulary words\n             oov_frequency: total frequency in vocab of oov words\n    \"\"\"\n    oov=[]\n    in_common=[]\n    in_common_freq = 0\n    oov_freq = 0\n\n    for word in vocab:\n        if word in embeddings_index:\n            in_common.append(word)\n            in_common_freq += vocab[word]\n        else: \n            oov.append(word)\n            oov_freq += vocab[word]\n    \n    print('Found embeddings for {:.2%} of vocab'.format(len(in_common) / len(vocab)))\n    print('Found embeddings for  {:.2%} of all text'.format(in_common_freq / (in_common_freq + oov_freq)))\n\n    return sorted(in_common)[::-1], sorted(oov)[::-1], in_common_freq, oov_freq\n\n#Returns the list of out-of-vocabulary words sorted by their frequency \n# stated in the vocab dict\ndef sort_oov_words(oov, vocab, threshold = 0.5, min_len = 5):\n    # Sort oov words by their frequency in the text\n    sorted_oov= sorted(oov, key =lambda x: vocab[x], reverse=True )\n    nr_tokens = 0\n    i = 0\n    ratio = 0.\n    pruned_sorted_oov = []\n    # Show oov words and their frequencies\n    if (len(sorted_oov)>0):\n        for word in sorted_oov:\n            if len(word) >= min_len:\n                if  re.search(r'[0-9]+', word, flags=0) == None:\n                    nr_tokens +=vocab[word]\n                    pruned_sorted_oov.append(word)\n        print(\"Total number of oov instances: {}\".format(nr_tokens))\n        for word in pruned_sorted_oov:\n            i += 1\n            #print(\"%s\\t%s\"%(word, vocab[word]))\n            ratio += vocab[word]\n            if ratio/nr_tokens >= threshold:\n                break       \n    else:\n        print(\"No words were out of vocabulary.\")\n    print(\"Number of oov words selected: {}/{} corresponding to {} instances\".format(i, len(pruned_sorted_oov),ratio))  \n    return pruned_sorted_oov[:i]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b7f40a0a071acfe0dedb6896abf13f1c3a29fdb0"},"cell_type":"markdown","source":"# Data preprocessing"},{"metadata":{"trusted":true,"_uuid":"57c4e7cf23c354bf5ac94fd7ccb979c49d49d6f3"},"cell_type":"code","source":"#extracts train data from train.csv\ntrain_data = load_data()\ntrain_data.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7402eee785e1d797c0646b1a178507a183a08baa"},"cell_type":"markdown","source":"Extraction of questions from the train_data"},{"metadata":{"trusted":true,"_uuid":"36528e79e601e2da32a728fa3950bafeb148943a"},"cell_type":"code","source":"questions_list = train_data[\"question_text\"].values","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c43f5f12b21b59a1637ab5aa4031b42587275caf"},"cell_type":"markdown","source":"Extraction of pre-trained GloVe embedding"},{"metadata":{"trusted":true,"_uuid":"85c6a3920bc75f7a7436819045d41408a4cc499f"},"cell_type":"code","source":"embeddings = glove_embeddings()\nprint(\"The GloVe embedding contains {} unique tokens\".format(len(embeddings.keys())))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d1319a52854909c743889cf66507fc3ad7164aa6"},"cell_type":"markdown","source":"Tokenisation of the questions and creation of a vocabulary list"},{"metadata":{"trusted":true,"_uuid":"2a7f22d486733b4b633d6557448ecc8d1cd649f8"},"cell_type":"code","source":"tokenized_questions = tokenize(questions_list)\ntoken_dict = get_vocab(tokenized_questions)\nprint(\"The training dataset contains {} unique tokens\".format(len(token_dict)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3a1ffbff77063e29a6f17bb404cfd509c2c76681"},"cell_type":"markdown","source":"Compares vocabulary and embeddings to return the complete oov words list, then sort and returns a subset of oov words wich represents the top 20% of all mispelled instances present in the question list"},{"metadata":{"trusted":true,"_uuid":"1ae8ea2a5596a2e33dd62cf9a2d81c12e5ada834"},"cell_type":"code","source":"in_common, oov, _, _ = compare_vocab_and_embeddings(token_dict, embeddings)\noov_words = sort_oov_words(oov, token_dict, threshold = 0.2)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3444fd1c6520c4cff55526fe9e8a3fe09b1c596e"},"cell_type":"markdown","source":"Generates a dict containing the oov_words selected at the previous step and their \"closest relative\" in the embedding list, together with their similarity score"},{"metadata":{"trusted":true,"_uuid":"22db5a47f1abe92e809ecf25c8e03006d14d93dc"},"cell_type":"code","source":"correction_scored_dict = correction_score_generator(oov_words, embeddings)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"751c5c6d0dfeebafb3d2cf10310c7cf127165935"},"cell_type":"markdown","source":"Returns a list of correted train questions based on the correction dict and the embeddings, de-tokenize it and save it in an external file for further use"},{"metadata":{"trusted":true,"_uuid":"6375c0c70803c52779e612315c18cacb45f129ba"},"cell_type":"code","source":"corrected_questions = sentence_correcter(tokenized_questions, embeddings, correction_scored_dict, threshold = 0.8)\nde_corrected_questions = de_tokenize(corrected_questions)\ncorrected_train_data = train_data.copy()\ncorrected_train_data['corrected_question_text'] = de_corrected_questions\ncorrected_train_data.to_csv(CORRECTED_TRAIN_FILE, index = False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8a7d8221a24cb87b0fa0c565d501e9b3757e0b68"},"cell_type":"markdown","source":"Repeat the same process with the test questions"},{"metadata":{"trusted":true,"_uuid":"54a2e3f4066f22677a8a47341d35039314f13bb8"},"cell_type":"code","source":"#extracts train data from train.csv\ntest_data = load_data(csv_path = os.path.join(INPUT_PATH,TEST_FILE))\ntest_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e4252cb310b216388fc757391427cb153e8e037"},"cell_type":"code","source":"test_questions_list = test_data[\"question_text\"].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cf1b02c6c242b4591bf03dc18c8438f289ac7edc"},"cell_type":"code","source":"tokenized_test_questions = tokenize(test_questions_list)\ntoken_test_dict = get_vocab(tokenized_test_questions)\nprint(\"The test dataset contains {} unique tokens\".format(len(token_test_dict)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c01c7052ee22a3f02df7737187130a1017a4575"},"cell_type":"code","source":"test_in_common, test_oov, _, _ = compare_vocab_and_embeddings(token_test_dict, embeddings)\ntest_oov_words = sort_oov_words(test_oov, token_test_dict, threshold = 0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"51657137cccd14c3c049e5c79ac2b404322c197f"},"cell_type":"code","source":"test_correction_scored_dict = correction_score_generator(test_oov_words, embeddings)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"23fb8ea291eb823199eb70b88f75b257f03c28dd"},"cell_type":"code","source":"corrected_test_questions = sentence_correcter(tokenized_test_questions, embeddings, test_correction_scored_dict, threshold = 0.8)\nde_corrected_test_questions = de_tokenize(corrected_test_questions)\ncorrected_test_data = test_data.copy()\ncorrected_test_data['corrected_question_text'] = de_corrected_test_questions\ncorrected_test_data.to_csv(CORRECTED_TEST_FILE, index = False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}