{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files areavailable in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport nltk\nnltk.download('stopwords')\nfrom nltk.tokenize import RegexpTokenizer\nfrom nltk.corpus import stopwords\nfrom nltk import word_tokenize\nfrom sklearn.model_selection import train_test_split\nfrom gensim import models\n\nimport re\nfrom collections import Counter\nimport gensim\nimport heapq\nfrom operator import itemgetter\nfrom multiprocessing import Pool","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = pd.read_csv('../input/quora-insincere-questions-classification/train.csv')\ntest_data = pd.read_csv('../input/quora-insincere-questions-classification/test.csv')\ndisplay(data.head())\ndata.target.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"stop_words = set(stopwords.words('english'))\ntokenizer = RegexpTokenizer(r'\\w+')\n\ndef removeStopWords(tokens): \n    return [word for word in tokens if word not in stop_words]\n\ndef lower_token(tokens): \n    return [w.lower() for w in tokens]   \n\n#https://towardsdatascience.com/nlp-learning-series-part-1-text-preprocessing-methods-for-deep-learning-20085601684b \n\ndef clean_numbers(sen):\n    res = []\n    for word in sen:\n        if bool(re.search(r'\\d', word)):\n            word = re.sub('[0-9]{5,}', '#####', word)\n            word = re.sub('[0-9]{4}', '####', word)\n            word = re.sub('[0-9]{3}', '###', word)\n            word = re.sub('[0-9]{2}', '##', word)\n            word = re.sub('[0-9]{1}', '#', word)\n        res.append(word)\n    return res\n\n\nword2vec_path = '../input/quora-insincere-questions-classification/embeddings/\\\nGoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\nword2vec = models.KeyedVectors.load_word2vec_format(word2vec_path, binary=True)\nwords = word2vec.index2word\n\nw_rank = {}\nfor i,word in enumerate(words):\n    w_rank[word] = i\n\nWORDS = w_rank\n\ndef words(text): return re.findall(r'\\w+', text.lower())\n\ndef P(word): \n    \"Probability of `word`.\"\n    # use inverse of rank as proxy\n    # returns 0 if the word isn't in the dictionary\n    return - WORDS.get(word, 0)\n\ndef correction(word): \n    \"Most probable spelling correction for word.\"\n    return max(candidates(word), key=P)\n\ndef correct(sen):\n    return [misspell_dict[word] if word in misspell_dict else word for word in sen]\n\ndef candidates(word): \n    \"Generate possible spelling corrections for word.\"\n    return (known([word]) or known(edits1(word)) or known(edits2(word)) or [word])\n\ndef known(words): \n    \"The subset of `words` that appear in the dictionary of WORDS.\"\n    return set(w for w in words if w in WORDS)\n\ndef edits1(word):\n    \"All edits that are one edit away from `word`.\"\n    letters    = 'abcdefghijklmnopqrstuvwxyz'\n    splits     = [(word[:i], word[i:])    for i in range(len(word) + 1)]\n    deletes    = [L + R[1:]               for L, R in splits if R]\n    transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R)>1]\n    replaces   = [L + c + R[1:]           for L, R in splits if R for c in letters]\n    inserts    = [L + c + R               for L, R in splits for c in letters]\n    return set(deletes + transposes + replaces + inserts)\n\ndef edits2(word): \n    \"All edits that are two edits away from `word`.\"\n    return (e2 for e1 in edits1(word) for e2 in edits1(e1))\n\ndef build_vocab(sentences):\n    vocab = {}\n    for sentence in sentences:\n        for word in sentence:\n            try:\n                vocab[word] += 1\n            except KeyError:\n                vocab[word] = 1\n    return vocab","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def process(data):\n    data, filtered_nums = process_no_misspells(data)\n    return process_misspells(data, filtered_nums)\n\ndef process_no_misspells(data):\n    tokens = [tokenizer.tokenize(sen) for sen in data['question_text']]\n    lower_tokens = [lower_token(token) for token in tokens]\n    filtered_nums = [clean_numbers(sen) for sen in lower_tokens]\n    data['basic'] = [' '.join(sen) for sen in filtered_nums]\n    return data, filtered_nums\n\ndef process_misspells(data, filtered_nums):\n    no_stopwords = [removeStopWords(sen) for sen in filtered_nums]\n    no_misspells = [correct(sen) for sen in no_stopwords]\n    data['no_misspels'] = [' '.join(sen) for sen in no_misspells]\n    return data, no_misspells \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import time\nfrom tqdm import tqdm_notebook\ndata, filtered_nums = process_no_misspells(data)\n\nstart_time = time.time()\nvocab = build_vocab(filtered_nums)\ntop_90k_words = dict(heapq.nlargest(90000, vocab.items(), key=itemgetter(1)))\ncorrected_words = map(correction,list(top_90k_words.keys()))\nprint(\"vocab and correction %s seconds\" % (time.time() - start_time),  flush=True)\n\nmisspell_dict = {}\nstart_time = time.time()\nfor _, (word,corrected_word) in tqdm_notebook(enumerate(zip(top_90k_words,corrected_words))):\n    if word!=corrected_word:\n        corrected_num, real_num = 0, 0\n        try:\n            corrected_num = vocab[corrected_word]\n        except:\n            pass\n        real_num = vocab[word]\n        if corrected_num > 2 * real_num:\n            misspell_dict[word] = corrected_word\nprint(\"misspell dict %s seconds\" % (time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data, no_misspells = process_misspells(data, filtered_nums)\ndisplay(data.head())\ntest_data, test_no_misspells = process(test_data)\ndisplay(test_data.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display(test_data.head(50))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.to_csv('processed_train.csv')\ntest_data.to_csv('processed_test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\ndef _get_mispell(misspell_dict):\n    misspell_re = re.compile('(%s)' % '|'.join(misspell_dict.keys()))\n    return misspell_dict, misspell_re\n\nmispellings, mispellings_re = _get_mispell(misspell_dict)\ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n    return mispellings_re.sub(replace, text)\n\n# Usage\nprint(replace_typical_misspell(\"Whta is demonitisation\"))\nprint(replace_typical_misspell(\"become\"))\ntry:\n    print(misspell_dict['whta'])\nexcept:\n    pass\n'''\nprint(correction('whta'))\nprint(correction('becoe'))\nprint(correction('become'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\nprint(correction('whta'))\nprint(correction('becoe'))\nprint(correction('become'))\nfiltered_misspellings = [correct(sen) for sen in filtered_nums]\nno_stopwords = [removeStopWords(sen) for sen in filtered_misspellings]\ndata['Text_Final_corrected'] = [' '.join(sen) for sen in no_stopwords]\ndata['tokens_corrected'] = no_stopwords\ndata.to_csv('tokens.csv')\n''''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.head(50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"''''\ndata_train, data_test = train_test_split(data, \n                                         test_size=0.10, \n                                         random_state=42)\n                                    '''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\nall_training_words = [word for tokens in data_train[\"tokens\"] for word in tokens]\ntraining_sentence_lengths = [len(tokens) for tokens in data_train[\"tokens\"]]\nTRAINING_VOCAB = sorted(list(set(all_training_words)))\nprint(\"%s words total, with a vocabulary size of %s\" % (len(all_training_words), len(TRAINING_VOCAB)))\nprint(\"Max sentence length is %s\" % max(training_sentence_lengths))\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\nall_test_words = [word for tokens in data_test[\"tokens\"] for word in tokens]\ntest_sentence_lengths = [len(tokens) for tokens in data_test[\"tokens\"]]\nTEST_VOCAB = sorted(list(set(all_test_words)))\nprint(\"%s words total, with a vocabulary size of %s\" % (len(all_test_words), len(TEST_VOCAB)))\nprint(\"Max sentence length is %s\" % max(test_sentence_lengths))\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_average_word2vec(tokens_list, vector, generate_missing=False, k=300):\n    if len(tokens_list)<1:\n        return np.zeros(k)\n    if generate_missing:\n        vectorized = [vector[word] if word in vector else np.random.rand(k) for word in tokens_list]\n    else:\n        vectorized = [vector[word] if word in vector else np.zeros(k) for word in tokens_list]\n    length = len(vectorized)\n    summed = np.sum(vectorized, axis=0)\n    averaged = np.divide(summed, length)\n    return averaged\n\ndef get_word2vec_embeddings(vectors, data, generate_missing=False):\n    embeddings = data['tokens'].apply(lambda x: get_average_word2vec(x, vectors, \n                                                                                generate_missing=generate_missing))\n    data['embeddings'] = embedings\n    return list(embeddings)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#training_embeddings = get_word2vec_embeddings(word2vec, data_train, generate_missing=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MAX_SEQUENCE_LENGTH = 50\nEMBEDDING_DIM = 300","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}