{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nfrom tqdm import tqdm\n\ntqdm.pandas()\n\n# Any results you write to the current directory are saved as output.","execution_count":1,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train= pd.read_csv(\"../input/train.csv\")\ntest= pd.read_csv(\"../input/test.csv\")\nprint('train shape:',train.shape)\nprint('test shape:',test.shape)","execution_count":2,"outputs":[{"output_type":"stream","text":"train shape: (1306122, 3)\ntest shape: (375806, 2)\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_vocab(sentences, verbose =  True):\n    vocab = {}\n    for sentence in tqdm(sentences, disable = (not verbose)):\n        for word in sentence:\n            try:\n                vocab[word] += 1\n            except KeyError:\n                vocab[word] = 1\n    return vocab","execution_count":3,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sentences = train[\"question_text\"].progress_apply(lambda x: x.split()).values\nvocab = build_vocab(sentences)\nprint({k: vocab[k] for k in list(vocab)[:5]})","execution_count":4,"outputs":[{"output_type":"stream","text":"100%|██████████| 1306122/1306122 [00:08<00:00, 147288.93it/s]\n100%|██████████| 1306122/1306122 [00:07<00:00, 168540.00it/s]\n","name":"stderr"},{"output_type":"stream","text":"{'How': 261930, 'did': 33489, 'Quebec': 97, 'nationalists': 91, 'see': 9003}\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.info()","execution_count":5,"outputs":[{"output_type":"stream","text":"<class 'pandas.core.frame.DataFrame'>\nRangeIndex: 1306122 entries, 0 to 1306121\nData columns (total 3 columns):\nqid              1306122 non-null object\nquestion_text    1306122 non-null object\ntarget           1306122 non-null int64\ndtypes: int64(1), object(2)\nmemory usage: 29.9+ MB\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"from gensim.models import KeyedVectors\n\nnews_path = '../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\nembeddings_index = KeyedVectors.load_word2vec_format(news_path, binary=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import operator \n\ndef check_coverage(vocab,embeddings_index):\n    a = {}\n    oov = {}\n    k = 0\n    i = 0\n    for word in tqdm(vocab):\n        try:\n            a[word] = embeddings_index[word]\n            k += vocab[word]\n        except:\n\n            oov[word] = vocab[word]\n            i += vocab[word]\n            pass\n\n    print('Found embeddings for {:.2%} of vocab'.format(len(a) / len(vocab)))\n    print('Found embeddings for  {:.2%} of all text'.format(k / (k + i)))\n    sorted_x = sorted(oov.items(), key=operator.itemgetter(1))[::-1]\n\n    return sorted_x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov = check_coverage(vocab,embeddings_index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'?' in embeddings_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'&' in embeddings_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_text(x):\n\n    x = str(x)\n    for punct in \"/-'\":\n        x = x.replace(punct, ' ')\n    for punct in '&':\n        x = x.replace(punct, f' {punct} ')\n    for punct in '?!.,\"#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~' + '“”’':\n        x = x.replace(punct, '')\n    return x\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train[\"question_text\"] = train[\"question_text\"].progress_apply(lambda x: clean_text(x))\nsentences = train[\"question_text\"].apply(lambda x: x.split())\nvocab = build_vocab(sentences)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov = check_coverage(vocab,embeddings_index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(10):\n    print(embeddings_index.index2entity[i])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import re\n\ndef clean_numbers(x):\n\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train[\"question_text\"] = train[\"question_text\"].progress_apply(lambda x: clean_numbers(x))\nsentences = train[\"question_text\"].progress_apply(lambda x: x.split())\nvocab = build_vocab(sentences)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov = check_coverage(vocab,embeddings_index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov[:20]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\n\nmispell_dict = {'colour':'color',\n                'centre':'center',\n                'didnt':'did not',\n                'doesnt':'does not',\n                'isnt':'is not',\n                'shouldnt':'should not',\n                'favourite':'favorite',\n                'travelling':'traveling',\n                'counselling':'counseling',\n                'theatre':'theater',\n                'cancelled':'canceled',\n                'labour':'labor',\n                'organisation':'organization',\n                'wwii':'world war 2',\n                'citicise':'criticize',\n                'instagram': 'social medium',\n                'whatsapp': 'social medium',\n                'snapchat': 'social medium'\n\n                }\nmispellings, mispellings_re = _get_mispell(mispell_dict)\n\ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n\n    return mispellings_re.sub(replace, text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train[\"question_text\"] = train[\"question_text\"].progress_apply(lambda x: replace_typical_misspell(x))\nsentences = train[\"question_text\"].progress_apply(lambda x: x.split())\nto_remove = ['a','to','of','and']\nsentences = [[word for word in sentence if not word in to_remove] for sentence in tqdm(sentences)]\nvocab = build_vocab(sentences)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov = check_coverage(vocab,embeddings_index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov[:20]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}