{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nimport re # Regular expression\nimport nltk # text processing\nfrom collections import namedtuple, defaultdict\nimport numbers\nfrom wordcloud import WordCloud\n\n# Plotting library\nimport matplotlib.pyplot as plt\n% matplotlib inline\n\nfrom sklearn.feature_extraction import stop_words\nfrom sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, TfidfTransformer # text processing\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.linear_model import LogisticRegression,SGDClassifier\nfrom sklearn.metrics import roc_curve, roc_auc_score, auc, accuracy_score, log_loss\n\nimport scipy.sparse as sp\nfrom nltk.stem import WordNetLemmatizer \n\nimport gc\n\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"20dee3cc0762770a9147c63a1528757416a172e6"},"cell_type":"markdown","source":"## Define functions "},{"metadata":{"trusted":true,"_uuid":"0f274699a3fdb6b015afbef89437df04cfb0a89e"},"cell_type":"code","source":"class BoW(object):\n    '''\n    Calculates frequencies of tokens across documents using Bag Of Words from scikit-learn\n    '''                 \n    def __init__(self):\n        self.vect = None\n        self.bow = None\n        \n    def sort_vocabulary(self, order=-1):\n        '''\n        Sorts the vocabulary in a given order\n        \n        Parameters\n        -------------\n        sort_order: {integer}, sorts the vocabulary.If order = -1, sort in descending\n        '''\n        tfs = np.array(self.bow.sum(axis=0)).ravel()\n        sort_indx = (order * tfs).argsort()\n        \n        labels = list()\n        values = list()\n        terms = list(self.vect.vocabulary_.keys())\n        indices = list(self.vect.vocabulary_.values())\n        for index in sort_indx:\n            labels.append(terms[indices.index(index)])\n            values.append(tfs[index])\n            \n        return labels, values\n        \n    def fit(self, corpus, token_pattern):\n        self.vect = CountVectorizer(token_pattern=token_pattern)\n        self.bow = self.vect.fit_transform(corpus)\n        \n    def clean(self):\n        self.vect = None\n        self.bow = None\n        \ndef search_pattern(token_pattern, corpus):\n    pat = re.compile(token_pattern)\n    matches = corpus.apply(lambda doc: re.findall(pat, doc))\n    return matches\n        \ndef create_vocab(tokens, ascending=None):\n    '''\n    Creates a vocabulary (key, value) pairs of words to frequencies\n    \n    Parameters\n    --------------\n    token_list : {array of array} - list of list of tokens\n    '''\n    \n    counter = defaultdict()\n    counter.default_factory = counter.__len__\n\n    for doc in tokens:\n        for token in doc:\n            counter[token] += 1\n            \n    counter = dict(counter)\n    if ascending is None:\n        return counter\n    \n    if ascending is False:\n        return sorted(counter.items(), key=lambda x: x[1], reverse=True)\n    return sorted(counter.items(), key=lambda x: x[1])\n\ndef plot_barchart(labels, values, chart_params, figsize=(10, 4), horizontal=False):\n    '''\n    Plots a barchart\n    \n    Parameters\n    -----------------\n    labels : {array}, labels on the x-axis\n    values : {array}, values on the y-axis\n    chart_params : {namedtuple}, chart configurations\n    '''\n    fig, ax = plt.subplots(figsize=figsize)\n    y_pos = np.arange(len(values))\n    if horizontal is False:\n        ax.bar(y_pos, values, chart_params.width, color=chart_params.colors)\n        ax.set_xticks(y_pos)\n        ax.set_xticklabels(labels)\n    else:\n        ax.barh(y_pos, values, chart_params.width, color=chart_params.colors)\n        ax.set_yticks(y_pos)\n        ax.set_yticklabels(labels)\n    \n    ax.set_title(chart_params.title, fontdict={'size': chart_params.title_size})\n    plt.show()\n\ndef gen_cloud_data(corpus):\n    '''\n    Generates a word cloud data\n    \n    Parameters\n    ------------\n    corpus : {array} - Array of array of tokens\n    '''\n    digit_cloud = ' '\n    for row in corpus:\n        digit_cloud = digit_cloud + '_' + re.sub(r'\\s+', '_', row) + ' '\n    return digit_cloud\n\ndef plot_wordcloud(cloud_data, stops, params, figsize=(9, 7)):\n    '''\n    Plots a word cloud from array of documents\n    '''\n    all_stops = stop_words.ENGLISH_STOP_WORDS.union(stops)\n    wordcloud = WordCloud(background_color ='white',\n                    stopwords = all_stops,\n                    max_words = 400,\n                    max_font_size = 120, \n                    random_state = 42).generate(cloud_data)\n\n    plt.figure(figsize=figsize)\n    plt.imshow(wordcloud)\n    plt.title(params.title, fontdict={\n        'size': params.title_size,\n        'color': 'green'\n    })\n    plt.axis(\"off\")\n    plt.show()\n    \ndef print_statements_from_index(indx_list, N=5):\n    for i in range(N):\n        indx = np.random.choice(indx_list)\n        print('Index -> {}'.format(indx))\n        print(train_df.loc[indx, 'question_text'])\n    \n# Declare namedtuples\n\nChartParams = namedtuple('ChartParams', ['title', 'title_size'])\n\nBarChartParams = namedtuple('BarChart', ['title', 'title_size', 'width', 'colors'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"185ff64de90924ecb2a698bb14ea51057112b7ad"},"cell_type":"markdown","source":"## Feature extraction functions"},{"metadata":{"trusted":true,"_uuid":"5d731e569ac4c3bc2d9c38d76c06bf0758c590f8"},"cell_type":"code","source":"def get_uppers(tokens):\n    return tokens.apply(lambda row: np.array(row)[np.where([token.isupper() for token in row])[0]])\n\ndef get_digits(tokens):\n    return tokens.apply(lambda row: np.array(row)[np.where([token.isdigit() for token in row])[0]])\n\ndef get_alphanumerics(tokens):\n    return tokens.apply(lambda row: [token for token in row if re.match(r\"([a-zA-Z]+\\d+|\\d+[a-zA-Z]+)\", token) is not None])\n\ndef get_currency(tokens):\n    return tokens.apply(lambda row: [token for token in row if re.match(r\"[$]\\d\\d+[a-z]+$\", token) is not None])\n\ndef get_hyperlinks(tokens):\n    return tokens.apply(lambda row: [token for token in row if re.match(r'\\bhttps?[://].*[a-zA-Z0-9-&_.]$\\b', token) is not None])\n\ndef get_masked_tokens(tokens, low=None, high=None):\n    if high is None and low is None:\n        return tokens\n    \n    masked = np.zeros(len(tokens), dtype=bool)\n    \n    if low is not None:\n        masked = tokens.apply(len) > low\n    if high is not None:\n        masked &= tokens.apply(len) <= high\n        \n        \n    return tokens[np.where(masked)[0]]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"103371f91308affa890a1be311f49e471339dc1f"},"cell_type":"markdown","source":"\n\n\n## Load data"},{"metadata":{"trusted":true,"_uuid":"e6c759c3d1fac02fc6f0c3ef80a055041ef219d4"},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")\n\nprint(\"Total records in train data = {0}\".format(train_df.shape[0]))\nprint(\"Total records in test data = {0}\".format(test_df.shape[0]))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cd9a6c602e363baee663251aea497748ab5f8c85"},"cell_type":"markdown","source":"## Extract features from text"},{"metadata":{"trusted":true,"_uuid":"2b9d51d9ad07d179f1f309c4bb96ccace59c76e3"},"cell_type":"code","source":"tokens = train_df.question_text.apply(lambda doc: doc.split())\nuppers = get_uppers(tokens)\ndigits = get_digits(tokens)\nalphanums = get_alphanumerics(tokens)\ncurrencies = get_currency(tokens)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3df973566c20c0d1c9cf0cdf9c09dc29ffda8a6c"},"cell_type":"markdown","source":"## 1.  Numbers"},{"metadata":{"trusted":true,"_uuid":"c23a603f7d40935054d49e50d80ac13f926cf79a"},"cell_type":"code","source":"N = 10\nnum_pat = r'\\b19\\d{2}|20\\d{2}\\b'\nbow = BoW()\nbow.fit(train_df.question_text, num_pat)\nlabels, values = bow.sort_vocabulary()\nplot_barchart(labels[:N], values[:N], BarChartParams('Top {} numbers'.format(N), 20, 0.5, 'm'))\n\nbow = None\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"67413464ece0e0d11aa35182de59879520ef07c2"},"cell_type":"code","source":"cloud_data = gen_cloud_data(np.hstack(get_masked_tokens(digits)))\nplot_wordcloud(cloud_data, set(), ChartParams('Most common numbers', 30))\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0a1695e8285073b3f04ef89584be54374d1fe154"},"cell_type":"markdown","source":"## 2. Alphanumerics"},{"metadata":{"trusted":true,"_uuid":"b8e6b17b9710e322aa8d07c275efc3b1b5499924"},"cell_type":"code","source":"N = 10\nalphaVocab = create_vocab(alphanums, ascending=True)[:N]\nplot_barchart([i[0] for i in alphaVocab], [i[1] for i in alphaVocab], BarChartParams('Top {} alphanumerics'.format(N), 20, 0.5, 'r'))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f1580e7af32cb07594816457f6e1d7d1af155ccc"},"cell_type":"markdown","source":"## 3. Currency "},{"metadata":{"trusted":true,"_uuid":"90704f19a980aec1da3d1892406a6e196f405ab8"},"cell_type":"code","source":"N = 10\nbar_data = create_vocab(get_masked_tokens(currencies, 0), ascending=True)[:10]\nplot_barchart([i[0] for i in bar_data], [i[1] for i in bar_data], BarChartParams('Top {} currencies'.format(N), 20, 0.5, '#d8db2e'))\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"244468db9cf94d539826334a70978487496db50e"},"cell_type":"markdown","source":"## 4. Bar plots for extracted features\n>* Token lengths\n>* Uppercase tokens\n>* Digits\n>* Alphanumerics"},{"metadata":{"trusted":true,"_uuid":"fe920a1284a0651bb646416f92c8773341ee5b82"},"cell_type":"code","source":"fig, ax = plt.subplots(nrows=2, ncols=2, figsize=(15, 7))\n\nax[0][0].hist(get_masked_tokens(tokens, 3, 10).apply(len) , color='#4f2951')\nax[0][1].hist(get_masked_tokens(uppers, 4, 10).apply(len), color='#13b230')\nax[1][0].hist(get_masked_tokens(digits, 1, 10).apply(len), color='#3c2ed3')\nax[1][1].hist(get_masked_tokens(alphanums, 1, 10).apply(len), color='#d8db2e')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d167eab3d5ab7b7b0ae3b687b0f14e70076f37e2"},"cell_type":"code","source":"hyperlinks_series = get_hyperlinks(tokens)\nhyperlinks = get_masked_tokens(hyperlinks_series, 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"36e18df323cfd066e8268000e92a0a4017050ce3"},"cell_type":"code","source":"hyperlinks_df = pd.DataFrame({\n    'links': hyperlinks\n})\nhyperlinks_df['domain'] = hyperlinks_df.links.apply(lambda links: [re.match(r\"(https?\\:\\/\\/).*\", link).group(1) for link in links])\nhyperlinks_df['url_first'] = hyperlinks_df.links.apply(lambda links: [re.match(r\"https?\\:\\/\\/([a-zA-Z0-9.-]+)\\/?.*\", link).group(1) for link in links if re.match(r\"https?\\:\\/\\/([\\w.])\\/?\", link) is not None])\nhyperlinks_df['query_params'] = hyperlinks_df.links.apply(lambda links: [re.match(r\"\\bhttps?\\:\\/\\/.*[?](.*)\\b\", link).group(1) for link in links if re.match(r\"\\bhttps?\\:\\/\\/.*[?](.*)\\b\", link) is not None])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7c5f3a96fc755af770bb08797713e9e3158de7a0"},"cell_type":"code","source":"N = 60\nbar_data = create_vocab(hyperlinks_df['url_first'], ascending=True)[30:N]\nplot_barchart([i[0] for i in bar_data], [i[1] for i in bar_data], BarChartParams(''.format(N), 20, 0.5, '#d8db2e'), figsize=(10, 8), horizontal=True)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3dfc317aea6a951eb25a19e9f2a71013be70e8df"},"cell_type":"markdown","source":">As our dataset contains sentences we have to convert them in a format that can be learned by machine learning models.One such format is the well known bow(bag of words) model where each document is a vector of frequency count of words in the document.\n\n>>There are different ways to calculate the frequency distribution of words in text.One such library is [scikit-learn](https://scikit-learn.org/stable/modules/feature_extraction.html).\n\n> <strong><font size=\"3\">[Bag-of-Words](https://scikit-learn.org/stable/modules/feature_extraction.html#the-bag-of-words-representation)</font></strong>"},{"metadata":{"trusted":true,"_uuid":"ffe2b873e6b9e4b609db78db3a1e0144647d0aa7"},"cell_type":"code","source":"def replace_urls(doc, url_pattern):\n    return re.sub(url_pattern, r'\\1', doc)\n\ndef replace_months(doc):\n    return re.sub(r'(jan|feb|march|april|may|june|july|august|september|october|november|december)', r'month', doc)\n\ndef preprocess(doc):\n    doc = replace_months(doc)\n    doc = re.sub(r'(youtu)[.](be)', r'\\1\\2', doc)\n    doc = re.sub(r'i[.](imgur)', r'\\1', doc)\n    doc = re.sub(r'\\b\\s{2}\\b', r'', doc)\n    return doc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c3d0b92bfa5b5a400580b1ea20a8f77fff51da76"},"cell_type":"code","source":"train_df['preprocessed_text'] = train_df.question_text.apply(lambda doc: preprocess(doc))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c0869a90017132fe2403b9035838c01cee638dd2"},"cell_type":"code","source":"# Custom stop words list\n\nX = train_df.preprocessed_text\ny = train_df.target\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.33, random_state=42)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4f5bef42f18de1e966f27e9250c9d8ee7594679c"},"cell_type":"code","source":"STOPS = set(['to'])\n\ndef filter_tokens(tokens):\n    \n#     tokens = [t for t in tokens if t not in STOPS]\n    original_tokens = list(tokens)\n    return tokens\n\nclass CustomVectorizer(TfidfVectorizer):\n    def build_tokenizer(self):\n        tokenize = super(CustomVectorizer, self).build_tokenizer()\n        return lambda doc: list(filter_tokens(tokenize(doc)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"37620e23acb8f983f2bef506f2c0c8cb6661d8ca"},"cell_type":"code","source":"MAX_FEATURES = 10000\ncount_vect = CustomVectorizer(min_df=5, max_features=MAX_FEATURES, lowercase=False)\ncount_vect.fit(X_train)\ndtm = count_vect.transform(X_train)\n\nclf = LogisticRegression()\nclf.fit(dtm, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e928827fdc9b9a8d19ff4ef1bacddff9e62ac925"},"cell_type":"code","source":"def sort_bow_vocab(vect, dtm, order=-1):\n    tfs = np.asarray(dtm.sum(axis=0)).ravel()\n    sorted_indices = (order * tfs).argsort()\n\n    terms = list(vect.vocabulary_.keys())\n    indices = list(vect.vocabulary_.values())\n    \n    labels = []\n    values = []\n    for i in sorted_indices:\n        values.append(tfs[i])\n        labels.append(terms[indices.index(i)])\n        \n    return labels, values\n\nlabels, values = sort_bow_vocab(count_vect, dtm, order=1)\n\nplot_barchart(labels[:10], values[:10], BarChartParams('Top {} words'.format(10), 20, 0.5, 'm'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1aea2bde4637c60a22820250f4acc2d665225a1e"},"cell_type":"code","source":"X_test_vectors = count_vect.transform(X_test)\npredictions = clf.predict(X_test_vectors)\n\nfalse_positive_rate, true_positive_rate, thresholds = roc_curve(y_test, predictions)\nroc_auc = auc(false_positive_rate, true_positive_rate)\nprint('AUC: ', roc_auc_score(y_test, predictions))\nprint('Accuracy: ', accuracy_score(y_test, predictions))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"26315f888d31c4d37afe82eb827a909bc9c1f1e5"},"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(8, 4))\nax.plot(false_positive_rate, true_positive_rate, color='green', lw=2, label='ROC curve (area = %0.5f)' % roc_auc)\nax.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')\nplt.xlim([0.0, 1.0])\nplt.ylim([0.0, 1.05])\nplt.xlabel('False Positive Rate')\nplt.ylabel('True Positive Rate')\nplt.title('Receiver operating characteristic')\nplt.legend(loc=\"lower right\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1bd6874463e0d4c21271c7e491f35d84f3b58bc0"},"cell_type":"code","source":"# AUC:  0.7077320472912008\n# Accuracy:  0.9541437656169885","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a194fb34b5ae3e9752a1699124b64c73cf074332"},"cell_type":"markdown","source":"># Keras"},{"metadata":{"trusted":true,"_uuid":"1abc93c45074a2725cdc757dd9de6ebbff367310"},"cell_type":"code","source":"!ls -l '../input/embeddings'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"16297e8684e9d278bbdee5c4b5b56eb89d32ea75"},"cell_type":"code","source":"# from gensim.models import KeyedVectors\n\n# EMBEDDINGS = '../input/embeddings/'\n# embeddings_index = KeyedVectors.load_word2vec_format(os.path.join(EMBEDDINGS, 'GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'), binary=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cadf72dacbe925557fa5910b5b165da526fcc348"},"cell_type":"code","source":"# test_predictions.loc[:, 'qid'] = test_df.loc[:, 'qid']\n# test_predictions.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}