{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\nimport os\nimport re\nimport time\nfrom string import punctuation\n\nimport matplotlib.pyplot as plt\nfrom wordcloud import WordCloud\nfrom collections import namedtuple, defaultdict\n\nfrom sklearn.feature_extraction import stop_words\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_curve, roc_auc_score, auc\n\nfrom sklearn.linear_model import LogisticRegression\nimport gc\n\nimport logging\nlogging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)\n\nfrom pprint import pprint","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c7f712d0dd514e5ef3bf386354b7917a6c89626d"},"cell_type":"markdown","source":"## Loading functions"},{"metadata":{"trusted":true,"_uuid":"d92b5e78b01df76090d7cdf4ee4a9d41f9170518","_kg_hide-input":true},"cell_type":"code","source":"def get_stats(original_function):\n    def wraps(*args, **kwargs):\n        df = original_function(*args, **kwargs)\n        print(\"nrows : %d\" % df.shape[0])\n        print(\"ncolumns : %d\" % df.shape[1])\n        return df\n    return wraps\n        \ndef log_time(original_function):\n    def wraps(*args, **kwargs):\n        begin = time.time()\n        results = original_function(*args, **kwargs)\n        print(\"Elapsed time %fs\" % (begin - time.time()))\n        return results\n    return wraps\n\n@get_stats\ndef read_csv(path):\n    return pd.read_csv(path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"549f56d1718fa8915bbf9e207e446b779d1b9067","_kg_hide-input":true},"cell_type":"code","source":"def sort_vocab(func):\n    def wraps(*args, **kwargs):\n        vectorizer, X = func(*args, **kwargs)\n        tfs = np.asarray(X.sum(axis=0)).ravel()\n        multiplier = -1\n        mask_inds = (multiplier * tfs).argsort()\n        \n        terms = list(vectorizer.vocabulary_.keys())\n        indices = list(vectorizer.vocabulary_.values())\n        labels = list()\n        for i, index in enumerate(mask_inds):\n            labels.append(terms[indices.index(index)])\n            \n        return labels, list(tfs[mask_inds])\n    return wraps\n\ndef fit_transform(original_function):\n    def wraps(*args, **kwargs):\n        corpus = original_function(*args, **kwargs)\n        X_train = corpus.apply(lambda doc: \" \".join(doc))\n        vect = CountVectorizer(lowercase=False).fit(X_train)\n        X = vect.transform(X_train)\n        return vect, X\n    return wraps\n\ndef limit_features(func):\n    def wraps(*args, **kwargs):\n        matches = func(*args, **kwargs)\n        pos = np.where(matches.apply(len) > 0)[0]\n        return matches[pos]\n    return wraps\n\n@sort_vocab\n@fit_transform\n@limit_features\ndef search_pattern(raw_documents, use_vect=True, token_pattern=None):\n    if use_vect:\n        pat = re.compile(token_pattern)\n        return raw_documents.apply(lambda doc: pat.findall(doc))\n    \n    return raw_documents.apply(lambda doc: \" \".join(doc))\n\ndef plot_barh(response, title='', color='m', N=5, xlim=None, figsize=(9, 8)):\n    '''Plots a horizontal bar chart'''\n    labels, values = response\n    fig, ax = plt.subplots(figsize=figsize)\n    y_pos = np.arange(N)\n    rects = ax.barh(y_pos, values[:N], color=color, align='center')\n    ax.set_yticks(y_pos)\n    ax.set_yticklabels(labels[:N], fontdict={'size': 15, 'color': 'b'})\n    ax.set_title(title, fontdict={'size': 20})\n    if xlim is not None:\n        ax.set_xlim(xlim)\n        \n    for rect in rects:\n        w = rect.get_width()\n        xloc = w - 10\n        yloc = rect.get_y() + (rect.get_height()/2.0)\n        ax.text(xloc, yloc, \"%.5f\" % w if w < 1 else w, horizontalalignment='center', verticalalignment='center', color='white', weight='bold', fontdict={'size': 14}, clip_on=True)\n        \n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"86c4387bac07a29bad69b4f78e90747f8dd43939","_kg_hide-input":true},"cell_type":"code","source":"CloudParams = namedtuple('CloudParams', ['title', 'size', 'color'])\n\ndef prepare_cloud_data(corpus):\n    return \" \".join(corpus)\n\ndef prepare_cloud_data_from_tokens(corpus):\n    wordcloud = ' '\n    for item in np.hstack(corpus):\n        wordcloud = wordcloud + ' ' + item\n    return wordcloud\n\ndef plot_wordcloud(text, stops, params, figsize=(8, 6)):\n    stopwords = stop_words.ENGLISH_STOP_WORDS\n    if stops is not None:\n        stopwords = stopwords.union(stops)\n        \n    wordcloud = WordCloud(background_color ='white',\n                    stopwords = stopwords,\n                    random_state = 42).generate(text)\n\n    plt.figure(figsize=figsize)\n    plt.imshow(wordcloud)\n    plt.title(params.title, fontdict={\n        'size': params.size,\n        'color': params.color\n    })\n    plt.axis(\"off\")\n    plt.tight_layout()\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1ef516f3ffaa9032d5403e7e0318d0485f04e3cf","_kg_hide-input":true},"cell_type":"code","source":"def split_data(func):\n    def wrapper(*args, **kwargs):\n        X = func(*args, **kwargs)\n        X_train, X_test, y_train, y_test = train_test_split(X, args[1], test_size=0.2, random_state=42)\n        print(X_train.shape, y_train.shape)\n        return X_train, X_test, y_train, y_test\n    return wrapper\n\ndef fit_model(func):\n    def wrapper(*args, **kwargs):\n        clf = args[3]\n        X_train, X_test, y_train, y_test= func(*args, **kwargs)\n        \n        clf.fit(X_train, y_train)\n        return X_test, y_test, clf\n    return wrapper\n\ndef fit_vectorizer(func):\n    def wrapper(*args, **kwargs):\n        corpus = args[0]\n        vectorizer = args[2]\n        X = vectorizer.fit_transform(corpus)\n        print(\"Fitting vectorizer...\", X.shape)\n        display_stats(X, vectorizer, 10)\n        return X\n    return wrapper\n\ndef display_stats(X, vectorizer, N):\n    tfs = np.asarray(X.sum(axis=0)).ravel()\n    mask_inds = (-tfs).argsort()[:N]\n\n    vocab_values = list(vectorizer.vocabulary_.values())\n    terms = list(vectorizer.vocabulary_.keys())\n\n    labels = []\n    for i, j in enumerate(mask_inds):\n        labels.append(terms[vocab_values.index(j)])\n\n    plot_barh((labels, tfs[mask_inds]), title='Most occurred words', N=N, figsize=(8, 6))\n\ndef score_model(func):\n    def wrapper(*args, **kwargs):\n        X_test, y_true, clf = func(*args, **kwargs)\n        y_pred = clf.predict(X_test)\n        \n        TP = np.sum(y_pred[np.where(y_true.ravel() == 1)[0]])\n        TN = len(np.where(y_true.ravel() == 0)[0]) - np.sum(y_pred[np.where(y_true.ravel() == 0)[0]])\n        FP = np.sum(y_pred[np.where(y_true.ravel() == 0)[0]])\n        FN = len(np.where(y_true.ravel() == 1)[0]) - np.sum(y_pred[np.where(y_true.ravel() == 1)[0]])\n        \n        accuracy = (TP + TN) / len(y_true.ravel())\n        precision = TP / (TP + FP)\n        recall = TP / (TP + FN)\n        f1_score = 2 * ((precision * recall) / (precision + recall))\n        \n        response = (accuracy, precision, recall, f1_score)\n        plot_barh((['accuracy', 'precision', 'recall', 'F1'], response), N=4, figsize=(8, 4))\n        return response\n    return wrapper\n\n@score_model\n@fit_model\n@split_data\n@fit_vectorizer\ndef run_pipeline(raw_documents, target, vectorizer, model):\n    pass ","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"47e68f4a3486df138a5887bf7ec03f4de0e6b79a"},"cell_type":"code","source":"def create_vocab(docs):\n    vocab = dict()\n    for tokens in docs:\n        for token in tokens:\n            if token not in vocab:\n                vocab[token] = 1\n            else:\n                vocab[token] += 1\n    return vocab\n\ndef find_docs_matching_pattern(docs, search):\n    occurrences = docs[docs.apply(len) > 0]\n    occurrences = occurrences.apply(lambda x: np.sum([search in x]))\n    pos = occurrences.index.values\n    return train_df.loc[pos[np.where(occurrences.values > 0)[0]], 'question_text']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e772395a2675e97127c9dec07a5daa2b6d60ae08","_kg_hide-input":true},"cell_type":"code","source":"!ls \"../input\"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7f68f2f1cb3dca22409503dbeae8546989a8968e"},"cell_type":"markdown","source":"## Load the data"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"train_df = read_csv(\"../input/train.csv\")\nprint()\ntest_df = read_csv(\"../input/test.csv\")\nprint()\nsubmission_df = read_csv(\"../input/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d287ac3df823b7d90765361ab3ffe44212d85960"},"cell_type":"markdown","source":" # 1. Has data imbalance?"},{"metadata":{"trusted":true,"_uuid":"056c5f3bef82aa383ac6a2705c56ac60d2d30ccf","_kg_hide-input":true},"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(6, 4))\nlabels = ['Train', 'Test']\nsizes = [len(train_df), len(test_df)]\ncolors = ['#BB1AF7', \"#20DAFF\"]\n\npatches, texts, autotexts = ax.pie(sizes, explode=(0, 0.1), labels=labels, colors=colors, autopct='%1.1f%%')\nfor text in texts:\n    text.set_fontsize(20)\nax.axis('equal')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ffaf74a055d9389cc694e69ffced07f2997913d3"},"cell_type":"markdown","source":"# 2. Wordcloud"},{"metadata":{"trusted":true,"_uuid":"b4de65674a5012e3689a337280c1150bc3c2f8cf","_kg_hide-input":true},"cell_type":"code","source":"cloud_data = prepare_cloud_data(train_df.loc[train_df.target == 1, 'question_text'])\n\nplot_wordcloud(cloud_data, set(), CloudParams('Insincere', 30, 'b'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6ad24efcd86747fe6b55f9a16670dc0166b82885","_kg_hide-input":true},"cell_type":"code","source":"cloud_data = prepare_cloud_data(train_df.loc[train_df.target == 0, 'question_text'][:10000])\n\nplot_wordcloud(cloud_data, set(), CloudParams('Sincere', 30, 'b'))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a9cad355c42d8f0bdebc186437012836e74d1604"},"cell_type":"markdown","source":"# 3. Generate features"},{"metadata":{"_uuid":"5ffb01cdfe5aab791b3e893681667be5153bcd30"},"cell_type":"markdown","source":"> ### Punctuations"},{"metadata":{"trusted":true,"_uuid":"d0580a69cae5942a9c3dcd730c7e8906f3c63866","_kg_hide-input":true},"cell_type":"code","source":"puncts = train_df.question_text.apply(lambda doc: [item for item in doc.split() if item in punctuation])\npunct_vocabulary = create_vocab(puncts)\npunct_vocabulary = sorted(punct_vocabulary.items(), key=lambda x: x[1], reverse=True)\n\nplot_barh(([i[0] for i in punct_vocabulary], [i[1] for i in punct_vocabulary]), title='Puncts', N=10, figsize=(8, 6))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d50ba0ea98fe30c8a8291490285d67fcaf4d704","_kg_hide-input":true},"cell_type":"code","source":"ALL_PUNCTS = list(set([i[0] for i in punct_vocabulary]))\nnp.asarray(ALL_PUNCTS)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"78ee67620a4e1c9363c70f8683584805a2f27dab"},"cell_type":"markdown","source":"> ### UpperCaps"},{"metadata":{"trusted":true,"_uuid":"b6e8102ad99524eca46d4e703482d1982ebcf05e","_kg_hide-input":true},"cell_type":"code","source":"caps = train_df.question_text.apply(lambda row: [token for token in row.split() if re.match(r\"\\b[A-Z]{3,}\\b\", token) is not None])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fea280cf21b3e8ff2d7df60d33a42936f57c22e2","_kg_hide-input":true},"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(5, 4))\nindex = np.arange(2)\nrects = ax.bar(index, [np.sum(caps.apply(len) == 0), np.sum(caps.apply(len) == 1)], width=0.35, color='m')\nax.set_xticks(index)\nax.set_xticklabels(['NO CAPS', 'HAS CAPS'])\nax.set_title('# CAPS', fontdict={'size': 15})\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"37f6b0f73e446bd37b80e52445b438b486dbcd18"},"cell_type":"code","source":"caps.apply(len).value_counts().sort_index()[::-1][:5]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ce60c15293d6e045f99671c5d80f7732b8cb7b2c"},"cell_type":"markdown","source":"From the above example we can infer there are docs where the number of uppercase words are more than 20."},{"metadata":{"trusted":true,"_uuid":"e06f2cd3e23773704b893d0e2fc6c51d72585ebb"},"cell_type":"code","source":"for i in range(3):\n    print(train_df.loc[np.random.choice(np.where(caps.apply(len) >= 10)[0]), 'question_text'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d7b7ea0b5767838525754c701fe08711bc2407d7"},"cell_type":"markdown","source":"### Numbers"},{"metadata":{"trusted":true,"_uuid":"0dd8fa0956993964c32d5a71a3b057848934d965"},"cell_type":"code","source":"digits = train_df.question_text.apply(lambda row: [token for token in row.split() if token.isdigit()])\n\ndlens = train_df.question_text.apply(lambda row: np.sum([len(token) for token in row.split() if token.isdigit()]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c466f333bbc6674045b042afbfdb5473b078418d"},"cell_type":"code","source":"dlens.value_counts().sort_index()[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"88cdb2729ae685d684d8177fdb78448e24319b09"},"cell_type":"code","source":"digits.apply(len).value_counts().sort_index()[::-1][:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8230909c1e24a76c16425392cc387dffcfcd0492"},"cell_type":"code","source":"years = train_df.question_text.apply(lambda row: [token for token in row.split() if re.match(r\"\\b20\\d{2}$\\b\", token) is not None])\n\nyears = years[years.apply(len) > 0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"edb53a9ab94f5305166cac52962148ced7a01065"},"cell_type":"code","source":"row = []\ncol = []\ndata = []\n\nvocabulary = defaultdict(int)\nvocabulary.default_factory = vocabulary.__len__\n\nfor i, row in enumerate(years):\n    feature_counter = {}\n    for token in row:\n        feature_indx = vocabulary[token]\n        if feature_indx not in feature_counter:\n            feature_counter[feature_indx] = 1\n        else:\n            feature_counter[feature_indx] += 1\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"af6a32ef280169acab683a97d44792ad6d0f90c8"},"cell_type":"code","source":"vocabulary","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3ad073cf4f05a7c0f2e978625aaa6aa6517271b9"},"cell_type":"markdown","source":"## Modeling"},{"metadata":{"trusted":true,"_uuid":"90a395169de5b186533e07978d048c32898ece20","_kg_hide-input":true},"cell_type":"code","source":"def replace_year(doc):\n    doc = re.sub(r\"19[0-9][0-9]\", r\"19_yy\", doc)\n    return doc\n\ndef replace_number_abbrv(doc):\n    doc = re.sub(r\"\\b(\\d{1,2})000\\b\", r\"\\1k\", doc)\n    return doc\n\ndef clean_text(doc):\n    doc = replace_year(doc)\n#     doc = replace_number_abbrv(doc)\n    return doc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"99ffc4ed84951ade9aab2eea5d77c65d71206fc8"},"cell_type":"code","source":"filtered = train_df.copy()\nfiltered['cleaned_text'] = filtered.question_text.apply(lambda doc: clean_text(doc))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12b6d630fe38cadd770b77875a1884bc4ff89023","_kg_hide-input":true},"cell_type":"code","source":"stoplist = set('and'.split())\nvectorizer = CountVectorizer(max_features=10000, min_df=2, stop_words=stoplist)\nlr = LogisticRegression()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3c837c2af6d3b7871f33d2ec1c93c4a103670066"},"cell_type":"code","source":"stoplist","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"5d548eedf2f371e17160e00bfbd1b88d972c0aad"},"cell_type":"code","source":"indx = np.random.choice(filtered.index.values)\nprint(filtered.loc[indx, 'question_text'])\nprint()\nprint(vectorizer.build_tokenizer()(filtered.loc[indx, 'cleaned_text']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1262154e69fde52ac5f047810ae57ba491f00727","_kg_hide-input":true},"cell_type":"code","source":"response = run_pipeline(filtered.cleaned_text, filtered.target, vectorizer, lr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c1c2342743915f0a5fdfeaf88a2cf535a52f8c99"},"cell_type":"code","source":"response","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c94df5a9173d41c99c3a4059d1ee80f2cc3bd6b7"},"cell_type":"code","source":"#(0.954238683127572, 0.6929009294047854, 0.4419778002018164, 0.5396996534462841)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ad9ea37346b41a20ae8f43cbe1789d2c1e0aca9a"},"cell_type":"code","source":"print(vectorizer.get_feature_names()[:200])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3f70e77762e94a1d5d7eeca91386688a9688e92"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}