{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import re\nimport json\nimport string\nimport numpy as np\nimport pandas as pd\nfrom pandas.io.json import json_normalize\nimport matplotlib.pyplot as plt\nimport seaborn as sns\ncolor = sns.color_palette()\nimport gc\nimport pickle\nfrom tqdm import tqdm\ntqdm.pandas()\n\n%matplotlib inline\n\nfrom sklearn import model_selection, preprocessing, metrics, ensemble, naive_bayes, linear_model\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\nfrom sklearn.decomposition import TruncatedSVD\nimport lightgbm as lgb\n\nimport time\nfrom contextlib import contextmanager\n\npd.options.mode.chained_assignment = None\npd.options.display.max_columns = 999","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"82d2391af823459f2098155df25cf387d1653b87","trusted":true},"cell_type":"code","source":"@contextmanager\ndef timer(task_name=\"timer\"):\n    # a timer cm from https://www.kaggle.com/lopuhin/mercari-golf-0-3875-cv-in-75-loc-1900-s\n    print(\"----> {} started\".format(task_name))\n    t0 = time.time()\n    yield\n    print(\"----> {} done in {:.0f} seconds\".format(task_name, time.time() - t0))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f04a359d79728c00c84a2dd69722dab006cf2c80"},"cell_type":"markdown","source":"# Text Preprocess"},{"metadata":{"_uuid":"05ce5b7c297f022603857e9af184ff0f0bad0a97","trusted":true},"cell_type":"code","source":"with timer(\"reading_data\"):\n    train_df = pd.read_csv(\"../input/train.csv\")\n    test_df = pd.read_csv(\"../input/test.csv\")\n    df = pd.concat([train_df ,test_df]).reset_index(drop=True)\n\n    y_train = train_df[\"target\"].values\n\n    print('Total:', df.shape)\n    print('Train:', train_df.shape)\n    print('Test:', test_df.shape)\n    print(\"Number of texts: \", df.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f767a75f0f84e5e23adb16db90907d2307c711ea","trusted":true},"cell_type":"code","source":"print(train_df['target'].value_counts())\nsns.countplot(train_df['target'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b110492a29fa9cf33d8b094b3b4336954c9b6a0"},"cell_type":"code","source":"import psutil\nfrom multiprocessing import Pool\n\nnum_cores = psutil.cpu_count()  # number of cores on your machine\nnum_partitions = num_cores  # number of partitions to split dataframe\n\nprint('number of cores:', num_cores)\ndef df_parallelize_run(df, func):\n    df_split = np.array_split(df, num_partitions)\n    pool = Pool(num_cores)\n    df = pd.concat(pool.map(func, df_split))\n    pool.close()\n    pool.join()\n    return df\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f5a47f9dab3f58fec40686ed4e44128e3215db2b","trusted":true},"cell_type":"code","source":"puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\n\ndef clean_text(x):\n    x = str(x)\n    for punct in puncts:\n        x = x.replace(punct, f' {punct} ')\n    return x\n\ndef clean_numbers(x):\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    return x\n\nmispell_dict = {\"aren't\" : \"are not\",\n\"can't\" : \"cannot\",\n\"couldn't\" : \"could not\",\n\"didn't\" : \"did not\",\n\"doesn't\" : \"does not\",\n\"don't\" : \"do not\",\n\"hadn't\" : \"had not\",\n\"hasn't\" : \"has not\",\n\"haven't\" : \"have not\",\n\"he'd\" : \"he would\",\n\"he'll\" : \"he will\",\n\"he's\" : \"he is\",\n\"i'd\" : \"I would\",\n\"i'd\" : \"I had\",\n\"i'll\" : \"I will\",\n\"i'm\" : \"I am\",\n\"isn't\" : \"is not\",\n\"it's\" : \"it is\",\n\"it'll\":\"it will\",\n\"i've\" : \"I have\",\n\"let's\" : \"let us\",\n\"mightn't\" : \"might not\",\n\"mustn't\" : \"must not\",\n\"shan't\" : \"shall not\",\n\"she'd\" : \"she would\",\n\"she'll\" : \"she will\",\n\"she's\" : \"she is\",\n\"shouldn't\" : \"should not\",\n\"that's\" : \"that is\",\n\"there's\" : \"there is\",\n\"they'd\" : \"they would\",\n\"they'll\" : \"they will\",\n\"they're\" : \"they are\",\n\"they've\" : \"they have\",\n\"we'd\" : \"we would\",\n\"we're\" : \"we are\",\n\"weren't\" : \"were not\",\n\"we've\" : \"we have\",\n\"what'll\" : \"what will\",\n\"what're\" : \"what are\",\n\"what's\" : \"what is\",\n\"what've\" : \"what have\",\n\"where's\" : \"where is\",\n\"who'd\" : \"who would\",\n\"who'll\" : \"who will\",\n\"who're\" : \"who are\",\n\"who's\" : \"who is\",\n\"who've\" : \"who have\",\n\"won't\" : \"will not\",\n\"wouldn't\" : \"would not\",\n\"you'd\" : \"you would\",\n\"you'll\" : \"you will\",\n\"you're\" : \"you are\",\n\"you've\" : \"you have\",\n\"'re\": \" are\",\n\"wasn't\": \"was not\",\n\"we'll\":\" will\",\n\"didn't\": \"did not\",\n\"tryin'\":\"trying\"}\n\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\nmispellings, mispellings_re = _get_mispell(mispell_dict)\ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n    return mispellings_re.sub(replace, text)\n\ndef text_cleaning(text):\n    text = clean_text(text)\n    text = clean_numbers(text)\n    text = replace_typical_misspell(text)\n    return text\n\ndef text_clean_wrapper(df):\n    df[\"question_text\"] = df[\"question_text\"].progress_apply(text_cleaning)\n    return df","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"734530806bea916b6e72559c77981df6a71b646e","trusted":true},"cell_type":"code","source":"with timer(\"basic_feature_engineering\"):\n    from nltk.corpus import stopwords\n    STOPWORDS = list(set(stopwords.words('english')))\n\n    def count_regexp_occ(regexp=\"\", text=None):\n        \"\"\" Simple way to get the number of occurence of a regex\"\"\"\n        return len(re.findall(regexp, text))\n\n    # Count number of \\n\n    df[\"ant_slash_n\"] = df['question_text'].progress_apply(lambda x: count_regexp_occ(r\"\\n\", x))\n    # Get length in words and characters\n    df[\"raw_word_len\"] = df['question_text'].progress_apply(lambda x: len(x.split()))\n    df[\"raw_char_len\"] = df['question_text'].progress_apply(lambda x: len(x))\n    # Check number of upper case, if you're angry you may write in upper case\n    df[\"nb_upper\"] = df['question_text'].progress_apply(lambda x: count_regexp_occ(r\"[A-Z]\", x))\n    df[\"nb_title\"] = df[\"question_text\"].progress_apply(lambda x: len([w for w in str(x).split() if w.istitle()]))\n    # stopwords count\n    df[\"num_stopwords\"] = df[\"question_text\"].progress_apply(lambda x: len([w for w in str(x).lower().split() if w in STOPWORDS]))\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"58ef8e6f48085c21260c9367f147bfbf54c53c73","trusted":true},"cell_type":"code","source":"# text cleaning\nwith timer(\"text_cleaning\"):\n    df = df_parallelize_run(df, text_clean_wrapper)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4e58ad2f40ad061a2515fa6bf12f857bbc2c4328","trusted":true},"cell_type":"code","source":"train_df = df.loc[:train_df.shape[0] - 1, :]\ntest_df = df.loc[train_df.shape[0]:, :]\ndel test_df['target']; del df\ngc.collect()\n\ntrain_df.reset_index(drop=True, inplace=True)\ntest_df.reset_index(drop=True, inplace=True)\n\nprint('Train:', train_df.shape)\nprint('Test:', test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a609c4862cce796522c07801484e8476f4e02511","trusted":true},"cell_type":"code","source":"handcraft_feature_names = [f_ for f_ in test_df.columns if f_ not in [\"question_text\", \"qid\"]]\ntrain_handcraft_features = train_df[handcraft_feature_names]\ntest_handcraft_features = test_df[handcraft_feature_names]\nprint('handcraft feature count:', len(handcraft_feature_names))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0b1f41d68576d1ed9d8bded09ec54cb77c088458"},"cell_type":"markdown","source":"## Word&Char TFIDF Vectorizer"},{"metadata":{"_uuid":"59f417035bd06185fe94317df96d10dd58bcca58","trusted":true},"cell_type":"code","source":"all_text = pd.concat([train_df['question_text'], test_df['question_text']], axis =0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7c832571c53bf9206ee59eea1965358db8dc799a","trusted":false},"cell_type":"code","source":"with timer(\"word_vectorizer\"):\n    word_vectorizer = TfidfVectorizer(\n                    ngram_range=(1,4),\n                    token_pattern=r'\\w{1,}',\n                    min_df=3,\n                    max_df=0.9,\n                    strip_accents='unicode',\n                    use_idf=True,\n                    smooth_idf=True,\n                    sublinear_tf=True,\n                    max_features=100000\n                    )\n    \n    word_vectorizer.fit(all_text)\n    train_word_tfidf_features  = word_vectorizer.transform(train_df['question_text'])\n    test_word_tfidf_features  = word_vectorizer.transform(test_df['question_text'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"eadaa122a53f5ec1f8cbf71ca561417dfd6a32e3","trusted":false},"cell_type":"code","source":"def prepare_for_char_n_gram(text):\n    \"\"\"\n    The word hashing method described here aim to reduce the\n    dimensionality of the bag-of-words term vectors. It is based on\n    letter n-gram, and is a new method developed especially for our\n    task. Given a word (e.g. good), we first add word starting and\n    ending marks to the word (e.g. #good#). Then, we break the word\n    into letter n-grams (e.g. letter trigrams: #go, goo, ood, od#).\n    Finally, the word is represented using a vector of letter n-grams. \n    \"\"\"\n    text = re.sub(\" \", \"# #\", text)  # Replace space\n    text = \"#\" + text + \"#\"  # add leading and trailing #\n    return text\n\ntrain_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: prepare_for_char_n_gram(x))\ntest_df[\"question_text\"] = test_df[\"question_text\"].progress_apply(lambda x: prepare_for_char_n_gram(x))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9163dcf4096e09c13c1e0fff363461df3493f069","trusted":false},"cell_type":"code","source":"with timer(\"char_vectorizer\"):\n    def char_analyzer(text):\n        \"\"\"\n        This is used to split strings in small lots\n        Word Hashing: https://www.microsoft.com/en-us/research/wp-content/uploads/2016/02/cikm2013_DSSM_fullversion.pdf\n        \"\"\"\n        tokens = text.split()\n        return [token[i: i + 3] for token in tokens for i in range(len(token) - 2)]\n\n    char_vectorizer = TfidfVectorizer(\n                        ngram_range=(1,1),\n                        tokenizer=char_analyzer,\n                        min_df=3,\n                        max_df=0.9,\n                        strip_accents='unicode',\n                        use_idf=True,\n                        smooth_idf=True,\n                        sublinear_tf=True,\n                        max_features=50000\n                        )\n\n    char_vectorizer.fit(all_text)\n    train_char_tfidf_features = char_vectorizer.transform(train_df['question_text'])\n    test_char_tfidf_features = char_vectorizer.transform(test_df['question_text'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1963263ce50da0ecedc312bbc7853d745227de00","trusted":false},"cell_type":"code","source":"from scipy.sparse import hstack, csr_matrix\nfrom sklearn.base import BaseEstimator, TransformerMixin\n\nclass NBTransformer(BaseEstimator, TransformerMixin):\n    \"\"\"\n    https://nlp.stanford.edu/pubs/sidaw12_simple_sentiment.pdf\n    \"\"\"\n    def __init__(self, alpha=1):\n        self.r = None\n        self.alpha = alpha\n\n    def fit(self, X, y):\n        # store smoothed log count ratio\n        p = self.alpha + X[y==1].sum(0)\n        q = self.alpha + X[y==0].sum(0)\n        self.r = csr_matrix(np.log(\n            (p / (self.alpha + (y==1).sum())) /\n            (q / (self.alpha + (y==0).sum()))\n        ))\n        return self\n\n    def transform(self, X, y=None):\n        return X.multiply(self.r)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"34d1a29b43f2e0a9e18226e6ae5fb821b219c14b","trusted":false},"cell_type":"code","source":"with timer(\"transform_Naive_Bayes\"):\n    # transform to Naive Bayes feature\n    nb_transformer = NBTransformer(alpha=1).fit(train_word_tfidf_features, y_train)\n    train_word_tfidf_features = nb_transformer.transform(train_word_tfidf_features)\n    test_word_tfidf_features = nb_transformer.transform(test_word_tfidf_features)\n    \n    nb_transformer = NBTransformer(alpha=1).fit(train_char_tfidf_features, y_train)\n    train_char_tfidf_features = nb_transformer.transform(train_char_tfidf_features)\n    test_char_tfidf_features = nb_transformer.transform(test_char_tfidf_features)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"97e3d16cf0cda5d8078349afaa69a588b1c6d8d8"},"cell_type":"markdown","source":"## Features concate"},{"metadata":{"_uuid":"177b69e8eb2962ed29c4f70d791204803dcaa02b","trusted":false},"cell_type":"code","source":"with timer(\"feature_concate\"):\n    feature_names = word_vectorizer.get_feature_names() + char_vectorizer.get_feature_names() + handcraft_feature_names\n    del all_text; gc.collect()\n\n    X_train = hstack(\n            [\n                train_word_tfidf_features,\n                train_char_tfidf_features,\n                train_handcraft_features\n            ]\n        ).tocsr()\n\n    del train_word_tfidf_features; del train_char_tfidf_features; del train_handcraft_features\n    gc.collect()\n\n    X_test = hstack(\n        [\n            test_word_tfidf_features,\n            test_char_tfidf_features,\n            test_handcraft_features\n        ]\n    ).tocsr()\n    del test_word_tfidf_features; del test_char_tfidf_features; del test_handcraft_features\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d6fdf9f46f78c81804ca759ee2bf6d69ca6011a9","trusted":false},"cell_type":"code","source":"print('Train:', X_train.shape)\nprint('Test:', X_test.shape)\nprint('feature count:', len(feature_names))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f3166e1cfdc9909515f5c486601a75db95ad0295","trusted":false},"cell_type":"code","source":"import lightgbm as lgb\nfrom sklearn.model_selection import KFold, StratifiedKFold\n\nparam = {\n        \"objective\": \"binary\",\n        'metric': {'auc'},\n        \"boosting_type\": \"gbdt\",\n        \"num_threads\": -1,\n        \"bagging_fraction\": 0.8,\n        \"feature_fraction\": 0.8,\n        \"learning_rate\": 0.1,\n        \"num_leaves\": 31,\n        \"min_split_gain\": .1,\n        \"reg_alpha\": .1,\n        \n        \"scale_pos_weight\": 15,\n    }","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"f48f12c6b5af1319e382eb862c8d990d80b6fb97"},"cell_type":"code","source":"with timer(\"run-lightgbm-out-of-fold\"):\n    n_folds = 5\n    sk_folds = StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=2019)\n\n    oof = np.zeros(X_train.shape[0])\n    predictions = np.zeros(X_test.shape[0])\n    feature_importance_df = pd.DataFrame()\n\n    train_aucs = []\n    valid_aucs = []\n    for fold_, (trn_idx, val_idx) in enumerate(sk_folds.split(X_train, y_train)):\n        with timer(\"run fold {}/{}\".format(fold_ + 1, n_folds)):\n            trn_data = lgb.Dataset(X_train[trn_idx], label=y_train[trn_idx])\n            val_data = lgb.Dataset(X_train[val_idx], label=y_train[val_idx])\n            num_round = 10000\n            clf = lgb.train(param, trn_data, num_round, \n                            valid_sets = [trn_data, val_data], \n                            verbose_eval=100, early_stopping_rounds = 200)\n            oof[val_idx] = clf.predict(X_train[val_idx], num_iteration=clf.best_iteration)\n            predictions += clf.predict(X_test, num_iteration=clf.best_iteration) / sk_folds.n_splits\n\n            train_aucs.append(clf.best_score['training']['auc'])\n            valid_aucs.append(clf.best_score['valid_1']['auc'])\n\n            # 当前 fold 训练的 feature importance\n            fold_importance_df = pd.DataFrame()\n#             fold_importance_df[\"feature\"] = used_features\n            fold_importance_df[\"feature\"] = feature_names\n            fold_importance_df[\"importance\"] = clf.feature_importance()\n            fold_importance_df[\"fold\"] = fold_ + 1\n            feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n\n    print('cv_auc')\n    print('-'*50)\n    print('Mean train-auc: {:<8.5f}, valid-auc: {:<8.5f}'.format(np.mean(train_aucs), np.mean(valid_aucs)))\n    print('-'*50)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"57e403978a6b10c76f3725dad772586c33c6981d"},"cell_type":"markdown","source":"# Optimal threshold"},{"metadata":{"_uuid":"975cb8641c0ae0f8af88fa3e4e9a7d6075eb00c8","trusted":false},"cell_type":"code","source":"def threshold_searching(y_true, y_proba, verbose=True):\n    from sklearn.metrics import roc_curve, precision_recall_curve, f1_score\n    from sklearn.model_selection import RepeatedStratifiedKFold\n\n    def threshold_search(y_true, y_proba):\n        precision , recall, thresholds = precision_recall_curve(y_true, y_proba)\n        thresholds = np.append(thresholds, 1.001) \n        F = 2 / (1/precision + 1/recall)\n        best_score = np.max(F)\n        best_th = thresholds[np.argmax(F)]\n        return best_th \n\n\n    rkf = RepeatedStratifiedKFold(n_splits=5, n_repeats=2)\n\n    scores = []\n    ths = []\n    for train_index, test_index in rkf.split(y_true, y_true):\n        y_prob_train, y_prob_test = y_proba[train_index], y_proba[test_index]\n        y_true_train, y_true_test = y_true[train_index], y_true[test_index]\n\n        # determine best threshold on 'train' part \n        best_threshold = threshold_search(y_true_train, y_prob_train)\n\n        # use this threshold on 'test' part for score \n        sc = f1_score(y_true_test, (y_prob_test >= best_threshold).astype(int))\n        scores.append(sc)\n        ths.append(best_threshold)\n\n    best_th = np.mean(ths)\n    score = np.mean(scores)\n\n    if verbose: print(f'Best threshold: {np.round(best_th, 4)}, Score: {np.round(score,5)}')\n\n    return best_th, score","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4ee325bdc1cb429bac19d014c1ce3b58e979a615","trusted":false},"cell_type":"code","source":"with timer(\"search-threshold\"):\n    best_threshold, cv_f1_score = threshold_searching(y_train, oof)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f268549edb3cb58bbce6f4ee5d88937a06ce2591","trusted":false},"cell_type":"code","source":"pred_test_y = (predictions > best_threshold).astype(int)\nsub = test_df[['qid']]\nsub['prediction'] = pred_test_y\nsub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"a33260272e4b259a1d4ac3ab7600856320cee513"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.7"}},"nbformat":4,"nbformat_minor":1}