{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":10737,"databundleVersionId":290346}],"dockerImageVersionId":15317,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## ELI5 & SHAP - LGBM/LR - Interpretable ML\n\n_By Nick Brooks, November 2018_\n\nText Processing and LGBM parameters taken from @Olivier, please give him credit where it is due! <br>\nhttps://www.kaggle.com/ogrellier/lgbm-with-words-and-chars-n-gram","metadata":{"_uuid":"e6bdba0cc6085149197908490e3377367977c519"}},{"cell_type":"code","source":"# from IPython.display import HTML\n\n# HTML('''<script>\n# code_show=true; \n# function code_toggle() {\n#  if (code_show){\n#  $('div.input').hide();\n#  } else {\n#  $('div.input').show();\n#  }\n#  code_show = !code_show\n# } \n# $( document ).ready(code_toggle);\n# </script>\n# The raw code for this IPython notebook is by default hidden for easier reading.\n# To toggle on/off the raw code, click <a href=\"javascript:code_toggle()\">here</a>.''')","metadata":{"trusted":true,"_uuid":"97976625742469a37eb8640c0ec482b0dd600c6d"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Load Packages and Data**","metadata":{"_uuid":"c3156fa9e14eb47eedef3c2b0208b36434d9b12c"}},{"cell_type":"code","source":"# HYPER PARAMS\nmax_boosting_rounds = 5500\n\nimport time\nnotebookstart= time.time()\n\nimport datetime\nimport pandas as pd\nimport numpy as np\nimport random\nimport time\n\n# Viz\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nimport os\nfrom wordcloud import WordCloud\nimport missingno as mn\nfrom yellowbrick.text import TSNEVisualizer\n\n# Hide Warnings\nWarning = True\nif Warning is False:\n    import warnings\n    warnings.filterwarnings(action='ignore')\n    warnings.filterwarnings(action='ignore', category=DeprecationWarning)\n    warnings.filterwarnings(action='ignore', category=FutureWarning)\n\n# Modeling..\nimport eli5\nimport lightgbm as lgb\nimport shap\nshap.initjs()\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import cross_validate\nfrom sklearn import metrics\nfrom sklearn.linear_model import LogisticRegressionCV\nfrom sklearn.pipeline import make_pipeline\nimport scikitplot as skplt\nfrom sklearn import preprocessing\n\n# Tf-Idf\nfrom sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.pipeline import FeatureUnion\nfrom scipy.sparse import hstack, csr_matrix\n\nnp.random.seed(2018)\n\nfrom contextlib import contextmanager\nimport re\nimport string\nimport gc\n\n@contextmanager\ndef timer(name):\n    \"\"\"\n    Taken from Konstantin Lopuhin https://www.kaggle.com/lopuhin\n    in script named : Mercari Golf: 0.3875 CV in 75 LOC, 1900 s\n    https://www.kaggle.com/lopuhin/mercari-golf-0-3875-cv-in-75-loc-1900-s\n    \"\"\"\n    t0 = time.time()\n    yield\n    print(f'[{name}] done in {time.time() - t0:.0f} s')\n    \n# Data Visualization\ndef cloud(text, title, size = (10,7)):\n    # Processing Text\n    wordcloud = WordCloud(width=800, height=400,\n                          collocations=False\n                         ).generate(\" \".join(text))\n    \n    # Output Visualization\n    fig = plt.figure(figsize=size, dpi=80, facecolor='k',edgecolor='k')\n    plt.imshow(wordcloud,interpolation='bilinear')\n    plt.axis('off')\n    plt.title(title, fontsize=25,color='w')\n    plt.tight_layout(pad=0)\n    plt.show()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\", index_col= 'qid')#.sample(50000)\ntest = pd.read_csv(\"../input/test.csv\", index_col= 'qid')#.sample(5000)\ntestdex = test.index\n\ntarget_names = [\"Sincere\",\"Insincere\"]\ny = train['target'].copy()","metadata":{"trusted":true,"_uuid":"ba30efe1514a5471649bc2d20f781b61007dd682"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Take a Glimpse**","metadata":{"_uuid":"7bb2c3be4cac203b9b87d83d489543404dbf250d"}},{"cell_type":"code","source":"print(train.shape)\ntrain.head()","metadata":{"trusted":true,"_uuid":"213db020da0e362719b616e1c257c386bf4f51bb"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Class Balance..\")\ntrain.target.value_counts(normalize=True)","metadata":{"trusted":true,"_uuid":"3d6ff889697559a84d1db858533f9e6ae45ca0e3"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i,name in [(0,\"Sincere\"),(1,\"Insincere\")]:\n     cloud(train.loc[train.target == i,\"question_text\"].str.title(), title=\"{} WordCloud\".format(name), size=[8,5])","metadata":{"trusted":true,"_uuid":"d85b4ff01b05e6d3e451581a547e045e4eeaaa42"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"***\n\n## Logistic Regression and Count Vectorizer\n\nLets start with a simple model. <br>\n\n**TF-IDF**","metadata":{"_uuid":"8646a00bbf7ee3664a49d6d530106cfa001cab67"}},{"cell_type":"code","source":"test['target'] = np.nan\nall_text = pd.concat([train['question_text'],test['question_text']], axis =0)\n\nword_vect = TfidfVectorizer(\n            sublinear_tf=True,\n            strip_accents='unicode',\n            analyzer='word',\n            token_pattern=r'\\w{1,}',\n            stop_words='english',\n            ngram_range=(1, 2),\n            max_features=20000)\n\nwith timer(\"Word Grams TFIDF\"):\n    word_vect.fit(all_text)\n    X  = word_vect.transform(train['question_text'])\n    testing  = word_vect.transform(test['question_text'])","metadata":{"trusted":true,"_uuid":"fb314cce75553e7b9a4f1862f17b66e92572d7aa"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Train Test Split\nX_train, X_valid, y_train, y_valid = train_test_split(\n        X, y, test_size=0.20, random_state=23, stratify=y)","metadata":{"trusted":true,"_uuid":"9c7bb9d755b0660879a5817ffec8b7ebf203561b"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**TSNE - Visual Clustering**","metadata":{"_uuid":"94b73e5a0b439a3ddcfdc76edc32c21124271bb8"}},{"cell_type":"code","source":"# Create the visualizer and draw the vectors\nplt.figure(figsize = [15,9])\ntsne = TSNEVisualizer()\nn = 20000\ntsne.fit(X[:n], train.target[:n].map({1: target_names[1],0:target_names[0]}))\ntsne.poof()","metadata":{"trusted":true,"_uuid":"fdfd3c6407bf9d096eb17ae95f1014fcae8a9d7e"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Model and Model Evaluation:**","metadata":{"_uuid":"3bf6b136d68f52d06a81e16ff28ee68368e0fbe4"}},{"cell_type":"code","source":"# Fit Model\nmodel = LogisticRegression(solver = 'sag')\nmodel.fit(X_train, y_train)\n\n# Predict..\nvalid_logistic_pred = model.predict(X_valid)\ntrain_logistic_pred = model.predict(X_train)\nvalid_logistic_pred_proba = model.predict_proba(X_valid)\nvalid_logistic_pred_proba = [x[1] for x in valid_logistic_pred_proba]","metadata":{"trusted":true,"_uuid":"5e734d09b7a9f89c3fc7c1a31c9257ccf2738f37"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Train Set Accuracy: {}\".format(metrics.accuracy_score(train_logistic_pred, y_train)))\nprint(\"Train Set ROC: {}\".format(metrics.roc_auc_score(train_logistic_pred, y_train)))\nprint(\"Train Set F1 Score: {}\\n\".format(metrics.f1_score(train_logistic_pred, y_train)))\n\nprint(\"Validation Set Accuracy: {}\".format(metrics.accuracy_score(valid_logistic_pred, y_valid)))\nprint(\"Validation Set ROC: {}\".format(metrics.roc_auc_score(valid_logistic_pred, y_valid)))\nprint(\"Validation Set F1 Score: {}\\n\".format(metrics.f1_score(valid_logistic_pred, y_valid)))\n\nprint(metrics.classification_report(valid_logistic_pred, y_valid))\n\n# Confusion Matrix\nskplt.metrics.plot_confusion_matrix(valid_logistic_pred, y_valid)\nplt.show()","metadata":{"trusted":true,"_uuid":"d36dac611a176798efb4a8aa609d7620b6dfffc3"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"eli5.show_weights(model, vec = word_vect, top=(50,50),\n                  target_names=target_names)","metadata":{"trusted":true,"_uuid":"cc3b49c97808e70faf8b0cf8e2066aef209222fc"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Visualize Logistic Regression Predictions that are MOST wrong with ELI5\n\n**NOTE:** For the sentence highlight, the greener the highlight, the more the model believes it contributes to the y= CLASS. Sometimes green may mean its a high contributor to INSINCERE..","metadata":{"_uuid":"3f7b1440fd5c3f67fbb99d8a6eed05ea666defe2"}},{"cell_type":"code","source":"def eli5_plotter(df, n = 5):\n    for iteration in range(n):\n        samp = random.randint(1,df.shape[0]-1)\n        print(\"Ground Truth: {} \\nPredicted: {}\".format(\n            pd.Series(df.target.iloc[samp]).map({0:'Sincere', 1: 'Insincere'})[0],\n            pd.Series(df.predicted.iloc[samp]).map({0:'Sincere', 1: 'Insincere'})[0]))\n        display(eli5.show_prediction(model, df.question_text.iloc[samp], vec=word_vect,\n                             target_names=target_names))\n        \n# Prepare Validation Set\nraw_valid = train.loc[train.index.isin(y_valid.index), :]\nraw_valid['predicted'] = valid_logistic_pred\nraw_valid['predicted_proba'] = valid_logistic_pred_proba\n\nraw_valid['wrong_degree'] = abs(raw_valid['predicted_proba'] - raw_valid['target'])","metadata":{"trusted":true,"_uuid":"9f64ba3e10b4ae7785b28b9a88f22fc9b78f92c2"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"temp = raw_valid.loc[raw_valid.sort_values(by='wrong_degree', ascending = False).index[:20], [\"question_text\", \"target\", \"predicted\"]]\neli5_plotter(temp, n=16)","metadata":{"trusted":true,"_uuid":"ffa22ac7372fc9c696813e705b0ba211dd7d0cf1"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Expand to see more Cases..:**","metadata":{"_uuid":"689e4a0d594c7a2c38c750b4e630faae1fff03dd"}},{"cell_type":"code","source":"eli5_plotter(temp, n=30)","metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"8334af718147b5989218ef3f80aa5c0567edaa53"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Has to do with Trump:**","metadata":{"_uuid":"83a79b0c3b854482d09ba04326eb9fe328be59b3"}},{"cell_type":"code","source":"temp = raw_valid.loc[raw_valid.question_text.str.contains(\"(?i)trump\"),\n                     [\"question_text\", \"target\", \"predicted\"]]\neli5_plotter(temp, n=10)","metadata":{"trusted":true,"_uuid":"bcdf9c8a507b1ce06c06a28698369e0ec50a5e7d"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# submit_df = pd.DataFrame({\"qid\": test.index, \"prediction\": model.predict(testing).astype(int)})\n# submit_df.to_csv(\"submission.csv\", index=False)\n# print(submit_df.head())\n# del submit_df","metadata":{"trusted":true,"_uuid":"08a80e010a4eb3b153c0e9785ca88145045c9805"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## TF-IDF - Word and Character Grams & Regular NLP\n\nUpvote this :) - https://www.kaggle.com/ogrellier/lgbm-with-words-and-chars-n-gram","metadata":{"_uuid":"a39ced413137badaef3491af309cf75798afc935"}},{"cell_type":"code","source":"###########################################################################################\n### Upvote this :) - https://www.kaggle.com/ogrellier/lgbm-with-words-and-chars-n-gram ####\n###########################################################################################\n\n# The better written the code, the easier the copy pasta\n\n# Contraction replacement patterns\ncont_patterns = [\n    (b'(W|w)on\\'t', b'will not'),\n    (b'(C|c)an\\'t', b'can not'),\n    (b'(I|i)\\'m', b'i am'),\n    (b'(A|a)in\\'t', b'is not'),\n    (b'(\\w+)\\'ll', b'\\g<1> will'),\n    (b'(\\w+)n\\'t', b'\\g<1> not'),\n    (b'(\\w+)\\'ve', b'\\g<1> have'),\n    (b'(\\w+)\\'s', b'\\g<1> is'),\n    (b'(\\w+)\\'re', b'\\g<1> are'),\n    (b'(\\w+)\\'d', b'\\g<1> would'),\n]\npatterns = [(re.compile(regex), repl) for (regex, repl) in cont_patterns]\n\ndef prepare_for_char_n_gram(text):\n    \"\"\" Simple text clean up process\"\"\"\n    # 1. Go to lower case (only good for english)\n    # Go to bytes_strings as I had issues removing all \\n in r\"\"\n    clean = bytes(text.lower(), encoding=\"utf-8\")\n    # 2. Drop \\n and  \\t\n    clean = clean.replace(b\"\\n\", b\" \")\n    clean = clean.replace(b\"\\t\", b\" \")\n    clean = clean.replace(b\"\\b\", b\" \")\n    clean = clean.replace(b\"\\r\", b\" \")\n    # 3. Replace english contractions\n    for (pattern, repl) in patterns:\n        clean = re.sub(pattern, repl, clean)\n    # 4. Drop puntuation\n    # I could have used regex package with regex.sub(b\"\\p{P}\", \" \")\n    exclude = re.compile(b'[%s]' % re.escape(bytes(string.punctuation, encoding='utf-8')))\n    clean = b\" \".join([exclude.sub(b'', token) for token in clean.split()])\n    # 5. Drop numbers - as a scientist I don't think numbers are toxic ;-)\n    clean = re.sub(b\"\\d+\", b\" \", clean)\n    # 6. Remove extra spaces - At the end of previous operations we multiplied space accurences\n    clean = re.sub(b'\\s+', b' ', clean)\n    # Remove ending space if any\n    clean = re.sub(b'\\s+$', b'', clean)\n    # 7. Now replace words by words surrounded by # signs\n    # e.g. my name is bond would become #my# #name# #is# #bond#\n    # clean = re.sub(b\"([a-z]+)\", b\"#\\g<1>#\", clean)\n    clean = re.sub(b\" \", b\"# #\", clean)  # Replace space\n    clean = b\"#\" + clean + b\"#\"  # add leading and trailing #\n\n    return str(clean, 'utf-8')\n\ndef count_regexp_occ(regexp=\"\", text=None):\n    \"\"\" Simple way to get the number of occurence of a regex\"\"\"\n    return len(re.findall(regexp, text))\n\ndef get_indicators_and_clean_comments(df, text_var):\n    \"\"\"\n    Check all sorts of content as it may help find toxic comment\n    Though I'm not sure all of them improve scores\n    \"\"\"\n    # Count number of \\n\n    df[\"ant_slash_n\"] = df[text_var].apply(lambda x: count_regexp_occ(r\"\\n\", x))\n    # Get length in words and characters\n    df[\"raw_word_len\"] = df[text_var].apply(lambda x: len(x.split()))\n    df[\"raw_char_len\"] = df[text_var].apply(lambda x: len(x))\n    # Check number of upper case, if you're angry you may write in upper case\n    df[\"nb_upper\"] = df[text_var].apply(lambda x: count_regexp_occ(r\"[A-Z]\", x))\n    # Number of F words - f..k contains folk, fork,\n    df[\"nb_fk\"] = df[text_var].apply(lambda x: count_regexp_occ(r\"[Ff]\\S{2}[Kk]\", x))\n    # Number of S word\n    df[\"nb_sk\"] = df[text_var].apply(lambda x: count_regexp_occ(r\"[Ss]\\S{2}[Kk]\", x))\n    # Number of D words\n    df[\"nb_dk\"] = df[text_var].apply(lambda x: count_regexp_occ(r\"[dD]ick\", x))\n    # Number of occurence of You, insulting someone usually needs someone called : you\n    df[\"nb_you\"] = df[text_var].apply(lambda x: count_regexp_occ(r\"\\W[Yy]ou\\W\", x))\n    # Just to check you really refered to my mother ;-)\n    df[\"nb_mother\"] = df[text_var].apply(lambda x: count_regexp_occ(r\"\\Wmother\\W\", x))\n    # Just checking for toxic 19th century vocabulary\n    df[\"nb_ng\"] = df[text_var].apply(lambda x: count_regexp_occ(r\"\\Wnigger\\W\", x))\n    # Some Sentences start with a <:> so it may help\n    df[\"start_with_columns\"] = df[text_var].apply(lambda x: count_regexp_occ(r\"^\\:+\", x))\n    # Check for time stamp\n    df[\"has_timestamp\"] = df[text_var].apply(lambda x: count_regexp_occ(r\"\\d{2}|:\\d{2}\", x))\n    # Check for dates 18:44, 8 December 2010\n    df[\"has_date_long\"] = df[text_var].apply(lambda x: count_regexp_occ(r\"\\D\\d{2}:\\d{2}, \\d{1,2} \\w+ \\d{4}\", x))\n    # Check for date short 8 December 2010\n    df[\"has_date_short\"] = df[text_var].apply(lambda x: count_regexp_occ(r\"\\D\\d{1,2} \\w+ \\d{4}\", x))\n    # Check for http links\n    df[\"has_http\"] = df[text_var].apply(lambda x: count_regexp_occ(r\"http[s]{0,1}://\\S+\", x))\n    # check for mail\n    df[\"has_mail\"] = df[text_var].apply(\n        lambda x: count_regexp_occ(r'[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\\.[a-zA-Z0-9-.]+', x)\n    )\n    # Looking for words surrounded by == word == or \"\"\"\" word \"\"\"\"\n    df[\"has_emphasize_equal\"] = df[text_var].apply(lambda x: count_regexp_occ(r\"\\={2}.+\\={2}\", x))\n    df[\"has_emphasize_quotes\"] = df[text_var].apply(lambda x: count_regexp_occ(r\"\\\"{4}\\S+\\\"{4}\", x))\n\n    # Now clean comments\n    df[\"clean_comment\"] = df[text_var].apply(lambda x: prepare_for_char_n_gram(x))\n\n    # Get the new length in words and characters\n    df[\"clean_word_len\"] = df[\"clean_comment\"].apply(lambda x: len(x.split()))\n    df[\"clean_char_len\"] = df[\"clean_comment\"].apply(lambda x: len(x))\n    # Number of different characters used in a comment\n    # Using the f word only will reduce the number of letters required in the comment\n    df[\"clean_chars\"] = df[\"clean_comment\"].apply(lambda x: len(set(x)))\n    df[\"clean_chars_ratio\"] = df[\"clean_comment\"].apply(lambda x: len(set(x))) / df[\"clean_comment\"].apply(\n        lambda x: 1 + min(99, len(x)))\n    \ndef char_analyzer(text):\n    \"\"\"\n    This is used to split strings in small lots\n    I saw this in an article (I can't find the link anymore)\n    so <talk> and <talking> would have <Tal> <alk> in common\n    \"\"\"\n    tokens = text.split()\n    return [token[i: i + 3] for token in tokens for i in range(len(token) - 2)]\n\nall_text = pd.concat([train['question_text'],test['question_text']], axis =0)\n\nword_vect = TfidfVectorizer(\n            sublinear_tf=True,\n            strip_accents='unicode',\n            analyzer='word',\n            token_pattern=r'\\w{1,}',\n            stop_words='english',\n            ngram_range=(1, 2),\n            max_features=20000)\n\nchar_vectorizer = TfidfVectorizer(\n            sublinear_tf=True,\n            strip_accents='unicode',\n            tokenizer=char_analyzer,\n            analyzer='word',\n            ngram_range=(1, 1),\n            max_features=50000)\n\nwith timer(\"Word Grams TFIDF\"):\n    word_vect.fit(all_text)\n    train_word_features  = word_vect.transform(train['question_text'])\n    test_word_features  = word_vect.transform(test['question_text'])\n\nwith timer(\"Character Grams TFIDF\"):\n    char_vectorizer.fit(all_text)\n    train_char_features = char_vectorizer.transform(train['question_text'])\n    test_char_features = char_vectorizer.transform(test['question_text'])\n\nwith timer(\"Performing basic NLP\"):\n    get_indicators_and_clean_comments(train, 'question_text')\n    get_indicators_and_clean_comments(test,  'question_text')\n    \n    num_features = [f_ for f_ in train.columns\n                if f_ not in [\"question_text\", \"clean_comment\", \"remaining_chars\",\n                              'has_ip_address', 'target']]\n    \n# Get Sparse Matrix Feature Names..\nfeature_names = word_vect.get_feature_names() + char_vectorizer.get_feature_names() + num_features\ndel all_text; gc.collect()\n\nwith timer(\"Sparse Combine\"):\n    X = hstack(\n        [\n            train_char_features,\n            train_word_features,\n            train[num_features]\n        ]\n    ).tocsr()\n\n    del train_char_features\n    gc.collect()\n\n    testing = hstack(\n        [\n            test_char_features,\n            test_word_features,\n            test[num_features]\n        ]\n    ).tocsr()\n    del test_char_features; gc.collect()","metadata":{"trusted":true,"_uuid":"dd2adbed09d4b593ca06c8ec56eb353c8adb1d0f"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create the visualizer and draw the vectors\nplt.figure(figsize = [15,9])\ntsne = TSNEVisualizer()\nn = 20000\ntsne.fit(X[:n], train.target[:n].map({1: target_names[1],0:target_names[0]}))\ntsne.poof()","metadata":{"trusted":true,"_uuid":"2755d1c2a781a6b1b63dc6fa6502d9e0cf2c1cbb"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Light Gradient Boosting Binary Classifier","metadata":{"_uuid":"e1e115d68035a0dac48565140ea2fa4e33646459"}},{"cell_type":"code","source":"X_train, X_valid, y_train, y_valid = train_test_split(\n    X, y, test_size=0.20, random_state=23, stratify=y)\n\nprint(\"Light Gradient Boosting Classifier: \")\nlgbm_params = {\n        \"objective\": \"binary\",\n        'metric': {'auc'},\n        \"boosting_type\": \"gbdt\",\n        \"num_threads\": 4,\n        \"bagging_fraction\": 0.8,\n        \"feature_fraction\": 0.8,\n        \"learning_rate\": 0.1,\n        \"num_leaves\": 31,\n        \"min_split_gain\": .1,\n        \"reg_alpha\": .1\n    }\n\nmodelstart= time.time()\n# LGBM Dataset Formatting \nlgtrain = lgb.Dataset(X_train, y_train,\n                feature_name=feature_names)\nlgvalid = lgb.Dataset(X_valid, y_valid,\n                feature_name=feature_names)\n\n# Go Go Go\nlgb_clf = lgb.train(\n    lgbm_params,\n    lgtrain,\n    num_boost_round= max_boosting_rounds,\n    valid_sets=[lgtrain, lgvalid],\n    valid_names=['train','valid'],\n    early_stopping_rounds=150,\n    verbose_eval=100\n)\n\ndel lgtrain, lgvalid ;  gc.collect();","metadata":{"trusted":true,"_uuid":"2ba84a7f5a2453295e54ea002480a862d1c2ad4e"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Get Optimal Threshold**","metadata":{"_uuid":"fb6af9b254a2fab15ea1e0a65bf0cba71083e875"}},{"cell_type":"code","source":"valid_pred = lgb_clf.predict(X_valid)\n_thresh = []\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    _thresh.append([thresh, metrics.f1_score(y_valid, (valid_pred>thresh).astype(int))])\n#     print(\"F1 score at threshold {0} is {1}\".format(thresh, metrics.f1_score(y_valid, (valid_pred>thresh).astype(int))))\n\n_thresh = np.array(_thresh)\nbest_id = _thresh[:,1].argmax()\nbest_thresh = _thresh[best_id][0]\nprint(\"Best Threshold: {}\\nF1 Score: {}\".format(best_thresh, _thresh[best_id][1]))","metadata":{"trusted":true,"_uuid":"88b07fbb444f853af18002e552bb6cb71d12b2d9"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Seed Diversification\n\nHere I build two final models with different seeds in order to reach better generalization.","metadata":{"_uuid":"6a7b03a6cebf8af4e423ddaace616ec3cbfe953b"}},{"cell_type":"code","source":"allmodelstart= time.time()\n# Run Model with different Seeds\nmulti_seed_pred = dict()\nall_feature_importance_df  = pd.DataFrame()\n\noptimal_rounds = lgb_clf.best_iteration\n\nlgtrain = lgb.Dataset(X, y, feature_name=feature_names)\n\nall_seeds = [27,22]\nfor seeds_x in all_seeds:\n    modelstart= time.time()\n    print(\"Seed: \", seeds_x,)\n    # Go Go Go\n    lgbm_params[\"seed\"] = seeds_x\n    lgb_seed_clf = lgb.train(\n        lgbm_params,\n        lgtrain,\n        num_boost_round = optimal_rounds + 1,\n        verbose_eval=200)\n\n    # Feature Importance\n    fold_importance_df = pd.DataFrame()\n    fold_importance_df[\"feature\"] = feature_names\n    fold_importance_df[\"importance\"] = lgb_seed_clf.feature_importance()\n    all_feature_importance_df = pd.concat([all_feature_importance_df, fold_importance_df], axis=0)\n\n    multi_seed_pred[seeds_x] =  list(lgb_seed_clf.predict(testing))\n    print(\"Model Runtime: %0.2f Minutes\"%((time.time() - modelstart)/60))\n    print(\"###########################################################################################\")\n    del lgb_seed_clf\n    \ndel lgtrain, X","metadata":{"trusted":true,"_uuid":"002cee9cfc6aa107d431d196491a4b8b043a0000"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Model Evaluation**","metadata":{"_uuid":"d5e118ffda862c0fabfd5ecee060e38c02a8f78b"}},{"cell_type":"code","source":"cols = all_feature_importance_df[[\"feature\", \"importance\"]].groupby(\"feature\").mean().sort_values(\n    by=\"importance\", ascending=False)[:50].index\nbest_features = all_feature_importance_df.loc[all_feature_importance_df.feature.isin(cols)]\nplt.figure(figsize=(8,10))\nsns.barplot(x=\"importance\", y=\"feature\", \n            data=best_features.sort_values(by=\"importance\", ascending=False))\nplt.title('LightGBM Features (avg over folds)')\nplt.tight_layout()\nplt.savefig('lgbm_importances.png')\nprint(\"All Model Runtime: %0.2f Minutes\"%((time.time() - allmodelstart)/60))\n\n# To DataFrame\nsub_preds = pd.DataFrame.from_dict(multi_seed_pred).replace(0,0.000001)\ndel multi_seed_pred; gc.collect();\n\n# Correlation Plot\nf, ax = plt.subplots(figsize=[8,6])\nsns.heatmap(sub_preds.corr(),\n            annot=True, fmt=\".2f\",cbar_kws={'label': 'Percentage %'},cmap=\"plasma\",ax=ax)\nax.set_title(\"Correlation Plot for Seed Diversified Models\")\nplt.show()","metadata":{"trusted":true,"_uuid":"82b5560a72a341a0a6a2fbbf04a569765160f742"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Submit**","metadata":{"_uuid":"523c94a5edf6dbb128e216d0466fd54bbc737841"}},{"cell_type":"code","source":"# Take Mean over Seed prediction\ntarget_var = 'prediction'\nmean_sub = sub_preds.mean(axis=1).rename(target_var)\nmean_sub = (mean_sub > best_thresh).astype(int)\nmean_sub.index = testdex\n\n# Submit\nmean_sub.to_csv('submission.csv',index = True, header=True)\nprint(mean_sub.value_counts(normalize=True))\nmean_sub.head()","metadata":{"trusted":true,"_uuid":"ad22ea8e1a77009616bfbecc46b964ee883337eb"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**SHAP** <br>\nHaving big memory issues here.","metadata":{"_uuid":"21502b19f29a8eb7ac3f6f85aee2d87fda8fa962"}},{"cell_type":"code","source":"non_sparse = pd.DataFrame(X_valid[:400].toarray(), columns = feature_names)\nprint(non_sparse.shape)","metadata":{"trusted":true,"_uuid":"5df882b7dd83c12cc2c5b634739871ae37639d1f","scrolled":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"explainer = shap.TreeExplainer(lgb_clf)\nshap_values = explainer.shap_values(non_sparse)","metadata":{"trusted":true,"_uuid":"619541f249c8b861ce4eca81d7dff5f59f1d3ad0"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# summarize the effects of all the features\nshap.summary_plot(shap_values, non_sparse)","metadata":{"trusted":true,"_uuid":"16e27b11f347b96443b971eafc26998559f70567"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# visualize the first prediction's explanation\nshap.force_plot(explainer.expected_value, shap_values[0,:], non_sparse.iloc[0,:])","metadata":{"trusted":true,"_uuid":"74e872a954d92a58d64fe6361dda8f018abafdfb"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# valid_data = train.loc[train.index.isin(y_valid.index), :]\n\n# # visualize the first prediction's explanation\n# for iteration in range(15):\n#     samp = random.randint(1,non_sparse.shape[0])\n#     print(\"Real Label: {}\".format(valid_data.target.iloc[samp]))\n#     display(valid_data.question_text.iloc[samp])\n#     display(shap.force_plot(explainer.expected_value, shap_values[samp,:], non_sparse.iloc[samp,:]))","metadata":{"trusted":true,"_uuid":"38cb8ff90b78f8bc0b786800fc6e80361cdf3c79"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"shap.dependence_plot(\"trump\", shap_values, non_sparse)","metadata":{"trusted":true,"_uuid":"51dfc50b1bd3162ea4d8a9411bc981d1893fb293"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Notebook Runtime: %0.2f Minutes\"%((time.time() - notebookstart)/60))","metadata":{"trusted":true,"_uuid":"11061797063972a160943341bec4ca4110ac52bf"},"outputs":[],"execution_count":null}]}