{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**XGboost Intro**\n\nXGboost is tree based model and one of the most powerful machine learning techniques; it can be used with patterns, numbers and text problems. However RNN models more common for text problems.\n\n**Different models structure and design produce better ensemble or stacking results.**\n\nThis model can be used as an ensemble or stack item alongside with RNN models to produce better results than any of the two models.","metadata":{"_uuid":"48111b174e6c7f374d0caf9cd028524ef2943a58"}},{"cell_type":"markdown","source":"# I. Introduction and Imports","metadata":{}},{"cell_type":"markdown","source":"First we load in the **Quora Insincere Data Training set** that has **been preprocessed**. Here we want to predict whether a person's question was either insincere or sincere.","metadata":{}},{"cell_type":"code","source":"# kaggle standard imports\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n# extra imports\nnp.random.seed(235)\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score, roc_auc_score, make_scorer, balanced_accuracy_score, roc_curve\nfrom sklearn.model_selection import GridSearchCV, StratifiedKFold\nfrom sklearn.datasets import make_classification\nfrom matplotlib import pyplot\nfrom sklearn import decomposition\nimport matplotlib.pyplot as plt\nimport gc\nimport re\n\n# XGboost related\nfrom sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\nimport xgboost as xgb\nfrom xgboost import XGBClassifier\nfrom scipy.sparse import csr_matrix, hstack\n\n# Any results you write to the current directory are saved as output.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-25T02:16:22.561727Z","iopub.execute_input":"2021-07-25T02:16:22.562125Z","iopub.status.idle":"2021-07-25T02:16:23.748651Z","shell.execute_reply.started":"2021-07-25T02:16:22.562094Z","shell.execute_reply":"2021-07-25T02:16:23.747694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Data Peparation\n references:\n* Data preparing and process inspired by (Shujian Liu) Kernals","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"}},{"cell_type":"code","source":"print('load data') \n# load training and testing data\ntrain_df = pd.read_csv(\"../input/train-processed/train_processed.csv\")\n\ntrain_df.shape[0]","metadata":{"execution":{"iopub.status.busy":"2021-07-25T02:16:23.750288Z","iopub.execute_input":"2021-07-25T02:16:23.750659Z","iopub.status.idle":"2021-07-25T02:16:27.302816Z","shell.execute_reply.started":"2021-07-25T02:16:23.750622Z","shell.execute_reply":"2021-07-25T02:16:27.302121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Here we just want to check if there are no miscellaneous outcomes\ntrain_df[\"target\"].unique()","metadata":{"execution":{"iopub.status.busy":"2021-07-25T02:16:27.304219Z","iopub.execute_input":"2021-07-25T02:16:27.304652Z","iopub.status.idle":"2021-07-25T02:16:27.326950Z","shell.execute_reply.started":"2021-07-25T02:16:27.304605Z","shell.execute_reply":"2021-07-25T02:16:27.326166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Our question_text is an object which is what we expect since it is text\ntrain_df.dtypes","metadata":{"execution":{"iopub.status.busy":"2021-07-25T02:16:27.328315Z","iopub.execute_input":"2021-07-25T02:16:27.328759Z","iopub.status.idle":"2021-07-25T02:16:27.336761Z","shell.execute_reply.started":"2021-07-25T02:16:27.328719Z","shell.execute_reply":"2021-07-25T02:16:27.336016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We need to determinne default behaviour for missing data and drop them\nprint(train_df.isna().sum())\ntrain_df.loc[train_df['question_text'].isna().values == True]\ntrain_df = train_df.dropna()\nprint(train_df.isna().sum())\n","metadata":{"execution":{"iopub.status.busy":"2021-07-25T02:16:27.337963Z","iopub.execute_input":"2021-07-25T02:16:27.338389Z","iopub.status.idle":"2021-07-25T02:16:28.398397Z","shell.execute_reply.started":"2021-07-25T02:16:27.338351Z","shell.execute_reply":"2021-07-25T02:16:28.397366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classification = ['Sincere','Insincere']\ninsincere = (train_df['target'] == 1).sum()\nsincere = (train_df['target'] == 0).sum()\ncount_of_sincerity = [sincere, insincere]\n\nplt.bar(classification, count_of_sincerity)\nplt.title('Sentiment type distribution in training set')\nplt.xlabel('Classification')\nplt.ylabel('Number of each classification')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-25T02:16:28.399654Z","iopub.execute_input":"2021-07-25T02:16:28.399958Z","iopub.status.idle":"2021-07-25T02:16:28.567176Z","shell.execute_reply.started":"2021-07-25T02:16:28.399931Z","shell.execute_reply":"2021-07-25T02:16:28.566259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#visualize the most frequent words\ninsincere = \" \".join([sentence for sentence in train_df['question_text'][train_df['target'] == 1]])\n\nfrom wordcloud import WordCloud\nwordcloud = WordCloud(width = 800, height = 500, random_state = 42, max_font_size =100).generate(insincere)\n\n#set up plots\nplt.figure(figsize=(15,8))\nplt.imshow(wordcloud, interpolation = 'bilinear')\nplt.axis('off')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-25T02:16:28.568332Z","iopub.execute_input":"2021-07-25T02:16:28.568610Z","iopub.status.idle":"2021-07-25T02:16:36.806180Z","shell.execute_reply.started":"2021-07-25T02:16:28.568577Z","shell.execute_reply":"2021-07-25T02:16:36.804998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count = train_df.groupby(by='target').count().copy()\ndisplay(count['question_text'][1])\nnumber_insincere = count['question_text'][1]","metadata":{"execution":{"iopub.status.busy":"2021-07-25T02:16:36.809357Z","iopub.execute_input":"2021-07-25T02:16:36.809752Z","iopub.status.idle":"2021-07-25T02:16:37.155118Z","shell.execute_reply.started":"2021-07-25T02:16:36.809712Z","shell.execute_reply":"2021-07-25T02:16:37.153961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# split training data to validation\ntrain_df, test_df = train_test_split(train_df, train_size=0.8)\ntrain_df = train_df.dropna()\ntest_df = test_df.dropna()","metadata":{"_uuid":"9ae2c3bb1724faf929f9a022038659caee9c4747","execution":{"iopub.status.busy":"2021-07-25T02:45:36.794731Z","iopub.execute_input":"2021-07-25T02:45:36.795253Z","iopub.status.idle":"2021-07-25T02:45:36.927958Z","shell.execute_reply.started":"2021-07-25T02:45:36.795214Z","shell.execute_reply":"2021-07-25T02:45:36.927118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('size of training data: ', train_df.shape)\nprint('Size of total data: ', )","metadata":{"_uuid":"ca84feac76b3d6b577574d26b155bd4dacc87da9","execution":{"iopub.status.busy":"2021-07-25T02:16:38.486473Z","iopub.execute_input":"2021-07-25T02:16:38.486906Z","iopub.status.idle":"2021-07-25T02:16:38.492410Z","shell.execute_reply.started":"2021-07-25T02:16:38.486862Z","shell.execute_reply":"2021-07-25T02:16:38.491430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **2. Prepare Vectors For XGboost input**","metadata":{"_uuid":"163c123c752aa75b4e0fa4c03e3e1d78022645f7"}},{"cell_type":"markdown","source":"**2.1 Count Vectorizer**","metadata":{}},{"cell_type":"markdown","source":"**2.2 Tf-IDF Vectorizer**","metadata":{}},{"cell_type":"code","source":"def tfidfvectorizer(X_train, X_val, X_test):\n    # word level tf-idf for XGB\n    tfidf_vect = TfidfVectorizer(analyzer='word', \n                                 token_pattern=r'\\w{1,}', \n                                 max_features=9000)\n    tfidf_vect.fit(train_df['question_text'])\n    xtrain_tfidf =  tfidf_vect.transform(X_train)\n    xvalid_tfidf =  tfidf_vect.transform(X_val)\n    xtest_tfidf =  tfidf_vect.transform(X_test)\n    # characters level tf-idf for XGB\n    tfidf_vect_ngram_chars = TfidfVectorizer(analyzer='char', \n                                             token_pattern=r'\\w{1,}', \n                                             ngram_range=(2,3), \n                                             max_features=9000)\n    tfidf_vect_ngram_chars.fit(train_df['question_text'])\n    xtrain_tfidf_ngram_chars =  tfidf_vect_ngram_chars.transform(X_train) \n    xvalid_tfidf_ngram_chars =  tfidf_vect_ngram_chars.transform(X_val) \n    xtest_tfidf_ngram_chars =  tfidf_vect_ngram_chars.transform(X_test) \n    return xtrain_tfidf, xvalid_tfidf, xtest_tfidf, xtrain_tfidf_ngram_chars, xvalid_tfidf_ngram_chars, xtest_tfidf_ngram_chars","metadata":{"_uuid":"0becd81182f9575e93cfcb39154ff4f817df791d","execution":{"iopub.status.busy":"2021-07-25T02:16:38.493723Z","iopub.execute_input":"2021-07-25T02:16:38.494135Z","iopub.status.idle":"2021-07-25T02:16:38.506966Z","shell.execute_reply.started":"2021-07-25T02:16:38.494094Z","shell.execute_reply":"2021-07-25T02:16:38.505942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**2.3 Topic Models for features**","metadata":{}},{"cell_type":"code","source":"# train a LDA Model\n#lda_model = decomposition.LatentDirichletAllocation(n_components=20, \n#                                                    learning_method='online', \n#                                                    max_iter=20)\n#X_topics = lda_model.fit_transform(xtrain_count)\n#topic_word = lda_model.components_ \n#vocab = count_vect.get_feature_names()\n\n# view the topic models\n#n_top_words = 10\n#topic_summaries = []\n#for i, topic_dist in enumerate(topic_word):\n#    topic_words = numpy.array(vocab)[numpy.argsort(topic_dist)][:-(n_top_words+1):-1]\n#    topic_summaries.append(' '.join(topic_words))","metadata":{"execution":{"iopub.status.busy":"2021-07-25T02:16:38.508330Z","iopub.execute_input":"2021-07-25T02:16:38.508720Z","iopub.status.idle":"2021-07-25T02:16:38.523672Z","shell.execute_reply.started":"2021-07-25T02:16:38.508681Z","shell.execute_reply":"2021-07-25T02:16:38.522762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**2.4 Features Engineering**","metadata":{"_uuid":"0b6c5ccd29443971829a92d454d54a0235f6f2b5"}},{"cell_type":"code","source":"gc.collect()","metadata":{"_uuid":"55ddb73c06a1826544301277403eccf5019e00cc","execution":{"iopub.status.busy":"2021-07-25T02:16:38.525052Z","iopub.execute_input":"2021-07-25T02:16:38.525456Z","iopub.status.idle":"2021-07-25T02:16:38.644571Z","shell.execute_reply.started":"2021-07-25T02:16:38.525330Z","shell.execute_reply":"2021-07-25T02:16:38.643371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_features(data):\n    for dataframe in data:\n        dataframe = pd.DataFrame(dataframe)\n        dataframe[\"text_size\"] = dataframe[\"question_text\"].apply(len).astype('uint16')\n        dataframe[\"exc_count\"] = dataframe[\"question_text\"].apply(lambda x: x.count(\"!\")).astype('uint16')\n        dataframe[\"quetion_count\"] = dataframe[\"question_text\"].apply(lambda x: x.count(\"?\")).astype('uint16')\n        dataframe[\"punctuation_count\"] = dataframe[\"question_text\"].apply(lambda x: sum(x.count(p) for p in '.,;:^_`')).astype('uint16')\n        dataframe[\"symbol_count\"] = dataframe[\"question_text\"].apply(lambda x: sum(x.count(p) for p in '*&$%')).astype('uint16')\n        dataframe[\"words_count\"] = dataframe[\"question_text\"].apply(lambda x: len(x.split())).astype('uint16')\n        dataframe[\"unique_words\"] = dataframe[\"question_text\"].apply(lambda x: (len(set(1 for w in x.split())))).astype('uint16')\n        dataframe[\"unique_rate\"] = dataframe[\"unique_words\"] / dataframe[\"words_count\"]\n        dataframe[\"word_max_length\"] = dataframe[\"question_text\"].apply(lambda x: max([len(word) for word in x.split()]) ).astype('uint16')\n    return data","metadata":{"_uuid":"9f4ccc3ad674626ed9e2cd7b7df02ed4fb66b6a3","execution":{"iopub.status.busy":"2021-07-25T02:16:38.646474Z","iopub.execute_input":"2021-07-25T02:16:38.647015Z","iopub.status.idle":"2021-07-25T02:16:38.661171Z","shell.execute_reply.started":"2021-07-25T02:16:38.646969Z","shell.execute_reply":"2021-07-25T02:16:38.659708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('generate the features')\n\ndata = train_df, test_df\ndata = get_features(data)\n","metadata":{"_uuid":"7fe8ed93b80740184d6728feb9dfbf85bdf8187b","execution":{"iopub.status.busy":"2021-07-25T02:16:38.662961Z","iopub.execute_input":"2021-07-25T02:16:38.663423Z","iopub.status.idle":"2021-07-25T02:16:56.266290Z","shell.execute_reply.started":"2021-07-25T02:16:38.663381Z","shell.execute_reply":"2021-07-25T02:16:56.265317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_cols = [\"text_size\", \"exc_count\", \"quetion_count\", \"punctuation_count\", \"symbol_count\", \"words_count\", \"unique_words\", \"unique_rate\", \"word_max_length\"]","metadata":{"_uuid":"534d5800ac72009dc4373d37c656dc3e0a0a7bff","execution":{"iopub.status.busy":"2021-07-25T02:16:56.267556Z","iopub.execute_input":"2021-07-25T02:16:56.267828Z","iopub.status.idle":"2021-07-25T02:16:56.272337Z","shell.execute_reply.started":"2021-07-25T02:16:56.267803Z","shell.execute_reply":"2021-07-25T02:16:56.271287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Hyper Paramater Tuning","metadata":{}},{"cell_type":"code","source":"param_test1 = {\n 'max_depth':range(3,10,2),\n 'min_child_weight':range(1,6,2)\n}\nX_train = csr_matrix(train_df[feature_cols].values)\ny_train = train_df[\"target\"]\ngsearch1 = GridSearchCV(estimator = XGBClassifier( learning_rate =0.1, n_estimators=140, max_depth=5,\n min_child_weight=1, gamma=0, subsample=0.8, colsample_bytree=0.8,\n objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27), \n param_grid = param_test1, scoring='roc_auc',n_jobs=4,iid=False, cv=4)\ngsearch1.fit(X_train, y_train)\ngsearch1.cv_results_, gsearch1.best_params_, gsearch1.best_score_","metadata":{"execution":{"iopub.status.busy":"2021-07-25T02:16:56.273397Z","iopub.execute_input":"2021-07-25T02:16:56.273664Z","iopub.status.idle":"2021-07-25T02:16:59.485703Z","shell.execute_reply.started":"2021-07-25T02:16:56.273632Z","shell.execute_reply":"2021-07-25T02:16:59.482347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"param_test2 = {\n 'max_depth':[4,5,6],\n 'min_child_weight':[4,5,6]\n}\ngsearch2 = GridSearchCV(estimator = XGBClassifier( learning_rate=0.1, n_estimators=140, max_depth=5,\n min_child_weight=2, gamma=0, subsample=0.8, colsample_bytree=0.8,\n objective= 'binary:logistic', nthread=4, scale_pos_weight=1,seed=27), \n param_grid = param_test2, scoring='roc_auc',n_jobs=4,iid=False, cv=5)\ngsearch2.fit(X_train,y_train)\ngsearch2.cv_results_, gsearch2.best_params_, gsearch2.best_score_","metadata":{"execution":{"iopub.status.busy":"2021-07-25T02:16:59.487080Z","iopub.status.idle":"2021-07-25T02:16:59.487585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"param_test3 = {\n 'gamma':[i/10.0 for i in range(0,5)]\n}\ngsearch3 = GridSearchCV(estimator = XGBClassifier( learning_rate =0.1, n_estimators=140, max_depth=4,\n min_child_weight=6, gamma=0, subsample=0.8, colsample_bytree=0.8,\n objective= 'binary:logistic', nthread=4, scale_pos_weight=1,seed=27), \n param_grid = param_test3, scoring='roc_auc',n_jobs=4,iid=False, cv=5)\ngsearch3.fit(X_train,y_train)\ngsearch3.cv_results_, gsearch3.best_params_, gsearch3.best_score_","metadata":{"execution":{"iopub.status.busy":"2021-07-25T02:16:59.488697Z","iopub.status.idle":"2021-07-25T02:16:59.489261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"param_test4 = {\n 'learning_rate':[i/10.0 for i in range(0,10)]\n}\ngsearch4 = GridSearchCV(estimator = XGBClassifier( learning_rate =0.1, n_estimators=140, max_depth=4,\n min_child_weight=6, gamma=0, subsample=0.8, colsample_bytree=0.8,\n objective= 'binary:logistic', nthread=4, scale_pos_weight=1,seed=27), \n param_grid = param_test4, scoring='roc_auc',n_jobs=4,iid=False, cv=5)\ngsearch4.fit(X_train,y_train)\ngsearch4.cv_results_, gsearch4.best_params_, gsearch4.best_score_","metadata":{"execution":{"iopub.status.busy":"2021-07-25T02:16:59.490334Z","iopub.status.idle":"2021-07-25T02:16:59.490762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Input Final Format**","metadata":{"_uuid":"2573ac8321b0e414ea833d5c55ef263fdd04925b"}},{"cell_type":"code","source":"print('final preparation for input')\ndef prep_input_vals(train_index, val_df_kfolds):\n    X_train = csr_matrix(train_df.iloc[train_index][feature_cols].dropna().values)\n    X_val = csr_matrix(val_df_kfolds[feature_cols].values)\n    X_test = csr_matrix(test_df[feature_cols].values)\n    return X_train, X_val, X_test\ngc.collect()","metadata":{"_uuid":"6422a7466b8b1cc4d5e5f7217441384b5b8f93bb","execution":{"iopub.status.busy":"2021-07-25T02:17:02.698362Z","iopub.execute_input":"2021-07-25T02:17:02.698714Z","iopub.status.idle":"2021-07-25T02:17:02.830024Z","shell.execute_reply.started":"2021-07-25T02:17:02.698683Z","shell.execute_reply":"2021-07-25T02:17:02.829236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_hstack(X_train, xtrain_tfidf, xtrain_tfidf_ngram_chars, X_val, xvalid_tfidf, xvalid_tfidf_ngram_chars, \n                 X_test, xtest_tfidf, xtest_tfidf_ngram_chars):\n    input_train = hstack([X_train, xtrain_tfidf, xtrain_tfidf_ngram_chars])\n    input_valid = hstack([X_val, xvalid_tfidf, xvalid_tfidf_ngram_chars])\n    input_test = hstack([X_test, xtest_tfidf, xtest_tfidf_ngram_chars])\n    return input_train, input_valid, input_test\n\n#print('input_train: ', input_train)\ntrain_word_vector = None\ntrain_char_vector = None\nvalid_word_vector = None\nvalid_char_vector = None\ntest_word_vector = None\ntest_char_vector = None\n#print('input_train: ', input_train)","metadata":{"_uuid":"7020d248f7a587caff1a9a52ecfc3570c9f64d3f","execution":{"iopub.status.busy":"2021-07-25T02:17:03.193184Z","iopub.execute_input":"2021-07-25T02:17:03.193533Z","iopub.status.idle":"2021-07-25T02:17:03.200818Z","shell.execute_reply.started":"2021-07-25T02:17:03.193505Z","shell.execute_reply":"2021-07-25T02:17:03.199858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Build The model**","metadata":{"_uuid":"2b76230d5c959cb961e5cafd7e25f0aa61e6454d"}},{"cell_type":"code","source":"'''reference: some settings inspired by Toxic competition kernels'''\ndef build_xgb(train_X, train_y, valid_X, valid_y=None, subsample=0.75):\n\n    xgtrain = xgb.DMatrix(train_X, label=train_y)\n    if valid_y is not None:\n        xgvalid = xgb.DMatrix(valid_X, label=valid_y)\n    else:\n        xgvalid = None\n    \n    model_params = {}\n    # binary 0 or 1\n    model_params['objective'] = 'binary:logistic'\n    # eta is the learning_rate, [default=0.3]\n    model_params['eta'] = 0.3\n    # depth of the tree, deeper more complex.\n    model_params['max_depth'] =7\n    # 0 [default] print running messages, 1 means silent mode\n    model_params['silent'] = 1\n    model_params['eval_metric'] = 'auc'\n    # will give up further partitioning [default=1]\n    model_params['min_child_weight'] =1\n    # subsample ratio for the training instance\n    model_params['subsample'] = subsample\n    # subsample ratio of columns when constructing each tree\n    model_params['colsample_bytree'] = subsample\n    # random seed\n    model_params['seed'] = 2021\n    \n    # convert params to list\n    model_params = model_params\n    \n    return xgtrain, xgvalid, model_params","metadata":{"_uuid":"c1c2f1433005c43d885f93ff65064aaa5c528518","execution":{"iopub.status.busy":"2021-07-25T02:17:03.275484Z","iopub.execute_input":"2021-07-25T02:17:03.275874Z","iopub.status.idle":"2021-07-25T02:17:03.283702Z","shell.execute_reply.started":"2021-07-25T02:17:03.275827Z","shell.execute_reply":"2021-07-25T02:17:03.282613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Train The Model**","metadata":{"_uuid":"a7285359448939dfe848af8f4f284a9eded1a0a9"}},{"cell_type":"code","source":"def run_model(xgtrain, xgvalid, model_params, num_rounds=500, patience=2):\n    # watchlist what information should be printed. specify validation monitoring\n    watchlist = [ (xgtrain, 'train'), (xgvalid, 'test') ]\n    #early_stopping_rounds = stop if performance does not improve for k rounds\n    model = xgb.train(model_params, xgtrain, num_rounds, watchlist, early_stopping_rounds=patience)\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2021-07-25T02:17:03.334225Z","iopub.execute_input":"2021-07-25T02:17:03.334587Z","iopub.status.idle":"2021-07-25T02:17:03.340104Z","shell.execute_reply.started":"2021-07-25T02:17:03.334555Z","shell.execute_reply":"2021-07-25T02:17:03.339040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Python program to get average of a list\ndef Average(lst):\n    return sum(lst) / len(lst)","metadata":{"execution":{"iopub.status.busy":"2021-07-25T02:17:03.467328Z","iopub.execute_input":"2021-07-25T02:17:03.467686Z","iopub.status.idle":"2021-07-25T02:17:03.472553Z","shell.execute_reply.started":"2021-07-25T02:17:03.467658Z","shell.execute_reply":"2021-07-25T02:17:03.471320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_roc(X_test, model, input_test):\n    y_test = test_df[\"target\"]\n    # generate a no skill prediction (majority class)\n    ns_probs = [0 for _ in range(len(y_test))]\n    predictions = np.zeros(( X_test.shape[0], 1) )\n    predictions[:,0] = model.predict(xgb.DMatrix(input_test), ntree_limit=model.best_ntree_limit)\n    xg_probs = predictions[:,0]\n    ns_auc = roc_auc_score(y_test, ns_probs)\n    xg_auc = roc_auc_score(y_test, xg_probs)\n    # summarize scores\n    print('No Skill: ROC AUC=%.3f' % (ns_auc))\n    print('Logistic: ROC AUC=%.3f' % (xg_auc))\n    # calculate roc curves\n    ns_fpr, ns_tpr, _ = roc_curve(y_test, ns_probs)\n    lr_fpr, lr_tpr, _ = roc_curve(y_test, xg_probs)\n    # plot the roc curve for the model\n    pyplot.plot(ns_fpr, ns_tpr, linestyle='--', label='No Skill')\n    pyplot.plot(lr_fpr, lr_tpr, marker='.', label='XGBoost')\n    # axis labels\n    pyplot.xlabel('False Positive Rate')\n    pyplot.ylabel('True Positive Rate')\n    # show the legend\n    pyplot.legend()\n    # show the plot\n    pyplot.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-25T02:17:03.672625Z","iopub.execute_input":"2021-07-25T02:17:03.672986Z","iopub.status.idle":"2021-07-25T02:17:03.682498Z","shell.execute_reply.started":"2021-07-25T02:17:03.672955Z","shell.execute_reply":"2021-07-25T02:17:03.681333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def thresholding(validate_hat, y_val):\n    best_f1 = 0\n    best_accuracy = 0\n    scores_list = []\n    accuracy_list = []\n    fscores = []    \n    thresholds = np.arange(0, 1, 0.01)\n    for threshold in thresholds:\n        score = f1_score(y_val, (validate_hat > threshold).astype(int))\n        fscores.append(score)\n        scores_list.append([threshold, score])\n        _accuracy = balanced_accuracy_score(y_val, (validate_hat > threshold).astype(int))\n        accuracy_list .append([threshold, _accuracy])\n        print('F1 score: {} for threshold: {}'.format(score, threshold))\n        print('accuracy score: {} for threshold: {}'.format(_accuracy, threshold))\n\n        accuracy_list.sort(key=lambda x:x[1] , reverse=True)\n        scores_list.sort(key=lambda x:x[1] , reverse=True)\n\n        best_threshold = scores_list[0][0]\n        print('best threshold to generate predictions: ', best_threshold)\n        print('best f1 score: ', scores_list[0][1])\n        print('best accuracy score: ', accuracy_list[0][1])\n        if scores_list[0][1] > best_f1:\n            best_f1 = scores_list[0][1]\n            best_accuracy = accuracy_list[0][1]\n    plt.plot(thresholds, fscores)\n    plt.xlabel(\"Threhold\")\n    plt.ylabel(\"F1-Score\")\n    plt.show()\n    return best_f1, best_accuracy, best_threshold","metadata":{"execution":{"iopub.status.busy":"2021-07-25T02:17:04.019427Z","iopub.execute_input":"2021-07-25T02:17:04.019777Z","iopub.status.idle":"2021-07-25T02:17:04.030597Z","shell.execute_reply.started":"2021-07-25T02:17:04.019748Z","shell.execute_reply":"2021-07-25T02:17:04.029288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_xgboost():\n\n    # split kfolds for tfidf char and word data\n    # split kfolds for train data\n    param_grid = [{'min_child_weight': np.arange(0.1, 10.1, 0.1)}] #set of trial values for min_child_weight\n    i=1\n    kf = StratifiedKFold(n_splits=4,random_state=1,shuffle=True)\n    X = train_df[\"question_text\"].dropna()\n    y = train_df[\"target\"].dropna()\n    for train_index,test_index in kf.split(X,y):\n        print('\\n{} of kfold {}'.format(i,kf.n_splits))\n        X_train,X_val = X.iloc[train_index].dropna(),X.iloc[test_index].dropna()\n        y_train,y_val = y.iloc[train_index].dropna(),y.iloc[test_index].dropna()\n        X_test, y_test = test_df[\"question_text\"],test_df[\"target\"]\n        # Vectorize data\n        print(\"Vectorizing Data...\")\n        xtrain_tfidf, xvalid_tfidf, xtest_tfidf, xtrain_tfidf_ngram_chars, xvalid_tfidf_ngram_chars, xtest_tfidf_ngram_chars = tfidfvectorizer(X_train, X_val, X_test)\n        gc.collect()\n        # We need to make sure for the hstack later that they are all of the same size\n        print(\"Prepping input values...\")\n        gc.collect()\n        # Get input values\n        X_train, X_val, X_test = prep_input_vals(train_index, train_df.iloc[test_index].dropna()) # pass in the index so we can get our spicey sparse matrix \n                                                                                                # + the validate matrix we also we want to pass into our spicey sparse\n        gc.collect()\n        print(\"Creating Hstack...\")\n        # Create hstack\n        input_train, input_valid, input_test = create_hstack(X_train, xtrain_tfidf, xtrain_tfidf_ngram_chars, \n                                                             X_val, xvalid_tfidf, xvalid_tfidf_ngram_chars, \n                                                             X_test, xtest_tfidf, xtest_tfidf_ngram_chars)\n        gc.collect()\n        print(\"Building model...\")\n        #Build the model\n        xgtrain, xgvalid, model_params = build_xgb(input_train, y_train ,input_valid, y_val)\n        gc.collect()\n        print(\"Running model...\")\n        model = run_model(xgtrain, xgvalid, model_params)\n        gc.collect()\n        print('predict validation...')\n        validate_hat = np.zeros(( X_val.shape[0], 1) )\n        validate_hat[:,0] = model.predict(xgb.DMatrix(input_valid), ntree_limit=model.best_ntree_limit)\n        \n        \n        print('Plot roc curve...')\n        plot_roc(X_test, model, input_test)\n        \n        print('Thresholding Validated Set...')\n        best_f1, best_accuracy, best_threshold = thresholding(validate_hat, y_val)\n        \n        \n        i+=1\n\n    test_hat = np.zeros(( X_test.shape[0], 1) )\n    test_hat[:,0] = model.predict(xgb.DMatrix(input_test), ntree_limit=model.best_ntree_limit)\n    \n    print('Thresholding Test Set...')\n    best_f1_test, best_accuracy_test, best_threshold_test = thresholding(test_hat, y_test)\n    \n    return model, best_f1, best_accuracy, best_f1_test, best_accuracy_test, best_threshold, best_threshold_test","metadata":{"_uuid":"e766ffa7a679026f213701f833e3df1a835dffdf","execution":{"iopub.status.busy":"2021-07-25T02:17:04.348546Z","iopub.execute_input":"2021-07-25T02:17:04.348899Z","iopub.status.idle":"2021-07-25T02:17:04.364524Z","shell.execute_reply.started":"2021-07-25T02:17:04.348868Z","shell.execute_reply":"2021-07-25T02:17:04.363460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('train the model')\nbest_f1score = 0\nmodel, best_f1score, best_accuracy, best_f1_test, best_accuracy_test, best_threshold, best_threshold_test = train_xgboost()","metadata":{"_uuid":"f10ccb2e8f3f61f334f7e56f39339bb1eb519f2c","execution":{"iopub.status.busy":"2021-07-25T02:17:04.762223Z","iopub.execute_input":"2021-07-25T02:17:04.762560Z","iopub.status.idle":"2021-07-25T02:33:42.043162Z","shell.execute_reply.started":"2021-07-25T02:17:04.762532Z","shell.execute_reply":"2021-07-25T02:33:42.041950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nprint(\"Best F1_score:\", best_f1score)\nprint(\"Best Accuracy: \", best_accuracy)\nprint(\"Best Threshold: \", best_threshold)\n\nprint(\"Best F1_score:\", best_f1_test)\nprint(\"Best Accuracy: \", best_accuracy_test)\nprint(\"Best Threshold: \", best_threshold_test)","metadata":{"execution":{"iopub.status.busy":"2021-07-25T02:33:42.045261Z","iopub.execute_input":"2021-07-25T02:33:42.045572Z","iopub.status.idle":"2021-07-25T02:33:42.053975Z","shell.execute_reply.started":"2021-07-25T02:33:42.045544Z","shell.execute_reply":"2021-07-25T02:33:42.052760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=7)\n#print(model_params)\n#grid_search = GridSearchCV(model, model_params, scoring=\"neg_log_loss\", n_jobs=-1, cv=kfold, verbose=1)\n#grid_result = grid_search.fit(X, label_encoded_y)","metadata":{"execution":{"iopub.status.busy":"2021-07-25T02:16:59.506570Z","iopub.status.idle":"2021-07-25T02:16:59.507034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Predict And Export Results**","metadata":{"_uuid":"4c375986cbe497613c4ebf0f8ded3e6424119188"}},{"cell_type":"code","source":"#print('predict results')\n#predictions = np.zeros(( X_test.shape[0], 1) )\n#predictions[:,0] = model.predict(xgb.DMatrix(input_test), ntree_limit=model.best_ntree_limit)\n#print(predictions)","metadata":{"_uuid":"fe262e4e294af12da0c44ff12c5fc4c6250dab81","execution":{"iopub.status.busy":"2021-07-25T02:16:59.508048Z","iopub.status.idle":"2021-07-25T02:16:59.508494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#def save_results(submit, y_hat, name, threshold=0.35):\n#    print('threshold is: ', threshold)\n#    results = (y_hat > threshold).astype(int)\n#    print(results[:100])\n#    submit['prediction'] = results\n#    save_to = (name+'.csv')\n#    submit.to_csv(save_to, index=False)","metadata":{"_uuid":"c17d05438b30e2d75ccc97e34c14a19a62193a21","execution":{"iopub.status.busy":"2021-07-25T02:16:59.509578Z","iopub.status.idle":"2021-07-25T02:16:59.510069Z"},"trusted":true},"execution_count":null,"outputs":[]}]}