{"cells":[{"metadata":{"_uuid":"48111b174e6c7f374d0caf9cd028524ef2943a58"},"cell_type":"markdown","source":"**XGboost Intro**\n\nXGboost is tree based model and one of the most powerful machine learning techniques; it can be used with patterns, numbers and text problems. However RNN models more common for text problems.\n\n**Different models structure and design produce better ensemble or stacking results.**\n\nThis model can be used as an ensemble or stack item alongside with RNN models to produce better results than any of the two models."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\n# kaggle standard imports\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nprint(os.listdir(\"../input\"))\n\n# extra imports\nnp.random.seed(235)\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score\nimport gc\nimport re\nfrom sklearn.metrics import f1_score\n\n# XGboost related\nfrom sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\nimport xgboost as xgb\nfrom xgboost import XGBClassifier\nfrom scipy.sparse import csr_matrix, hstack\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"**Prepare Data**\n\nreferences:\n* Data preparing and process inspired by (Shujian Liu) Kernals"},{"metadata":{"trusted":true,"_uuid":"9ae2c3bb1724faf929f9a022038659caee9c4747"},"cell_type":"code","source":"print('load data') \n# load training and testing data\ntrain_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")\n\n# split training data to validation\ntrain_df, val_df = train_test_split(train_df, train_size=0.9, random_state=235)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f0fc755b30c60e322fda999950ffd63240bbc1bf"},"cell_type":"code","source":"print('fill missing and get the values')\n# fill missing and get the values\nX_train = train_df[\"question_text\"].fillna(\"na_\").values\nX_val = val_df[\"question_text\"].fillna(\"na_\").values\nX_test = test_df[\"question_text\"].fillna(\"na_\").values\n\ny_train = train_df['target'].values\ny_val = val_df['target'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ca84feac76b3d6b577574d26b155bd4dacc87da9"},"cell_type":"code","source":"print('size of training data: ', X_train.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"163c123c752aa75b4e0fa4c03e3e1d78022645f7"},"cell_type":"markdown","source":"**Prepare Vectors For XGboost input**"},{"metadata":{"trusted":true,"_uuid":"08f314b8831c5be9573d75bd97e00415fbee7d6c"},"cell_type":"code","source":"char_vector = TfidfVectorizer(\n    ngram_range=(2,4),\n    max_features=20000,\n    stop_words='english',\n    analyzer='char_wb',\n    token_pattern=r'\\w{1,}',\n    strip_accents='unicode',\n    sublinear_tf=True, \n    max_df=0.98,\n    min_df=2\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0becd81182f9575e93cfcb39154ff4f817df791d"},"cell_type":"code","source":"print('fit char vector')\nchar_vector.fit(X_train[:85000])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bd9eb370e046837b06def19c5ca38fa5b6de1682"},"cell_type":"code","source":"print('transfer data based on char vector')\nprint('transfer train')\ntrain_char_vector = char_vector.transform(X_train).tocsr()\nprint('transfer validation')\nvalid_char_vector = char_vector.transform(X_val).tocsr()\nprint('transfer test')\ntest_char_vector = char_vector.transform(X_test).tocsr()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f3d8c9156b03f38f9cbbb73e9998929b31b622b6"},"cell_type":"code","source":"all_text = list(X_train) + list(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"329ce3fcbe18728279c53e6841dc190767ec0767"},"cell_type":"code","source":"word_vector = TfidfVectorizer(\n    ngram_range=(1,1), \n    max_features=9000,\n    sublinear_tf=True, \n    strip_accents='unicode', \n    analyzer='word', \n    token_pattern=\"\\w{1,}\", \n    stop_words=\"english\",\n    max_df=0.95,\n    min_df=2\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1731888ea00b98e3e36d959481a89ca2f0f12dc2"},"cell_type":"code","source":"print('fit word vector')\nword_vector.fit(all_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e8a9de95067a6a5c8abec89ca4249350cadbe04b"},"cell_type":"code","source":"print('transfer data based on word vector')\ntrain_word_vector = word_vector.transform(X_train).tocsr()\nvalid_word_vector = word_vector.transform(X_val).tocsr()\ntest_word_vector = word_vector.transform(X_test).tocsr()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0b6c5ccd29443971829a92d454d54a0235f6f2b5"},"cell_type":"markdown","source":"**Features Engineering**"},{"metadata":{"trusted":true,"_uuid":"55ddb73c06a1826544301277403eccf5019e00cc"},"cell_type":"code","source":"del all_text\ndel X_train\ndel X_val\ndel X_test\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"86d8367fac1c7a1a551e264697043b9e81cd52c7"},"cell_type":"code","source":"data = [train_df, val_df, test_df]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fde1b513faff3ad7f9effef329a9a87cdd6a6a7e"},"cell_type":"code","source":"# references: https://www.kaggle.com/theoviel/improve-your-score-with-some-text-preprocessing\nmistake_list = ['colour', 'centre', 'favourite', 'travelling', 'counselling', 'theatre', 'cancelled', 'labour', 'organisation', 'wwii', 'citicise', 'youtu ', 'youtube ', 'Qoura', 'sallary', 'Whta', 'narcisist', 'howdo', 'whatare', 'howcan', 'howmuch', 'howmany', 'whydo', 'doI', 'theBest', 'howdoes', 'mastrubation', 'mastrubate', \"mastrubating\", 'pennis', 'Etherium', 'narcissit', 'bigdata', '2k17', '2k18', 'qouta', 'exboyfriend', 'airhostess', 'whst', 'watsapp', 'demonitisation', 'demonitization', 'demonetisation']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f4ccc3ad674626ed9e2cd7b7df02ed4fb66b6a3"},"cell_type":"code","source":"def get_features(data):\n    for dataframe in data:\n        dataframe[\"text_size\"] = dataframe[\"question_text\"].apply(len).astype('uint16')\n        dataframe[\"capital_size\"] = dataframe[\"question_text\"].apply(lambda x: sum(1 for c in x if c.isupper())).astype('uint16')\n        dataframe[\"capital_rate\"] = dataframe.apply(lambda x: float(x[\"capital_size\"]) / float(x[\"text_size\"]), axis=1).astype('float16')\n        dataframe[\"exc_count\"] = dataframe[\"question_text\"].apply(lambda x: x.count(\"!\")).astype('uint16')\n        dataframe[\"quetion_count\"] = dataframe[\"question_text\"].apply(lambda x: x.count(\"?\")).astype('uint16')\n        dataframe[\"unq_punctuation_count\"] = dataframe[\"question_text\"].apply(lambda x: sum(x.count(p) for p in '∞θ÷α•à−β∅³π‘₹´°£€\\×™√²')).astype('uint16')\n        dataframe[\"punctuation_count\"] = dataframe[\"question_text\"].apply(lambda x: sum(x.count(p) for p in '.,;:^_`')).astype('uint16')\n        dataframe[\"symbol_count\"] = dataframe[\"question_text\"].apply(lambda x: sum(x.count(p) for p in '*&$%')).astype('uint16')\n        dataframe[\"words_count\"] = dataframe[\"question_text\"].apply(lambda x: len(x.split())).astype('uint16')\n        dataframe[\"unique_words\"] = dataframe[\"question_text\"].apply(lambda x: (len(set(1 for w in x.split())))).astype('uint16')\n        dataframe[\"unique_rate\"] = dataframe[\"unique_words\"] / dataframe[\"words_count\"]\n        dataframe[\"word_max_length\"] = dataframe[\"question_text\"].apply(lambda x: max([len(word) for word in x.split()]) ).astype('uint16')\n        dataframe[\"mistake_count\"] = dataframe[\"question_text\"].apply(lambda x: sum(x.count(w) for w in mistake_list)).astype('uint16')\n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7fe8ed93b80740184d6728feb9dfbf85bdf8187b"},"cell_type":"code","source":"print('generate the features')\ndata = get_features(data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"534d5800ac72009dc4373d37c656dc3e0a0a7bff"},"cell_type":"code","source":"feature_cols = [\"text_size\", \"capital_size\", \"capital_rate\", \"exc_count\", \"quetion_count\", \"unq_punctuation_count\", \"punctuation_count\", \"symbol_count\", \"words_count\", \"unique_words\", \"unique_rate\", \"word_max_length\", \"mistake_count\"]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2573ac8321b0e414ea833d5c55ef263fdd04925b"},"cell_type":"markdown","source":"**Input Final Format**"},{"metadata":{"trusted":true,"_uuid":"6422a7466b8b1cc4d5e5f7217441384b5b8f93bb"},"cell_type":"code","source":"print('final preparation for input')\nX_train = csr_matrix(train_df[feature_cols].values)\nX_val = csr_matrix(val_df[feature_cols].values)\nX_test = csr_matrix(test_df[feature_cols].values)\n\ndel val_df\ndel train_df\ndel test_df\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7020d248f7a587caff1a9a52ecfc3570c9f64d3f"},"cell_type":"code","source":"'''\ninput_train = hstack([X_train, train_char_vector,train_word_vector])\ninput_valid = hstack([X_val, valid_char_vector, valid_word_vector])\ninput_test = hstack([X_test, test_char_vector, test_word_vector])\n'''\ninput_train = hstack([X_train, train_word_vector, train_char_vector])\ninput_valid = hstack([X_val, valid_word_vector, valid_char_vector])\ninput_test = hstack([X_test, test_word_vector, test_char_vector])\n\n#print('input_train: ', input_train)\ntrain_word_vector = None\ntrain_char_vector = None\nvalid_word_vector = None\nvalid_char_vector = None\ntest_word_vector = None\ntest_char_vector = None\n#print('input_train: ', input_train)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2b76230d5c959cb961e5cafd7e25f0aa61e6454d"},"cell_type":"markdown","source":"**Build The model**"},{"metadata":{"trusted":true,"_uuid":"c1c2f1433005c43d885f93ff65064aaa5c528518"},"cell_type":"code","source":"'''reference: some settings inspired by Toxic competition kernels'''\ndef build_xgb(train_X, train_y, valid_X, valid_y=None, subsample=0.75):\n\n    xgtrain = xgb.DMatrix(train_X, label=train_y)\n    if valid_y is not None:\n        xgvalid = xgb.DMatrix(valid_X, label=valid_y)\n    else:\n        xgvalid = None\n    \n    model_params = {}\n    # binary 0 or 1\n    model_params['objective'] = 'binary:logistic'\n    # eta is the learning_rate, [default=0.3]\n    model_params['eta'] = 0.3\n    # depth of the tree, deeper more complex.\n    model_params['max_depth'] = 6\n    # 0 [default] print running messages, 1 means silent mode\n    model_params['silent'] = 1\n    model_params['eval_metric'] = 'auc'\n    # will give up further partitioning [default=1]\n    model_params['min_child_weight'] = 1\n    # subsample ratio for the training instance\n    model_params['subsample'] = subsample\n    # subsample ratio of columns when constructing each tree\n    model_params['colsample_bytree'] = subsample\n    # random seed\n    model_params['seed'] = 2018\n    # imbalance data ratio\n    #model_params['scale_pos_weight'] = \n    \n    # convert params to list\n    model_params = list(model_params.items())\n    \n    return xgtrain, xgvalid, model_params","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a7285359448939dfe848af8f4f284a9eded1a0a9"},"cell_type":"markdown","source":"**Train The Model**"},{"metadata":{"trusted":true,"_uuid":"e766ffa7a679026f213701f833e3df1a835dffdf"},"cell_type":"code","source":"def train_xgboost(xgtrain, xgvalid, model_params, num_rounds=500, patience=20):\n    \n    if xgvalid is not None:\n        # watchlist what information should be printed. specify validation monitoring\n        watchlist = [ (xgtrain, 'train'), (xgvalid, 'test') ]\n        #early_stopping_rounds = stop if performance does not improve for k rounds\n        model = xgb.train(model_params, xgtrain, num_rounds, watchlist, early_stopping_rounds=patience)\n    else:\n        model = xgb.train(model_params, xgtrain, num_rounds)\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f10ccb2e8f3f61f334f7e56f39339bb1eb519f2c"},"cell_type":"code","source":"print('train the model')\nxgtrain, xgvalid, model_params = build_xgb(input_train, y_train ,input_valid, y_val)\nmodel = train_xgboost(xgtrain, xgvalid, model_params)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4c375986cbe497613c4ebf0f8ded3e6424119188"},"cell_type":"markdown","source":"**Predict And Export Results**"},{"metadata":{"trusted":true,"_uuid":"dc8f12850f75a5bf84417036b0999726bdc25b7c"},"cell_type":"code","source":"print('predict validation')\nvalidate_hat = np.zeros(( X_val.shape[0], 1) )\nvalidate_hat[:,0] = model.predict(xgb.DMatrix(input_valid), ntree_limit=model.best_ntree_limit)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c3aab074192a679ba88f8414fda4f55940d924ec"},"cell_type":"code","source":"scores_list = []\nfor threshold in [0.2, 0.3, 0.31, 0.33, 0.4, 0.45, 0.5]:\n    score = f1_score(y_val, (validate_hat > threshold).astype(int))\n    scores_list.append([threshold, score])\n    print('F1 score: {} for threshold: {}'.format(score, threshold))\n        \n    scores_list.sort(key=lambda x:x[1] , reverse=True)\n    best_threshold = scores_list[0][0]\n    print('best threshold to generate predictions: ', best_threshold)\n    print('best score: ', scores_list[0][1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe262e4e294af12da0c44ff12c5fc4c6250dab81"},"cell_type":"code","source":"print('predict results')\npredictions = np.zeros(( X_test.shape[0], 1) )\npredictions[:,0] = model.predict(xgb.DMatrix(input_test), ntree_limit=model.best_ntree_limit)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c17d05438b30e2d75ccc97e34c14a19a62193a21"},"cell_type":"code","source":"def save_results(submit, y_hat, name, threshold=0.35):\n    print('threshold is: ', threshold)\n    results = (y_hat > threshold).astype(int)\n    print(results[:100])\n    submit['prediction'] = results\n    save_to = (name+'.csv')\n    submit.to_csv(save_to, index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b369ce3234fc775f16da7a947f4deb655dbaf768"},"cell_type":"code","source":"print('save results')\nsubmission = pd.read_csv('../input/sample_submission.csv')\nsave_results(submission, predictions, 'submission', threshold=best_threshold)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}