{"cells":[{"metadata":{"_uuid":"ddb510f1276a08c122568d5525eba524b24b604c"},"cell_type":"markdown","source":"### Thanks to this kernel:\n**https://www.kaggle.com/collinsjosh/xgboost-classifier**"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input/\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6d66334d6cf0f15fbcc3d06ce2a6ab0b90947570"},"cell_type":"markdown","source":"Import the data that we will use to learn from."},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_data = pd.read_csv('../input/train.csv', delimiter=',')\ntrain_data.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"63303d7c25f6db667c678296c4961e899cf15208"},"cell_type":"markdown","source":"I'm partitioning the training data into train and test sets just so I have something to test against without having to go to the online submission form of the competition to get a result.  I wish the whole result set was provided!"},{"metadata":{"trusted":true,"_uuid":"f3736a58dbe4edded6e8b8c09cb3c32928ea49f4"},"cell_type":"code","source":"X = train_data[['Id', 'ciphertext']]\ny = train_data['target']\nfrom sklearn.model_selection import train_test_split\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25)\n\n#I use these samples for testing the steps without waiting a long time\nsample_size = 1000\nX_sample = X_train.iloc[0:sample_size,0:2] #rows, columns\ny_sample = y_train.iloc[0:sample_size] #rows, columns","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6260de7976af69c22f3cebe669ba0d876d24ace6"},"cell_type":"markdown","source":"This is a tokenizer that will be used when transforming the message to a Bag of Words."},{"metadata":{"trusted":true,"_uuid":"5330f5fcd0459327194c950bc061581b6104b548"},"cell_type":"code","source":"from nltk.tokenize import word_tokenize\nfrom nltk.corpus import stopwords \nimport nltk\n\ndef Tokenizer(str_input):\n    str_input = str_input.lower()\n    words = word_tokenize(str_input)\n    #remove stopwords\n    stop_words = set(stopwords.words('english'))\n    words = [w for w in words if not w in stop_words]\n    #stem the words\n    porter_stemmer=nltk.PorterStemmer()\n    words = [porter_stemmer.stem(word) for word in words]\n    return words","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9f227e591d5872d2586cfc4544852c3374d16f33"},"cell_type":"markdown","source":"I'm switching to a pipeline.  It makes building multiple models with seperate data sets easier.  Also allows for Grid Search to work on hyper parameters."},{"metadata":{"trusted":true,"_uuid":"c3b56bf89f7bfdd0219efc348ad15250a5885140"},"cell_type":"code","source":"from sklearn.pipeline import Pipeline\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.decomposition import TruncatedSVD\nfrom xgboost import XGBClassifier\n\ntext_clf = Pipeline([\n    ('tfidf', TfidfVectorizer(tokenizer=Tokenizer, max_df=0.3, min_df=0.001, max_features=100000)),\n    #('svd',   TruncatedSVD(algorithm='randomized', n_components=500)),\n    ('clf',   XGBClassifier(objective='multi:softmax', n_estimators=500, num_class=20, learning_rate=0.075, colsample_bytree=0.7, subsample=0.8, eval_metric='merror')),\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2131e31a341d51a51156090735f5c2cd449664fd"},"cell_type":"code","source":"from sklearn.model_selection import GridSearchCV\n\nparameters = {\n    #'tfidf__max_df': (0.25, 0.5, 0.75),\n    #'tfidf__min_df': (0.001, 0.0025, 0.005),\n    #'tfidf__max_features': (50000, 100000, 150000),\n    #'tfidf__ngram_range': ((1, 1), (1, 2)),  # unigrams or bigrams\n    #'tfidf__use_idf': (True, False),\n    # 'tfidf__norm': ('l1', 'l2'),\n    #'svd__n_components': (250, 500, 750),\n    #'clf__n_estimators': (250, 500, 750),\n    'clf__max_depth': (4, 6, 8),\n    'clf__min_child_weight': (1, 5, 10),\n    #'clf__alpha': (0.00001, 0.000001),\n    #'clf__penalty': ('l2', 'elasticnet'),\n    #'clf__max_iter': (10, 50, 80),\n}\n\n#gs_clf = GridSearchCV(text_clf, parameters, cv=5, iid=False, n_jobs=-1)\n#gs_clf.fit(X_sample.message, y_sample)\n\n#print(\"Best score: %0.3f\" % gs_clf.best_score_)\n#print(\"Best parameters set:\")\n#best_parameters = gs_clf.best_estimator_.get_params()\n#for param_name in sorted(parameters.keys()):\n#    print(\"\\t%s: %r\" % (param_name, best_parameters[param_name]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5b8ef4d1c55c1f364081e6cdf1f14f933130ede9"},"cell_type":"code","source":"text_clf.fit(X_train.ciphertext, y_train)\npredictions = text_clf.predict(X_test.ciphertext)\nprint(\"The training predictions are ready\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"43ebd8932e82669244f01dc7683fd6a7a283bcbf"},"cell_type":"markdown","source":"Now we can build the input vectors for the classifier with the TFIDFVectorizer."},{"metadata":{"trusted":true,"_uuid":"ed1bd2e403142cba894969aa7cdd9d40338885dc"},"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer\nfrom nltk.tokenize import word_tokenize\n\nvectorizer = TfidfVectorizer(tokenizer=Tokenizer, min_df=0.001, max_df=0.3)\nX_tfidf = vectorizer.fit_transform(X.ciphertext)\n#vectorizer.get_feature_names()\nX_tfidf.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0a5786c676a854cbd1bd455fcb1b0b480373e9c3"},"cell_type":"markdown","source":"XGBoost won't accept the sparse matrix that comes from TFIDFVectorizer.  We will use the TruncatedSVD transformer to change the matrix into one that XGBoost can work with.  This is way complicated stuff."},{"metadata":{"trusted":true,"_uuid":"b34399cd8919d98144f247d10daa084d4344b421"},"cell_type":"code","source":"from sklearn.decomposition import TruncatedSVD\n\nsvd_transformer = TruncatedSVD(algorithm='randomized', n_components=300)\nX_svd = svd_transformer.fit_transform(X_tfidf)\nX_svd.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e9968ead341a21b8a4b4f83f36bf6ce577f2d8e4"},"cell_type":"code","source":"from xgboost import XGBClassifier\n\nxgb_classifier = XGBClassifier(max_depth=3, n_estimators=1000, learning_rate=0.075, colsample_bytree=0.7, subsample=0.8)\nxgb_classifier.fit(X_svd, y)\nprint(\"The model is ready.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1f45b8582d33a7aaf7caf2760c493c3bee5f4fe6"},"cell_type":"code","source":"X_test = pd.read_csv('../input/classifying-20-newsgroups-test/test.csv', delimiter=',')\nX_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4c5e55d26b10cb5a5d05fd63a846cc5e63edd6ec","scrolled":true},"cell_type":"code","source":"X_test_tfidf = vectorizer.transform(X_test.ciphertext)\nX_test_svd = svd_transformer.transform(X_test_tfidf)\n\nxgb_predictions = xgb_classifier.predict(X_test_svd)\npredictions = xgb_predictions\npredictions[0:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fad5586d7ad12a526a90aa379ffaaad49e2793bb"},"cell_type":"markdown","source":"Below are some metrics to measure interative tweaks to the model."},{"metadata":{"trusted":true,"_uuid":"7ea5e0ee1962f8ad4352adcb1b5eebc9f5c0e1e9","scrolled":false},"cell_type":"code","source":"from sklearn.metrics import accuracy_score, precision_score, classification_report, confusion_matrix\n\nprint(\"Accuracy:\", accuracy_score(y_test, predictions))\nprint(\"Precision:\", precision_score(y_test, predictions, average='weighted'))\nprint(classification_report(y_test, predictions))\nprint(confusion_matrix(y_test, predictions))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e7cdda002a822502945e4f73caf8bdcda061ce50"},"cell_type":"markdown","source":"Ready to package up the predictions and create the file to be submitted for scoring."},{"metadata":{"trusted":true,"_uuid":"2b7036150fb179fba9c5ca0ecbc7904ee326fc52"},"cell_type":"code","source":"output = X_test.copy()\noutput.insert(2, 'target', predictions)\noutput.to_csv('submission.csv', sep=',', columns=['id', 'topic'], index=False)\nprint(os.listdir(\"../working\"))\noutput.iloc[1000:5010, :]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0d68245921f462a90ed8ebdd83534ce52f3714f3"},"cell_type":"markdown","source":"This last block just gives a peek into the submission file to sanity check it."},{"metadata":{"trusted":true,"_uuid":"1397d508abfbd654f44aa5cdc418f8bd2fbfd216"},"cell_type":"code","source":"results = pd.read_csv('submission.csv', delimiter=',')\nresults.iloc[5000:5010, :]","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}