{"cells":[{"metadata":{"heading_collapsed":true,"_uuid":"ac3c8f094f0997395a2aeb58f656f7fe0c65c204"},"cell_type":"markdown","source":"# Import"},{"metadata":{"hidden":true,"trusted":false,"_uuid":"b8df0aae4b5754efd80d51cb3e98a65dc7cf261b"},"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport spacy\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nsns.set\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"heading_collapsed":true,"_uuid":"a510a0427bf7e6ba9bdca794947578d2ff03c9f0"},"cell_type":"markdown","source":"# Load the dataset"},{"metadata":{"hidden":true,"trusted":false,"_uuid":"f4c3a32ae35adf9b6b4ff21fb90ba802ea399e57"},"cell_type":"code","source":"path = '../input'\ndf_train = pd.read_csv(os.path.join(path, 'train.csv'))\ndf_test = pd.read_csv(os.path.join(path, 'test.csv'))","execution_count":null,"outputs":[]},{"metadata":{"heading_collapsed":true,"_uuid":"079973a2b3938460f9fd54a415bf6aff7c46e00b"},"cell_type":"markdown","source":"# Look at the dataset"},{"metadata":{"hidden":true,"scrolled":false,"trusted":false,"_uuid":"a924962fef971d97a84d91e4205b33d3b7acc148"},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"trusted":false,"_uuid":"813c9770addca3972b80e06f1487ce1e92dc4d9c"},"cell_type":"code","source":"df_train.shape, df_test.shape","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"hidden":true,"trusted":false,"_uuid":"b9f8ce020ddc457c9e4200f653b55af659bd6822"},"cell_type":"code","source":"df_train.info()","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"_uuid":"cd0f1ab0eb22ed74cd7c619dd3ffa64844f4b814"},"cell_type":"markdown","source":"The data is clean, there is no Naan values"},{"metadata":{"collapsed":true,"hidden":true,"trusted":false,"_uuid":"517fe237e341bd127a17de296d251af8cace992c"},"cell_type":"code","source":"df_train['target'].value_counts().plot(kind='bar');","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"trusted":false,"_uuid":"d04d3af34dda3157c97113e69eb96c90522f8c76"},"cell_type":"code","source":"insincere_ratio = (80810 / 1225312) * 100\ninsincere_ratio","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"trusted":false,"_uuid":"eb0617eb537f6580db1ea14144f593f0f6574315"},"cell_type":"code","source":"y = df_train['target']\nX = df_train['question_text']","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"hidden":true,"trusted":false,"_uuid":"df34458b9cbda0d87da06ff52ebe05bd8bbe2bed"},"cell_type":"code","source":"X_insincere = X[y == 1]\nX_insincere.head()","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"_uuid":"63c465aaa04973213e8104b2cc65c8b3050d5491"},"cell_type":"markdown","source":"We can already notice the troll content within the questions."},{"metadata":{"collapsed":true,"hidden":true,"trusted":false,"_uuid":"0bb7c9bb891e55fa790c96de6d1e15aeae612a67"},"cell_type":"code","source":"X_sincere = X[y == 0]\nX_sincere.head()","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"_uuid":"c34a8fcf3211b5d3c53f674810bd61ed59626bdd"},"cell_type":"markdown","source":"Whereas within the sincere question, the questions are legit."},{"metadata":{"_uuid":"73e0d647139f4c7f9e08f08ae675c82ef3f33e49"},"cell_type":"markdown","source":"# Preprocessing"},{"metadata":{"trusted":false,"_uuid":"cece32f31bf98b980526aa44acbc9d3904fcff61"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"cbcaef37ce1ff83d91d19b7a78acccf72aecd691"},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"4de3dc298c615c3e0bbdf497ba264d71737184de"},"cell_type":"code","source":"X_train.shape, X_test.shape, y_train.shape, y_test.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6eeb599a2afc37b6f47910c7131bc5a48721a018"},"cell_type":"markdown","source":"## Tokenizing"},{"metadata":{"_uuid":"6efba3ee2a148967a071499b4d2360ddc9f8fd4c"},"cell_type":"markdown","source":"Creation of a tokenize's function permitting to automatically tokenize our train and test set"},{"metadata":{"trusted":false,"_uuid":"fb392f059e098fe0528e4622b81c8f25f1950830"},"cell_type":"code","source":"from nltk.tokenize import word_tokenize\nfrom nltk.corpus import stopwords\nfrom nltk.stem import PorterStemmer","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"14dc703d3ad0856ec964e0e9f65bf544a169e6aa"},"cell_type":"code","source":"def tokenize(data):\n    corpus = [word_tokenize(token) for token in data]\n    lowercase_train = [[token.lower() for token in doc] for doc in corpus]\n    alphas = [[token for token in doc if token.isalpha()] for doc in lowercase_train]\n    stop_words = stopwords.words('english')\n    train_no_stop = [[token for token in doc if token not in stop_words] for doc in alphas]\n    stemmer = PorterStemmer()\n    stemmed = [[stemmer.stem(token) for token in doc] for doc in train_no_stop]\n    train_clean_str = [ ' '.join(doc) for doc in stemmed]\n    return train_clean_str","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e004c7e482b3983f3dc5b13b9569351ba9f283ab"},"cell_type":"code","source":"X_train = tokenize(X_train)\nX_test = tokenize(X_test)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ae94c43568cf2fbf163705d0c05a6691b1885bc6"},"cell_type":"markdown","source":"# Machine Learning"},{"metadata":{"heading_collapsed":true,"_uuid":"4e5e99e0e6422c7b2adba455c4f66732e002446d"},"cell_type":"markdown","source":"## Preprocessing"},{"metadata":{"hidden":true,"trusted":false,"_uuid":"22909fcf159d58539efe3f35214c22d38f9fe3ed"},"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\nfrom sklearn.decomposition import TruncatedSVD","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"trusted":false,"_uuid":"d27fe8be03cf40b39a0963883c2799c9f1e74cec"},"cell_type":"code","source":"tvec = TfidfVectorizer(stop_words='english')\ncvec = CountVectorizer(stop_words='english')\nsvd = TruncatedSVD(n_components=100, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"839eb702b8719f9f25321b25f559d556c7346060"},"cell_type":"markdown","source":"## Machine learning models"},{"metadata":{"_uuid":"90175e163dfd99d1c7abc1b931d2e4bab92baecc"},"cell_type":"markdown","source":"### MultinomialNB"},{"metadata":{"trusted":false,"_uuid":"e423d8bbc44841a099c38ce726d94b3b24adf461"},"cell_type":"code","source":"from sklearn.pipeline import Pipeline\nfrom sklearn.naive_bayes import MultinomialNB","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"8e77a232b9cb7aace5a15d032e092e1902b189e9"},"cell_type":"code","source":"mnb = MultinomialNB()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"68d3530500d113a38ffa51dedaad6771530e3cf1"},"cell_type":"code","source":"pipe = Pipeline([('vectorizer', cvec), ('mnb', mnb)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"ebda91627add597abfd3cceeaad4b7771071fdbf"},"cell_type":"code","source":"pipe.fit(X_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"3efc41d6513c2f34cdbeecf9e4fcf8bbeec65c0f"},"cell_type":"code","source":"y_pred = pipe.predict(X_test)\ny_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"bac9964a55c38abe40a5f028989fd2a27caa9c49"},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\ncm = confusion_matrix(y_test, y_pred)\ncm","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"a5367bf7b8315bbbd2f519898a0f4d8974f3bca7"},"cell_type":"code","source":"labels = ['sincere', 'unsincere']\ncm = pd.DataFrame(cm, columns=labels, index=labels)\ncm","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"28c1c148353e7284bcf89a4f1094305dfc697153"},"cell_type":"markdown","source":"## Validation"},{"metadata":{"trusted":false,"_uuid":"9395f02f5204e645d5e667a08d887fc42e5e5d74"},"cell_type":"code","source":"from sklearn.model_selection import cross_val_score\ncv = cross_val_score(pipe, X_test, y_test, scoring='f1_macro', cv=5).mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"313d9c5e8315ca98b193ae36c6887d05ebb648f4"},"cell_type":"code","source":"cv","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"c7f0bac4153b2d7b63923f37f76192803225defe"},"cell_type":"code","source":"from sklearn.metrics import classification_report\ncr = classification_report(y_test, y_pred)\nprint(cr)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"10507672f4988576dbfec68cd2fd1639d0ad4c6a"},"cell_type":"markdown","source":"# Test"},{"metadata":{"trusted":false,"_uuid":"9ad2c2eda57c7a8e0d11e260137de07edd963c7e"},"cell_type":"code","source":"test = df_test['question_text']\ntest = tokenize(test)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e3cddd39abc650757ddfcc504fbdbccf1da24fba"},"cell_type":"markdown","source":"## Machine Learning"},{"metadata":{"trusted":false,"_uuid":"5ad5ed73231256620ef31419eed835c53d245387"},"cell_type":"code","source":"y_pred = pipe.predict(test)\ny_pred","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9605e0c8ac5d0dbcb672d6e74c1c8241bd2ee734"},"cell_type":"markdown","source":"# Submit on the sample submission"},{"metadata":{"trusted":false,"_uuid":"0b61e881ff48a435d1e4f39b49313133cf6aebbe"},"cell_type":"code","source":"path = '../input'\ndf_sub = pd.read_csv(os.path.join(path, 'sample_submission.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"9fca90e74a2af988bf9ead59176c7a6ffc96aee3"},"cell_type":"code","source":"df_sub['prediction'] = y_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"cb596e1d9199a1a288663e6a6851c2313396f89f"},"cell_type":"code","source":"df_sub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.5"}},"nbformat":4,"nbformat_minor":1}