{"cells":[{"metadata":{"trusted":false},"cell_type":"code","source":"import os\nimport re\n\nfrom scipy.sparse import csr_matrix\nfrom scipy.sparse import hstack\nimport numpy as np\nimport pandas\n\nfrom sklearn.svm import SVC\nfrom sklearn.model_selection import KFold\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.model_selection import train_test_split\n\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import precision_score\nfrom sklearn.metrics import recall_score\nfrom sklearn.metrics import f1_score\n\nimport nltk\nstemmer = nltk.PorterStemmer()\n\nfrom collections import Counter","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def fetch_data(data_path='../input'):\n    \n    data_train = pandas.read_csv(os.path.join(data_path, 'train.csv'))\n    data_test = pandas.read_csv(os.path.join(data_path, 'test.csv'))\n    \n    X_train = data_train.drop(['qid', 'target'], axis=1).values\n    y_train = data_train['target'].values\n    X_test = data_test.drop(['qid'], axis=1).values\n    \n    return (X_train, y_train, X_test, data_train, data_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"(X_train, y_train, X_test, \\\n            data_train, data_test) = fetch_data()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"data_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"class MessageToWordCounterTransform(BaseEstimator, TransformerMixin):\n    def __init__(self, strip_headers=True, lower_case=True, remove_punctuation=True,\n                 replace_urls=True, replace_numbers=True, stemming=True):\n        self.strip_headers = strip_headers\n        self.lower_case = lower_case\n        self.remove_punctuation = remove_punctuation\n        self.replace_urls = replace_urls\n        self.replace_numbers = replace_numbers\n        self.stemming = stemming\n    \n    def fit(self, X, y=None):\n        return self\n    def transform(self, X, y=None):\n        \n        print('[INFO] Transforming Messages to Word Count')\n        \n        X_transformed = []\n        for message in X:\n            text = message[0] or \"\"\n            if self.lower_case:\n                text = text.lower()\n            if self.replace_numbers:\n                text = re.sub(r'\\d+(?:\\.\\d*(?:[eE]\\d+))?', 'NUMBER', text)\n            if self.remove_punctuation:\n                text = re.sub(r'\\W+', ' ', text, flags=re.M)\n            word_counts = Counter(text.split())\n            if self.stemming and stemmer is not None:\n                stemmed_word_counts = Counter()\n                for word, count in word_counts.items():\n                    stemmed_word = stemmer.stem(word)\n                    stemmed_word_counts[stemmed_word] += count\n                word_counts = stemmed_word_counts\n            X_transformed.append(word_counts)\n        return np.array(X_transformed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"class WordCounterToVectorTransformer(BaseEstimator, TransformerMixin):\n    def __init__(self, vocabulary_size=1000):\n        self.vocabulary_size = vocabulary_size\n    def fit(self, X, y=None):\n        total_count = Counter()\n        for word_count in X:\n            for word, count in word_count.items():\n                total_count[word] += min(count, 10)\n        most_common = total_count.most_common()[:self.vocabulary_size]\n        self.most_common_ = most_common\n        self.vocabulary_ = {word: index + 1 for index, (word, count) in enumerate(most_common)}\n        return self\n    def transform(self, X, y=None):\n        \n        print('[INFO] Transforming Word Counter to Word Vector Transformer')\n        \n        rows = []\n        cols = []\n        data = []\n        for row, word_count in enumerate(X):\n            for word, count in word_count.items():\n                rows.append(row)\n                cols.append(self.vocabulary_.get(word, 0))\n                data.append(count)\n        return csr_matrix((data, (rows, cols)), shape=(len(X), self.vocabulary_size + 1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"preprocess_pipeline = Pipeline([\n    (\"email_to_wordcount\", MessageToWordCounterTransform()),\n    (\"wordcount_to_vector\", WordCounterToVectorTransformer()),\n])\n\nX_train_transformed = preprocess_pipeline.fit_transform(X_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import cross_val_score\n\nkf = KFold(n_splits=5, shuffle=True, random_state=42)\n\nlog_clf = LogisticRegression(solver=\"liblinear\", random_state=42)\nscore = cross_val_score(log_clf, X_train_transformed, y_train, cv=kf, verbose=3)\nscore.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"X_test_transformed = preprocess_pipeline.transform(X_test)\n\nlog_clf = LogisticRegression(solver=\"liblinear\", random_state=42)\nlog_clf.fit(X_train_transformed, y_train)\n\ny_pred = log_clf.predict(X_test_transformed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"data_test['prediction'] = y_pred\ndata_test = data_test.drop('question_text', axis=1)\ndata_test.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}