{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Meeting 3 - Finishing the Competition [Baseline]\n\nWelcome to meeting 3. This time we'll finish and wrap up the competition (Quora that is).\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \nimport regex as re\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport spacy\nimport unicodedata\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.base import BaseEstimator, ClassifierMixin\nfrom sklearn.utils.validation import check_X_y, check_is_fitted\nfrom sklearn import metrics\nfrom sklearn.svm import LinearSVC\nfrom tqdm import tqdm, tqdm_notebook\nimport operator\ntqdm(tqdm_notebook).pandas()\n#tqdm.pandas()\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\ndatapath='../input'\nRANDOM_STATE = 2\nSHUFFLE = True\nTEST_SIZE = 0.8\n# Any results you write to the current directory are saved as output.","execution_count":1,"outputs":[{"output_type":"stream","text":"\r0it [00:00, ?it/s]","name":"stderr"},{"output_type":"stream","text":"['test.csv', 'train.csv', 'sample_submission.csv', 'embeddings']\n","name":"stdout"},{"output_type":"stream","text":"\n","name":"stderr"}]},{"metadata":{"_kg_hide-input":false,"trusted":true},"cell_type":"code","source":"import time\nfrom contextlib import contextmanager\n\n\n@contextmanager\ndef timer(name):\n    \"\"\"\n    Taken from Konstantin Lopuhin https://www.kaggle.com/lopuhin\n    in script named : Mercari Golf: 0.3875 CV in 75 LOC, 1900 s\n    https://www.kaggle.com/lopuhin/mercari-golf-0-3875-cv-in-75-loc-1900-s\n    \"\"\"\n    t0 = time.time()\n    yield\n    print(f'[{name}] done in {time.time() - t0:.0f} s')\n\n\ndef load_trained_model(model, weights_path):\n    model.load_weights(weights_path)\n    return model","execution_count":2,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Data reader - An abstraction to data reading\nI've built an simple data reader class that'll help us to read data & we'll supply a transformer to transform & preprocess data within this\n\nUsage:\n```python\ndr = DataReader(train_file_path, module, test_file_path=None)\nsplit = dr.get_split(split=..)\nor\nsplit_generator = dr.get_kfold(k=..)\nsplit_K_0 = next(split_generator)\n```"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"class DataReader(object):\n    def __init__(self,\n                 train_file,\n                 module,\n                 test_file=None):\n\n        if not train_file:\n            raise Exception(\"DataReader requires a train_file!\")\n        if not module:\n            raise Exception(\"DataReader requires a model that can transform data!\")\n        self.raw_test = None\n        if test_file:\n            print(\"Loading test_data (%s) into dataframe\" % test_file)\n            self.test_data = pd.read_csv(test_file)\n            self.raw_test = self.test_data[['question_text']]\n            print(\"Test data with shape: \", self.test_data.shape)\n\n        print(\"Loading train_data (%s) into dataframes\" % train_file)\n        self.train_data = pd.read_csv(train_file)\n        self.raw_train = self.train_data[['question_text']]\n        print(\"Train data with shape: \", self.train_data.shape)\n        train_test_cut = self.train_data.shape[0]\n        if isinstance(self.raw_test, pd.DataFrame):\n            df_all = pd.concat([self.raw_train, self.raw_test],\n                               axis=0).reset_index(drop=True)\n        else:\n            df_all = self.raw_train\n        self.df_all = df_all\n\n\n        print(\"Transforming the data\")\n        with timer('Transforming data'):\n            if module:\n                X_features = module.transform(df_all['question_text'])\n            else:\n                X_features = df_all['question_text']\n            # Multiple Inputs\n            if isinstance(X_features, list):\n                self.X_train = [X[:train_test_cut] for X in X_features]\n                self.X_test = [X[train_test_cut:] for X in X_features]\n            else:\n                self.X_train = X_features[:train_test_cut]\n                self.X_test = X_features[train_test_cut:]\n\n    def get_split(self, split=0.8, random_state=2, shuffle_data=True):\n        \"\"\"\n        :param split: float - % to be training data\n        :param random_state: int - init_state for random to keep random stale\n        :param shuffle_data: if to shuffle\n        :return: X_t, X_v, y_t, y_v where X = training and Y = validation.\n        t = training data & v = class\n        \"\"\"\n        print(\"Creating validation data by splitting (%s)\" % split)\n        train_data = self.train_data\n        X_train = self.X_train\n\n        X_t, X_v, y_t, y_v = train_test_split(\n            X_train, train_data.target,\n            test_size=(1 - split), random_state=random_state,\n            shuffle=shuffle_data, stratify=train_data.target)\n\n        return X_t, X_v, y_t, y_v\n\n    def get_kfold(self, k=5, shuffle_data=True, random_state=2):\n        \"\"\"\n        :param k: int - Number of folds.\n        :param shuffle_data: boolean - If we should shuffle\n        :param random_state: int - init_state for random to keep random stale\n        :return: a generator that yields the folds.\n        \"\"\"\n        print(\"Creating validation data by kfold (%s)\" % k)\n        kfold = StratifiedKFold(n_splits=k, shuffle=shuffle_data, random_state=random_state)\n        train_data = self.train_data\n        X_train = self.X_train\n        folded_data = kfold.split(X_train, train_data.target)\n\n        for i in range(k):\n            fold = next(folded_data)\n            X_t = X_train.iloc[fold[0]]\n            X_v = train_data.iloc[fold[0]]\n            y_t = X_train.iloc[fold[1]]\n            y_v = train_data.iloc[fold[1]]\n\n            yield X_t, X_v, y_t, y_v\n\n    def get_test(self):\n        if isinstance(self.test_data, pd.DataFrame):\n            return self.train_data, self.X_train, self.test_data, self.X_test\n        raise Exception(\"No test data provided!\")\n\n    def get_all_text(self):\n        return self.df_all['question_text']","execution_count":3,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Preprocessing - A tool to preprocess your data\n\nAs always, we need to preprocess our data.  \nWhy you might ask?  \n... Explain!!"},{"metadata":{"_kg_hide-input":false,"trusted":true},"cell_type":"code","source":"class PreProcessor(object):\n    def __init__(self, text):\n        self.text = text\n        self.puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$',\n                       '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',\n                       '~', '@', '£', '·', '_', '{', '}', '©', '^', '®', '`', '<',\n                       '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â',\n                       '█', '½', 'à', '…', '“', '★', '”', '–', '●', 'â', '►', '−', '¢',\n                       '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥',\n                       '▓', '—', '‹', '─', '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’',\n                       '▀', '¨', '▄', '♫', '☆', 'é', '¯',\n                       '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',\n                       '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³',\n                       '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√']\n        # TODO this varies depending on what task!\n        self.mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling',\n                             'counselling': 'counseling',\n                             'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor',\n                             'organisation': 'organization',\n                             'wwii': 'world war 2',\n                             'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary',\n                             'Whta': 'What',\n                             'narcisist': 'narcissist',\n                             'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much',\n                             'howmany': 'how many', 'whydo': 'why do',\n                             'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does',\n                             'mastrubation': 'masturbation',\n                             'mastrubate': 'masturbate',\n                             \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum',\n                             'narcissit': 'narcissist',\n                             'bigdata': 'big data',\n                             '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend',\n                             'airhostess': 'air hostess', \"whst\": 'what',\n                             'watsapp': 'whatsapp', 'demonitisation': 'demonetization',\n                             'demonitization': 'demonetization',\n                             'demonetisation': 'demonetization'}\n        self.mispellings_re = re.compile('(%s)' % '|'.join(self.mispell_dict.keys()))\n\n    def get_text(self):\n        return self.text\n\n    # TODO fix misspellings\n    def replace_typical_misspell(self):\n        def replace(match):\n            return self.mispell_dict[match.group(0)]\n\n        self.text = self.mispellings_re.sub(replace, self.text)\n\n        return self\n\n    def spacy_tokenize_words(self):\n        raise NotImplementedError\n\n    def normalize_unicode(self):\n        self.text = unicodedata.normalize('NFKD', self.text)\n        return self\n\n    def remove_newline(self):\n        \"\"\"\n        remove \\n and  \\t\n        \"\"\"\n        self.text = ' '.join(self.text.split())\n        return self\n\n    def decontracted(self):\n        # specific\n        text = re.sub(r\"(W|w)on(\\'|\\’)t\", \"will not\", self.text)\n        text = re.sub(r\"(C|c)an(\\'|\\’)t\", \"can not\", text)\n        text = re.sub(r\"(Y|y)(\\'|\\’)all\", \"you all\", text)\n        text = re.sub(r\"(Y|y)a(\\'|\\’)ll\", \"you all\", text)\n\n        # general\n        text = re.sub(r\"(I|i)(\\'|\\’)m\", \"i am\", text)\n        text = re.sub(r\"(A|a)in(\\'|\\’)t\", \"aint\", text)\n        text = re.sub(r\"n(\\'|\\’)t\", \" not\", text)\n        text = re.sub(r\"(\\'|\\’)re\", \" are\", text)\n        text = re.sub(r\"(\\'|\\’)s\", \" is\", text)\n        text = re.sub(r\"(\\'|\\’)d\", \" would\", text)\n        text = re.sub(r\"(\\'|\\’)ll\", \" will\", text)\n        text = re.sub(r\"(\\'|\\’)t\", \" not\", text)\n        self.text = re.sub(r\"(\\'|\\’)ve\", \" have\", text)\n\n        return self\n\n    def space_punctuation(self):\n        for punct in self.puncts:\n            if punct in self.text:\n                self.text = self.text.replace(punct, f' {punct} ')\n\n                # We could also remove all non p\\{L}...\n\n        return self\n\n    def remove_punctuation(self):\n        import string\n        re_tok = re.compile(f'([{string.punctuation}])')\n        self.text = re_tok.sub(' ', self.text)\n\n        return self\n\n    def clean_numbers(self):\n        text = self.text\n        if bool(re.search(r'\\d', text)):\n            text = re.sub('[0-9]{5,}', '#####', text)\n            text = re.sub('[0-9]{4}', '####', text)\n            text = re.sub('[0-9]{3}', '###', text)\n            text = re.sub('[0-9]{2}', '##', text)\n        self.text = text\n        return self\n\n    def clean_and_get_text(self):\n        self.clean_numbers() \\\n            .space_punctuation() \\\n            .decontracted() \\\n            .normalize_unicode() \\\n            .remove_newline() \\\n            .replace_typical_misspell()\n\n        return self.text","execution_count":4,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Base Classifier (Support Vector Machine) by Sklearn\n\nHere we have a basic implementation of a classifier which can find it's own best learning rate (by actually running different ones and thereafter report the best param).  \n\nBecause of how the whole environment is built locally two methods are required to be included in the Classifiers file\n1. `transform` - transform & preprocess your data somehow\n2. `get_model` - return the model class initiated, this model has to have methods such as fit & train"},{"metadata":{"_kg_hide-input":false,"trusted":true},"cell_type":"code","source":"class BaseClassifier(BaseEstimator, ClassifierMixin):\n    def __init__(self, C=1.0):\n        self.C = C\n        self._best_C, self._best_score, self._clf = None, None, None\n\n    def predict(self, X):\n        # Verify that model has been fit\n        check_is_fitted(self, ['_clf'])\n        return self._clf.predict(X)\n\n    def fit(self, X, y):\n        X, y = check_X_y(X, y, accept_sparse=True)\n\n        self._clf = LinearSVC(C=self.C).fit(X, y)\n        return self\n\n    def train(self, X_train, y_train, X_val, y_val, Cs=None):\n        \"\"\"\n        trainer to score auc over a grid of Cs\n        Parameters\n        ----------\n        X_train, y_train, X_val, y_val: features and targets\n        Cs: list of floats | int\n        Return\n        ------\n        self\n        \"\"\"\n        # init grid\n        origin_C = self.C\n        if Cs is None:\n            Cs = [0.01, 0.1, 0.5, 1, 2, 10]\n        # score\n        scores = {}\n        f1 = {}\n        for C in Cs:\n            # fit\n            self.C = C\n            model = self.fit(X_train, y_train)\n            # predict\n            y_pred = model.predict(X_val)\n            scores[C] = metrics.roc_auc_score(y_val, y_pred)\n            f1[C] = metrics.f1_score(y_val, y_pred)\n            print(\"Val AUC Score: {:.4f}, F1: {:.4f} with C = {}\".format(scores[C], f1[C], C))  # noqa\n\n        # get max\n        self._best_C, self._best_score = max(f1.items(), key=operator.itemgetter(1))  # noqa\n        # reset\n        self.C = origin_C\n        return self\n\n    @property\n    def best_param(self):\n        check_is_fitted(self, ['_clf'])\n        return self._best_C\n\n    @property\n    def best_score(self):\n        check_is_fitted(self, ['_clf'])\n        return self._best_score\n\ndef transform(df_text):\n    df_text.progress_apply(clean_text)\n    vectorizer = TfidfVectorizer(ngram_range=(1, 2),\n                                 strip_accents='ascii')\n    return vectorizer.fit_transform(list(df_text))\n\n\ndef get_model():\n    return BaseClassifier(2)\n\n\ndef clean_text(text):\n    return PreProcessor(text).clean_and_get_text()","execution_count":5,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Trainer - How to train & evaluate your system on train/validatio data\n\nWith Trainer we can train & validate how our classifier is performing.  \nLocally it's simple & in this Notebook it's even simpler!"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_and_eval(X_train, y_train, X_val, y_val):\n    \"\"\"\n    Parameters\n    ----------\n    X_train, y_train, X_val, y_val: features and targets\n    \n    Return\n    ------\n    training logs\n    \"\"\"\n    model = get_model()\n    print('Training model...')\n    model = model.train(X_train, y_train, X_val, y_val)\n    best_param = model.best_param\n    best_score = model.best_score\n    print(\"Best param: {:.4f} with best score: {}\".format(best_param, best_score))\n    return pd.DataFrame({'best_param': [best_param], 'best_score': [best_score]})\n\nt0 = time.time()\nclass fakemodule(object):\n    @staticmethod\n    def transform(a):\n        return transform(a)\ndr = DataReader('%s/train.csv' % datapath, fakemodule, os.path.join(datapath, 'test.csv'))\n\nwith timer(\"Load and Preprocess\"):\n    X_t, X_v, y_t, y_v = dr.get_split(TEST_SIZE)\n\nwith timer('Training and Tuning'):\n    #df_score = train_and_eval(X_t, y_t, X_v, y_v)\n    filepath = os.path.join(datapath, 'trainer_baseline.csv')\n    # df_score.to_csv(filepath)\n    print('Save CV score file to {}'.format(filepath))\n\nprint('Entire program is done and it took {:.2f}s'.format(time.time() - t0))","execution_count":6,"outputs":[{"output_type":"stream","text":"Loading test_data (../input/test.csv) into dataframe\nTest data with shape:  (375806, 2)\nLoading train_data (../input/train.csv) into dataframes\n","name":"stdout"},{"output_type":"stream","text":"\r  0%|          | 0/1681928 [00:00<?, ?it/s]","name":"stderr"},{"output_type":"stream","text":"Train data with shape:  (1306122, 3)\nTransforming the data\n","name":"stdout"},{"output_type":"stream","text":"100%|██████████| 1681928/1681928 [02:26<00:00, 11515.96it/s]\n","name":"stderr"},{"output_type":"stream","text":"[Transforming data] done in 269 s\nCreating validation data by splitting (0.8)\n[Load and Preprocess] done in 3 s\nSave CV score file to ../input/trainer_baseline.csv\n[Training and Tuning] done in 0 s\nEntire program is done and it took 277.27s\n","name":"stdout"}]},{"metadata":{},"cell_type":"markdown","source":"### Submission time"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_submission(X_train, y_train, X_test, df_test):\n    \"\"\"\n    train model with entire training data, predict test data,\n    and create submission file\n\n    Parameters\n    ----------\n    X_train, y_train, X_test: features and targets\n    df_test: dataframe, test data\n    module: a python module\n\n    Return\n    ------\n    df_summission\n    \"\"\"\n    model = get_model()\n    print('Training model...')\n    model = model.fit(X_train, y_train)\n    # predict\n    print('Predicting test...')\n    #y_pred = np.squeeze(model.predict_proba(X_test) > thres).astype('int')\n    y_pred = model.predict(X_test)\n    # create submission file\n    return pd.DataFrame({'qid': df_test.qid, 'prediction': y_pred})\n\nt0 = time.time()\n\nwith timer(\"Load and Preprocess\"):\n    # Only init if didn't run training.\n    # dr = DataReader(os.path.join(datapath, 'quora', 'train.csv'), fakemodule, os.path.join(datapath, 'quora', 'test.csv'))\n    df_train, X_train, df_test, X_test = dr.get_test()\n# 3. create submission file\nwith timer('Trainning and Creating Submission'):\n    filepath = os.path.join('submission.csv')\n    df_submission = create_submission(\n        X_train, df_train.target,\n        X_test, df_test)\n    df_submission.to_csv(filepath, index=False)\n    print('Save submission file to {}'.format(filepath))\n\nprint('Entire program is done and it took {:.2f}s'.format(time.time() - t0))","execution_count":10,"outputs":[{"output_type":"stream","text":"[Load and Preprocess] done in 0 s\nTraining model...\nPredicting test...\nSave submission file to submit.csv\n[Trainning and Creating Submission] done in 31 s\nEntire program is done and it took 31.48s\n","name":"stdout"}]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}