{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(os.listdir(\"/kaggle/input/quora-insincere-questions-classification/\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!unzip /kaggle/input/quora-insincere-questions-classification/embeddings.zip ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(os.listdir(\"/kaggle/working/\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !pip install contractions","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"31324d0c-34d6-4804-9720-680076da1800","_cell_guid":"7d0a5e27-a54e-4ae6-ae24-61a2d7b6d4f5","trusted":true},"cell_type":"code","source":"from __future__ import absolute_import\n\nimport re\nimport logging\nimport itertools\nimport unicodedata\n# import contractions\n\nfrom bs4 import BeautifulSoup\n\nclass TextCleaningUtils:\n    '''\n        This class contains implementations of various text cleaning operations (Static Methods)\n    '''\n\n\n    cleaning_regex_map = {\n        'web_links': r'(?i)(?:(?:http(?:s)?:)|(?:www\\.))\\S+',\n        'special_chars': r'[^a-zA-Z0-9\\s\\.,!?;:]+',\n        'redundant_spaces': r'\\s\\s+',\n        'redundant_newlines': r'[\\r|\\n|\\r\\n]+',\n        'twitter_handles': r'[#@]\\S+',\n        'punctuations': r'[\\.,!?;:]+'\n    }\n\n    @staticmethod\n    def clean_text_from_regex(text, text_clean_regex):\n        '''\n            Follow a particular cleaning expression, provided\n            as an input by an user to clean the text.\n        '''\n\n        text = text_clean_regex.sub(' ', text).strip()\n        return text\n\n    @staticmethod\n#     def replace_contractions(text):\n#         '''\n#             Replace contractions in string of text\n#         '''\n\n#         return contractions.fix(text)\n\n    @staticmethod\n    def strip_html(text):\n        soup = BeautifulSoup(text, \"html.parser\")\n        return soup.get_text()\n\n    @staticmethod\n    def remove_special_chars(text):\n        '''\n            Replace any special character provided as default,\n            which is present in the text with space\n        '''\n\n        special_chars_regex = re.compile(TextCleaningUtils.cleaning_regex_map['special_chars'])\n        text = TextCleaningUtils.clean_text_from_regex(text, special_chars_regex)\n        return text\n\n    @staticmethod\n    def remove_redundant_spaces(text):\n        '''\n            Remove any redundant space provided as default,\n            that is present in the text.\n        '''\n\n        redundant_spaces_regex = re.compile(\n            TextCleaningUtils.cleaning_regex_map['redundant_spaces'])\n        text = TextCleaningUtils.clean_text_from_regex(text, redundant_spaces_regex)\n        return text\n\n    @staticmethod\n    def remove_web_links(text):\n        '''\n            Removes any web link that follows a particular default expression,\n            present in the text.\n        '''\n\n        web_links_regex = re.compile(TextCleaningUtils.cleaning_regex_map['web_links'])\n        text = TextCleaningUtils.clean_text_from_regex(text, web_links_regex)\n        return text\n\n    @staticmethod\n    def remove_twitter_handles(text):\n        '''\n            Removes any twitter handle present in the text.\n        '''\n\n        twitter_handles_regex = re.compile(TextCleaningUtils.cleaning_regex_map['twitter_handles'])\n        text = TextCleaningUtils.clean_text_from_regex(text, twitter_handles_regex)\n        return text\n\n    @staticmethod\n    def remove_redundant_newlines(text):\n        '''\n            Removes any redundant new line present in the text.\n        '''\n\n        redundant_newlines_regex = re.compile(\n            TextCleaningUtils.cleaning_regex_map['redundant_newlines'])\n        text = TextCleaningUtils.clean_text_from_regex(text, redundant_newlines_regex)\n        return text\n\n    @staticmethod\n    def remove_punctuations(text):\n        '''\n            Removes any punctuation that follows the default expression, in the text.\n        '''\n\n        remove_punctuations_regex = re.compile(TextCleaningUtils.cleaning_regex_map['punctuations'])\n        text = TextCleaningUtils.clean_text_from_regex(text, remove_punctuations_regex)\n        return text\n\n    @staticmethod\n    def remove_exaggerated_words(text):\n        '''\n            Removes any exaggerated word present in the text.\n        '''\n\n        return ''.join(''.join(s)[:2] for _, s in itertools.groupby(text))\n\n    @staticmethod\n    def replace_multiple_chars(text):\n        '''\n            Replaces multiple characters present in the text.\n        '''\n\n        char_list = ['.', '?', '!', '#', '$', '/', '@', '*', '(', ')', '+']\n        final_text = ''\n        for i in char_list:\n            if i in text:\n                pattern = \"\\\\\" + i + '{2,}'\n                repl_str = i.replace(\"\\\\\", \"\")\n                text = re.sub(pattern, repl_str, text)\n                final_text = ' '.join(text.split())\n        return final_text\n\n    @staticmethod\n    def replace_sign(text):\n        '''\n            Replaces any sign with words like & with 'and', in the text.\n        '''\n        sign_list = {'&': ' and ', '/': ' or ', '\\xa0': ' '}\n        final_text = ''\n        for i in sign_list:\n            if i in text:\n                text = re.sub(i, sign_list[i], text)\n                final_text = ' '.join(text.split())\n        return final_text\n\n    @staticmethod\n    def remove_accented_char(text):\n        text = unicodedata.normalize('NFD', text) \\\n            .encode('ascii', 'ignore') \\\n            .decode(\"utf-8\")\n        return str(text)\n\n    @staticmethod\n    def replace_characters(text, replace_map):\n        '''\n            Replaces any character custom provided by an user.\n        '''\n\n        for char, replace_val in replace_map.items():\n            text = text.replace(char, replace_val)\n        return text\n\n\n# class TextCleaningRecipes:\n#     \"\"\"\n#         This class contains the recipes for a set of standard text cleaning operations\n\n#     \"\"\"\n\n#     DEFAULT_OPERATIONS = ['replace_contractions', 'remove_web_links', 'remove_special_chars',\n#                           'remove_redundant_newlines', 'remove_redundant_spaces']\n\n#     OPERATIONS_MAP = {\n#         'replace_contractions': TextCleaningUtils.replace_contractions,\n#         'remove_web_links': TextCleaningUtils.remove_web_links,\n#         'remove_twitter_handles': TextCleaningUtils.remove_twitter_handles,\n#         'replace_characters': TextCleaningUtils.replace_characters,\n#         'remove_special_chars': TextCleaningUtils.remove_special_chars,\n#         'remove_punctuations': TextCleaningUtils.remove_punctuations,\n#         'remove_redundant_newlines': TextCleaningUtils.remove_redundant_newlines,\n#         'remove_redundant_spaces': TextCleaningUtils.remove_redundant_spaces\n#     }\n\n#     OPERATIONS_ORDER = ['replace_contractions', 'remove_web_links', 'remove_twitter_handles',\n#                         'replace_characters',\n#                         'remove_special_chars', 'remove_punctuations',\n#                         'remove_redundant_newlines', 'remove_redundant_spaces']\n\n#     @staticmethod\n#     def exec_cleaning(text_values, config):\n#         '''\n#             This method executes various cleaning techniques together\n#         '''\n\n#         operations = TextCleaningRecipes.get_operations(config)\n#         logging.info(\"Executing %s Operations\", ', '.join(operations))\n\n#         cleaning_ops = []\n#         for _op in operations:\n#             op_func = TextCleaningRecipes.OPERATIONS_MAP[_op]\n#             cleaning_ops.append(op_func)\n\n#         c_text_values = []\n#         for text in text_values:\n\n#             if text is None:\n#                 c_text = ''\n#             else:\n#                 c_text = str(text)\n\n#             c_text = c_text.replace(')', ') ')\n\n#             for _op in cleaning_ops:\n#                 if 'replace_characters' in _op.__name__:\n\n#                     c_text = _op(c_text, config['replace_characters'])\n#                 else:\n#                     c_text = _op(c_text)\n\n#             c_text_values.append(c_text)\n\n#         return c_text_values\n\n#     @staticmethod\n#     def get_operations(config):\n\n#         operations = []\n#         if len(config) == 0:\n#             operations = TextCleaningRecipes.DEFAULT_OPERATIONS\n#             return operations\n\n#         for _op in TextCleaningRecipes.OPERATIONS_ORDER:\n#             if _op in config and config[_op]:\n#                 operations.append(_op)\n\n#         if not operations:\n#             operations = TextCleaningRecipes.DEFAULT_OPERATIONS\n\n#         logging.info(\"Operations: %s\", operations)\n#         return operations","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\n\n# fastext\nimport fasttext\n\nfrom sklearn.model_selection import train_test_split\n\nfrom tabulate import tabulate\n\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import classification_report","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.target.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_data(df,col_to_clean):\n#   df[col_to_clean] = df[col_to_clean].apply(TextCleaningUtils.replace_contractions)\n#   Remove Smiles and special chars\n#   df[col_to_clean] = df[col_to_clean].apply(TextCleaningUtils.transform_emojis)\n  df[col_to_clean] = df[col_to_clean].apply(TextCleaningUtils.remove_special_chars)\n  df[col_to_clean] = df[col_to_clean].apply(TextCleaningUtils.remove_redundant_spaces)\n  df[col_to_clean] = df[col_to_clean].apply(TextCleaningUtils.remove_punctuations)\n  df[col_to_clean] = df[col_to_clean].apply(TextCleaningUtils.remove_exaggerated_words)\n  df[col_to_clean] = df[col_to_clean].apply(TextCleaningUtils.remove_redundant_newlines)\n  df[col_to_clean] = df[col_to_clean].apply(TextCleaningUtils.remove_twitter_handles)\n  df[col_to_clean] = df[col_to_clean].apply(TextCleaningUtils.remove_web_links)\n#   df[col_to_clean] = df[col_to_clean].apply(TextCleaningUtils.replace_sign)\n  df[col_to_clean] = df[col_to_clean].astype(str)\n  df[col_to_clean] = df[col_to_clean].str.lower()\n  return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clean_df = clean_data(df,'question_text')\nsubmission_df = clean_data(submission_df,'question_text')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clean_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clean_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"map_={0:'__label__0',\n     1:'__label__1',\n     '0.0':'__label__0',\n     '1.0':'__label__1'}\nclean_df['label'] = clean_df['target'].map(map_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clean_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df, test_df = train_test_split(clean_df, random_state=50, stratify=clean_df['label'], test_size=0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def to_fwf(df, fname):\n    content = tabulate(df.values.tolist(), list(df.columns), tablefmt=\"plain\")\n    content = content[(content.find('\\n') + 1):]\n    open(fname, \"w\").write(content)\npd.DataFrame.to_fwf = to_fwf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df=train_df[['label','question_text']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"config={\n    'data_folder':'/kaggle/working/',\n    'training_csv_file':'/kaggle/input/quora-insincere-questions-classification/train.csv',\n    'training_fwf_file':'train_fwf.train',\n    'model_folder': '/kaggle/working/wiki-news-300d-1M/',\n    'pretrained_model': 'wiki-news-300d-1M.vec',\n    'model_version': 'm1'\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def to_fwf(df, fname):\n    content = tabulate(df.values.tolist(), list(df.columns), tablefmt=\"plain\")\n    content = content[(content.find('\\n') + 1):]\n    open(fname, \"w\").write(content)\npd.DataFrame.to_fwf = to_fwf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df=train_df[['label','question_text']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"config['data_folder']+config['training_fwf_file']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain_df.to_fwf(config['data_folder']+config['training_fwf_file'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_fn = os.path.join(config['data_folder'], config['training_fwf_file'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pretrainedvec_fn = os.path.join(config['model_folder'], config['pretrained_model'])\nmodel_fn = os.path.join(config['data_folder'], 'models','{}_{}.bin'.format('qoura_question',config['model_version']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('train_fn:{} \\n model_fn:{} \\n pretrainedvec_fn:{}'.format(train_fn,model_fn,pretrainedvec_fn))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nmodel = fasttext.train_supervised(input=train_fn,\n                                  pretrainedVectors=pretrainedvec_fn,\n                                  dim=300, \n                                  wordNgrams=2, \n                                  minCount=3, \n                                  epoch=20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !ls /kaggle/working/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_proba(x):\n    if x[1][0] + x[1][1] <1:\n        print(\"Not a classification\")\n    if x[0][0] == '__label__0':\n        return 1 - x[1][0] if x[1][0]<= 1 else 0\n    elif x[0][0] == '__label__1':\n        return x[1][0] if x[1][0]<= 1 else 1\n\ntrain_df['threshold'] = train_df['question_text'].apply(lambda x: get_proba(model.predict(x,k=2)))\ntest_df['threshold'] = test_df['question_text'].apply(lambda x: get_proba(model.predict(x,k=2)))\ntrain_df['debug_threshold'] = train_df['question_text'].apply(lambda x: model.predict(x,k=2))\ntest_df['debug_threshold'] = test_df['question_text'].apply(lambda x: model.predict(x,k=2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['target'] = train_df['label'].apply(lambda x: int(x.replace('__label__','')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['predict_threshold'] = train_df['threshold'].apply(lambda x: 0 if x <=0.06 else 1 )\ntest_df['predict_threshold'] = test_df['threshold'].apply(lambda x: 0 if x <=0.06 else 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"confusion_matrix(train_df['target'],train_df['predict_threshold'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"confusion_matrix(test_df['target'],test_df['predict_threshold'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(classification_report(train_df['target'],train_df['predict_threshold']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(classification_report(test_df['target'],test_df['predict_threshold']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# submission file","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_sample = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_sample.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df['threshold'] = submission_df['question_text'].apply(lambda x: get_proba(model.predict(x,k=2)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df['prediction'] = submission_df['threshold'].apply(lambda x: 0 if x <=0.06 else 1 )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 375806 - 315271-22660","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df['prediction'] = submission_df['prediction'].replace([np.inf, -np.inf, np.nan], 2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df['prediction'] = submission_df['prediction'].fillna(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df[submission_df['prediction'] == 2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df['debug_threshold'] = submission_df['question_text'].apply(lambda x: model.predict(x,k=2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df['prediction'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df['prediction'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df['prediction'] = submission_df['prediction'].astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df[submission_sample.columns]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df[submission_sample.columns].to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}