{"cells":[{"metadata":{"_uuid":"1be588f5414ded6fdb24ab23823ad5ade8cd6d5d"},"cell_type":"markdown","source":"# Feature Extraction\n\nPurpose is to add new features from text data to put into algorithm such as XGB and LGBM"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport re\n\npd.set_option('precision', 5)\npd.set_option('display.float_format', lambda x: '%.5f' % x)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":9,"outputs":[]},{"metadata":{"_uuid":"08e0db4a8b3fabefad38bafc0abf84940865ebe6","_cell_guid":"89a65825-2a6c-4bd3-815f-7a910dbbf134","trusted":true},"cell_type":"code","source":"tr = pd.read_csv('../input/train.csv')\nte = pd.read_csv('../input/test.csv')\nprint('train data shape is :', tr.shape)\nprint('test data shape is :', te.shape)","execution_count":2,"outputs":[]},{"metadata":{"_uuid":"094959c486491e29f1761316225c2eed0eac16e2","collapsed":true,"_cell_guid":"94b3355a-de55-474c-b764-5743883fb7e4","trusted":true},"cell_type":"code","source":"data = pd.concat([tr, te], axis=0)","execution_count":3,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eaecbc04555fd25bde8efeb6164dd9afa30ef5d2"},"cell_type":"code","source":"data.head()","execution_count":4,"outputs":[]},{"metadata":{"_uuid":"3f00383b930187dd0eee12f6f76d8af8f5826df2"},"cell_type":"markdown","source":"- Length of the comment - Assumption is the longer the description the more probability people will buy\n- Number of capitals - Captivating(a regex containing all uppercase letter ranges)\n- Proportion of capitals \n- Number of exclamation marks - Captivating\n- Number of question marks - Presentation concept in advertizing makes it more attractive?\n- Number of punctuation symbols \n- Number of symbols - Too many redundant text will disrupt interest or reader\n- Number of words - Shorter message means less informative?\n- Number of unique words\n- Proportion of unique words \n- Number of smiles - Looks authentic? LOL"},{"metadata":{"trusted":true,"_uuid":"31dba47acfc32550e086213cf3e311ecad0885b1"},"cell_type":"code","source":"texts = ['description', 'title']\nfor idx, feat in enumerate(texts):\n    data[feat] = data[feat].fillna(' ')\n    data['total_len_'+str(idx)] = data[feat].apply(len)\n    pLu = \"[A-Z\\u00C0-\\u00D6\\u00D8-\\u00DE\\u0100\\u0102\\u0104\\u0106\\u0108\\u010A\\u010C\\u010E\\u0110\\u0112\\u0114\\u0116\\u0118\\u011A\\u011C\\u011E\\u0120\\u0122\\u0124\\u0126\\u0128\\u012A\\u012C\\u012E\\u0130\\u0132\\u0134\\u0136\\u0139\\u013B\\u013D\\u013F\\u0141\\u0143\\u0145\\u0147\\u014A\\u014C\\u014E\\u0150\\u0152\\u0154\\u0156\\u0158\\u015A\\u015C\\u015E\\u0160\\u0162\\u0164\\u0166\\u0168\\u016A\\u016C\\u016E\\u0170\\u0172\\u0174\\u0176\\u0178\\u0179\\u017B\\u017D\\u0181\\u0182\\u0184\\u0186\\u0187\\u0189-\\u018B\\u018E-\\u0191\\u0193\\u0194\\u0196-\\u0198\\u019C\\u019D\\u019F\\u01A0\\u01A2\\u01A4\\u01A6\\u01A7\\u01A9\\u01AC\\u01AE\\u01AF\\u01B1-\\u01B3\\u01B5\\u01B7\\u01B8\\u01BC\\u01C4\\u01C7\\u01CA\\u01CD\\u01CF\\u01D1\\u01D3\\u01D5\\u01D7\\u01D9\\u01DB\\u01DE\\u01E0\\u01E2\\u01E4\\u01E6\\u01E8\\u01EA\\u01EC\\u01EE\\u01F1\\u01F4\\u01F6-\\u01F8\\u01FA\\u01FC\\u01FE\\u0200\\u0202\\u0204\\u0206\\u0208\\u020A\\u020C\\u020E\\u0210\\u0212\\u0214\\u0216\\u0218\\u021A\\u021C\\u021E\\u0220\\u0222\\u0224\\u0226\\u0228\\u022A\\u022C\\u022E\\u0230\\u0232\\u023A\\u023B\\u023D\\u023E\\u0241\\u0243-\\u0246\\u0248\\u024A\\u024C\\u024E\\u0370\\u0372\\u0376\\u037F\\u0386\\u0388-\\u038A\\u038C\\u038E\\u038F\\u0391-\\u03A1\\u03A3-\\u03AB\\u03CF\\u03D2-\\u03D4\\u03D8\\u03DA\\u03DC\\u03DE\\u03E0\\u03E2\\u03E4\\u03E6\\u03E8\\u03EA\\u03EC\\u03EE\\u03F4\\u03F7\\u03F9\\u03FA\\u03FD-\\u042F\\u0460\\u0462\\u0464\\u0466\\u0468\\u046A\\u046C\\u046E\\u0470\\u0472\\u0474\\u0476\\u0478\\u047A\\u047C\\u047E\\u0480\\u048A\\u048C\\u048E\\u0490\\u0492\\u0494\\u0496\\u0498\\u049A\\u049C\\u049E\\u04A0\\u04A2\\u04A4\\u04A6\\u04A8\\u04AA\\u04AC\\u04AE\\u04B0\\u04B2\\u04B4\\u04B6\\u04B8\\u04BA\\u04BC\\u04BE\\u04C0\\u04C1\\u04C3\\u04C5\\u04C7\\u04C9\\u04CB\\u04CD\\u04D0\\u04D2\\u04D4\\u04D6\\u04D8\\u04DA\\u04DC\\u04DE\\u04E0\\u04E2\\u04E4\\u04E6\\u04E8\\u04EA\\u04EC\\u04EE\\u04F0\\u04F2\\u04F4\\u04F6\\u04F8\\u04FA\\u04FC\\u04FE\\u0500\\u0502\\u0504\\u0506\\u0508\\u050A\\u050C\\u050E\\u0510\\u0512\\u0514\\u0516\\u0518\\u051A\\u051C\\u051E\\u0520\\u0522\\u0524\\u0526\\u0528\\u052A\\u052C\\u052E\\u0531-\\u0556\\u10A0-\\u10C5\\u10C7\\u10CD\\u13A0-\\u13F5\\u1E00\\u1E02\\u1E04\\u1E06\\u1E08\\u1E0A\\u1E0C\\u1E0E\\u1E10\\u1E12\\u1E14\\u1E16\\u1E18\\u1E1A\\u1E1C\\u1E1E\\u1E20\\u1E22\\u1E24\\u1E26\\u1E28\\u1E2A\\u1E2C\\u1E2E\\u1E30\\u1E32\\u1E34\\u1E36\\u1E38\\u1E3A\\u1E3C\\u1E3E\\u1E40\\u1E42\\u1E44\\u1E46\\u1E48\\u1E4A\\u1E4C\\u1E4E\\u1E50\\u1E52\\u1E54\\u1E56\\u1E58\\u1E5A\\u1E5C\\u1E5E\\u1E60\\u1E62\\u1E64\\u1E66\\u1E68\\u1E6A\\u1E6C\\u1E6E\\u1E70\\u1E72\\u1E74\\u1E76\\u1E78\\u1E7A\\u1E7C\\u1E7E\\u1E80\\u1E82\\u1E84\\u1E86\\u1E88\\u1E8A\\u1E8C\\u1E8E\\u1E90\\u1E92\\u1E94\\u1E9E\\u1EA0\\u1EA2\\u1EA4\\u1EA6\\u1EA8\\u1EAA\\u1EAC\\u1EAE\\u1EB0\\u1EB2\\u1EB4\\u1EB6\\u1EB8\\u1EBA\\u1EBC\\u1EBE\\u1EC0\\u1EC2\\u1EC4\\u1EC6\\u1EC8\\u1ECA\\u1ECC\\u1ECE\\u1ED0\\u1ED2\\u1ED4\\u1ED6\\u1ED8\\u1EDA\\u1EDC\\u1EDE\\u1EE0\\u1EE2\\u1EE4\\u1EE6\\u1EE8\\u1EEA\\u1EEC\\u1EEE\\u1EF0\\u1EF2\\u1EF4\\u1EF6\\u1EF8\\u1EFA\\u1EFC\\u1EFE\\u1F08-\\u1F0F\\u1F18-\\u1F1D\\u1F28-\\u1F2F\\u1F38-\\u1F3F\\u1F48-\\u1F4D\\u1F59\\u1F5B\\u1F5D\\u1F5F\\u1F68-\\u1F6F\\u1FB8-\\u1FBB\\u1FC8-\\u1FCB\\u1FD8-\\u1FDB\\u1FE8-\\u1FEC\\u1FF8-\\u1FFB\\u2102\\u2107\\u210B-\\u210D\\u2110-\\u2112\\u2115\\u2119-\\u211D\\u2124\\u2126\\u2128\\u212A-\\u212D\\u2130-\\u2133\\u213E\\u213F\\u2145\\u2183\\u2C00-\\u2C2E\\u2C60\\u2C62-\\u2C64\\u2C67\\u2C69\\u2C6B\\u2C6D-\\u2C70\\u2C72\\u2C75\\u2C7E-\\u2C80\\u2C82\\u2C84\\u2C86\\u2C88\\u2C8A\\u2C8C\\u2C8E\\u2C90\\u2C92\\u2C94\\u2C96\\u2C98\\u2C9A\\u2C9C\\u2C9E\\u2CA0\\u2CA2\\u2CA4\\u2CA6\\u2CA8\\u2CAA\\u2CAC\\u2CAE\\u2CB0\\u2CB2\\u2CB4\\u2CB6\\u2CB8\\u2CBA\\u2CBC\\u2CBE\\u2CC0\\u2CC2\\u2CC4\\u2CC6\\u2CC8\\u2CCA\\u2CCC\\u2CCE\\u2CD0\\u2CD2\\u2CD4\\u2CD6\\u2CD8\\u2CDA\\u2CDC\\u2CDE\\u2CE0\\u2CE2\\u2CEB\\u2CED\\u2CF2\\uA640\\uA642\\uA644\\uA646\\uA648\\uA64A\\uA64C\\uA64E\\uA650\\uA652\\uA654\\uA656\\uA658\\uA65A\\uA65C\\uA65E\\uA660\\uA662\\uA664\\uA666\\uA668\\uA66A\\uA66C\\uA680\\uA682\\uA684\\uA686\\uA688\\uA68A\\uA68C\\uA68E\\uA690\\uA692\\uA694\\uA696\\uA698\\uA69A\\uA722\\uA724\\uA726\\uA728\\uA72A\\uA72C\\uA72E\\uA732\\uA734\\uA736\\uA738\\uA73A\\uA73C\\uA73E\\uA740\\uA742\\uA744\\uA746\\uA748\\uA74A\\uA74C\\uA74E\\uA750\\uA752\\uA754\\uA756\\uA758\\uA75A\\uA75C\\uA75E\\uA760\\uA762\\uA764\\uA766\\uA768\\uA76A\\uA76C\\uA76E\\uA779\\uA77B\\uA77D\\uA77E\\uA780\\uA782\\uA784\\uA786\\uA78B\\uA78D\\uA790\\uA792\\uA796\\uA798\\uA79A\\uA79C\\uA79E\\uA7A0\\uA7A2\\uA7A4\\uA7A6\\uA7A8\\uA7AA-\\uA7AD\\uA7B0-\\uA7B4\\uA7B6\\uFF21-\\uFF3A]\"\n    p = re.compile(pLu)\n    data['capitals_'+str(idx)] = data[feat].apply(lambda comment: sum(1 for c in comment if p.match(c)))\n    data['caps_vs_length_'+str(idx)] = data.apply(lambda row: float(row['capitals_'+str(idx)])/float(row['total_len_'+str(idx)]),\n                                axis=1)\n    data['num_exclamation_marks_'+str(idx)] = data[feat].apply(lambda comment: comment.count('!'))\n    data['num_question_marks_'+str(idx)] = data[feat].apply(lambda comment: comment.count('?'))\n    data['num_punctuation_'+str(idx)] = data[feat].apply(\n        lambda comment: sum(comment.count(w) for w in '.,;:'))\n    data['num_symbols_'+str(idx)] = data[feat].apply(\n        lambda comment: sum(comment.count(w) for w in '*&$%'))\n    data['num_words_'+str(idx)] = data[feat].apply(lambda comment: len(comment.split()))\n    data['num_unique_words_'+str(idx)] = data[feat].apply(\n        lambda comment: len(set(w for w in comment.split())))\n    data['words_vs_unique_'+str(idx)] = data['num_unique_words_'+str(idx)] / data['num_words_'+str(idx)]\n    data['num_smilies_'+str(idx)] = data[feat].apply(\n        lambda comment: sum(comment.count(w) for w in (':-)', ':)', ';-)', ';)')))","execution_count":21,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b73ad1ec289a901fea75edf1267f3af60205805"},"cell_type":"code","source":"features = ('total_len', 'capitals', 'caps_vs_length', 'num_exclamation_marks',\n            'num_question_marks', 'num_punctuation', 'num_words', 'num_unique_words',\n            'words_vs_unique', 'num_smilies', 'num_symbols')\nfeature_description = ('total_len_0', 'capitals_0', 'caps_vs_length_0', 'num_exclamation_marks_0',\n            'num_question_marks_0', 'num_punctuation_0', 'num_words_0', 'num_unique_words_0',\n            'words_vs_unique_0', 'num_smilies_0', 'num_symbols_0')\ncolumns = ('deal_probability')\nfeature_topics = ('total_len_1', 'capitals_1', 'caps_vs_length_1', 'num_exclamation_marks_1',\n            'num_question_marks_1', 'num_punctuation_1', 'num_words_1', 'num_unique_words_1',\n            'words_vs_unique_1', 'num_smilies_1', 'num_symbols_1')\n\nrows = [{'description_deal_probability':data[f].corr(data['deal_probability'])} for f in feature_description]\ntopic_rows = [{'topic_deal_probability':data[f].corr(data['deal_probability'])} for f in feature_topics]\n\ndf_correlations = pd.DataFrame(rows, index=features)\ndf_correlations_topic = pd.DataFrame(topic_rows, index=features)\ndf_correlations['topic_deal_probability'] = df_correlations_topic['topic_deal_probability']","execution_count":36,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d5e5a54af02da7b227bae9a9c530b0ee83c66a26"},"cell_type":"code","source":"df_correlations['topic_deal_probability'] = df_correlations['topic_deal_probability'].fillna(0.0)\ndf_correlations","execution_count":37,"outputs":[]},{"metadata":{"_uuid":"d74a761634e6172f09c6933ef87570b68f61f0ee","_cell_guid":"c68a20dc-70f6-414a-bd45-6930f9ce2174","trusted":true},"cell_type":"code","source":"import seaborn as sns\n\nax = sns.heatmap(df_correlations, vmin=-0.2, vmax=0.2, center=0.0)","execution_count":33,"outputs":[]},{"metadata":{"_uuid":"7fa8f21e37b9b420a6dd891d2d5b36a9e5fa6b0b","collapsed":true,"_cell_guid":"4a47a6ca-ea07-48dc-bd8c-96b53ce9ef46","trusted":false},"cell_type":"markdown","source":"Cant get anything useful yet"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"551a5cd3650b855a7a4dbe8da78c9cdc1a987a7c"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}