{"cells":[{"metadata":{"_uuid":"710ed17d0c57bd287be0ee3b2782a53a54510561"},"cell_type":"markdown","source":"## IMPORTS "},{"metadata":{"_uuid":"abb7e3c30b8a412a50c6b451c49939e3cf4bc11b","scrolled":true,"trusted":true},"cell_type":"code","source":"import random\nimport pandas as pd\nimport numpy as np\nfrom multiprocessing import  Pool","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"63cb21525251b060aeb309e7be4b48772f8720f5","scrolled":true,"trusted":true},"cell_type":"code","source":"def parallelize_dataframe(df, func):\n    df_split = np.array_split(df, 4)\n    pool = Pool(4)\n    df = pd.concat(pool.map(func, df_split))\n    pool.close()\n    pool.join()\n    return df\n\ndef add_features(df):\n    df['question_text'] = df['question_text'].apply(lambda x:str(x))\n    df[\"lower_question_text\"] = df[\"question_text\"].apply(lambda x: x.lower())\n    df['total_length'] = df['question_text'].apply(len)\n    df['capitals'] = df['question_text'].apply(lambda comment: sum(1 for c in comment if c.isupper()))\n    df['caps_vs_length'] = df.apply(lambda row: float(row['capitals'])/float(row['total_length']),\n                                axis=1)\n    df['num_words'] = df.question_text.str.count('\\S+')\n    df['num_unique_words'] = df['question_text'].apply(lambda comment: len(set(w for w in comment.split())))\n    df['words_vs_unique'] = df['num_unique_words'] / df['num_words'] \n    df['num_exclamation_marks'] = df['question_text'].apply(lambda comment: comment.count('!'))\n    df['num_question_marks'] = df['question_text'].apply(lambda comment: comment.count('?'))\n    df['num_punctuation'] = df['question_text'].apply(lambda comment: sum(comment.count(w) for w in '.,;:'))\n    df['num_symbols'] = df['question_text'].apply(lambda comment: sum(comment.count(w) for w in '*&$%'))\n    df['num_smilies'] = df['question_text'].apply(lambda comment: sum(comment.count(w) for w in (':-)', ':)', ';-)', ';)')))\n    df['num_sad'] = df['question_text'].apply(lambda comment: sum(comment.count(w) for w in (':-<', ':()', ';-()', ';(')))\n    df[\"mean_word_len\"] = df[\"question_text\"].apply(lambda x: np.mean([len(w) for w in str(x).split()]))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%timeit\ntrain = parallelize_dataframe(train_df, add_features) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%timeit\ntrain = add_features(train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}