{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\nimport gc\nimport warnings\nwarnings.filterwarnings('ignore')\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)\nfrom tqdm.auto import tqdm\nimport itertools","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_parquet('../input/amex-data-integer-dtypes-parquet-format/train.parquet')\ntrain = train.iloc[:int(len(train)/1.5)]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = train.drop(['customer_ID', 'S_2'], axis = 1).columns.to_list()\ncat_features = [\n    \"B_30\",\n    \"B_38\",\n    \"D_114\",\n    \"D_116\",\n    \"D_117\",\n    \"D_120\",\n    \"D_126\",\n    \"D_63\",\n    \"D_64\",\n    \"D_66\",\n    \"D_68\",\n]\nnum_features = [col for col in features if col not in cat_features]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Baseline Implementation","metadata":{}},{"cell_type":"code","source":"def baseline_get_difference(data, num_features):\n    df1 = []\n    customer_ids = []\n    for customer_id, df in tqdm(data.groupby(['customer_ID'])):\n        # Get the differences\n        diff_df1 = df[num_features].diff(1).iloc[[-1]].values.astype(np.float32)\n        # Append to lists\n        df1.append(diff_df1)\n        customer_ids.append(customer_id)\n    # Concatenate\n    df1 = np.concatenate(df1, axis = 0)\n    # Transform to dataframe\n    df1 = pd.DataFrame(df1, columns = [col + '_diff1' for col in df[num_features].columns])\n    # Add customer id\n    df1['customer_ID'] = customer_ids\n    return df1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_diff = baseline_get_difference(train, num_features)\n\ndel train_diff","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Multi-Processing Version","metadata":{}},{"cell_type":"code","source":"import functools\nfrom multiprocessing import Pool, cpu_count\n\ndef apply_in_parallel(input_df,\n                      features,\n                      func):\n    \n    cust_count = input_df[\"customer_ID\"].nunique()\n    \n    with Pool(processes=3) as pool:\n        applied_dfs = pool.imap(func, input_df[features + [\"customer_ID\"]].groupby(\"customer_ID\"))\n        applied_dfs = tqdm(applied_dfs, total=cust_count)\n        applied_dfs = list(applied_dfs)\n        \n    applied_dfs = pd.concat(applied_dfs,\n                           axis=0)\n    return applied_dfs\n\ndef get_difference_for_parallel(input_df):\n    input_df = input_df[1].drop(\"customer_ID\", 1)\n    \n    # Get the differences\n    diff_df = input_df.diff(1).iloc[[-1]].values.astype(np.float32)\n    diff_df = pd.DataFrame(diff_df, columns=input_df.columns)\n    \n    return diff_df\n\ndef get_difference(data, num_features):\n\n    customer_ids = list(data.groupby('customer_ID').groups.keys())\n    \n    df1 = apply_in_parallel(data,\n                              features=num_features,\n                              func=get_difference_for_parallel)\n    \n    # Transform to dataframe\n    df1 = pd.DataFrame(df1.values, columns = [col + '_diff1' for col in data[num_features].columns])\n    # Add customer id\n    df1['customer_ID'] = customer_ids\n    \n    return df1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_diff = get_difference(train, num_features)\ndel train_diff","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## P.S.\n\nYou can get much better improvements on your local machines. Mine was **almost x2** with using **8** cores!","metadata":{}}]}