{"cells":[{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"Simple time benchmarks to know how to process features efficiently. Special thanks to Olivier's kernel: https://www.kaggle.com/ogrellier/multi-core-aggregations/code, which has some differences with my code and made me think how pandas processes the data. I'll use some of his features and single core for simplicity ;)"},{"metadata":{"trusted":true,"_uuid":"841cfb33a5b409f9e1395c20d904276bb50942ce"},"cell_type":"code","source":"import time\nimport numpy as np\nimport pandas as pd\n\ntr = pd.read_csv('../input/training_set.csv')\ngroups = tr.groupby(['object_id'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b16f57e62206c45d382512e5b650323175b6cd05"},"cell_type":"markdown","source":"### \"Apply\" function"},{"metadata":{"trusted":true,"_uuid":"ae072b8eaabea00afa9b1db5168c55d2d09f8911"},"cell_type":"code","source":"# Version using pandas Series\ndef compute_all_aggregated_features(df):\n    # Compute weighted mean\n    a_s = df['flux'] * np.power(df['flux'] / df['flux_err'], 2)\n    b_s = np.power(df['flux'] / df['flux_err'], 2)\n    wmean = np.sum(a_s) / np.sum(b_s)\n\n    flux_med = np.median(df['flux'])\n    # Compute normed flux\n    normed_flux = (df['flux'].max() - df['flux'].min()) / wmean\n\n    # normed_median_flux\n    normed_median_flux = np.median(np.abs(df['flux'] - flux_med) / wmean)\n    \n    return pd.Series([wmean, flux_med, normed_flux, normed_median_flux])\n\n# Same as above but using arrays\ndef compute_all_aggregated_features_v2(df):\n    # Compute weighted mean\n    a_s = df['flux'].values * np.power(df['flux'].values / df['flux_err'].values, 2)\n    b_s = np.power(df['flux'].values / df['flux_err'].values, 2)\n    wmean = np.sum(a_s) / np.sum(b_s)\n\n    flux_med = np.median(df['flux'].values)\n    # Compute normed flux\n    normed_flux = (np.max(df['flux'].values) - np.min(df['flux'].values)) / wmean\n\n    # normed_median_flux\n    normed_median_flux = np.median(np.abs(df['flux'] - flux_med) / wmean)\n    \n    return pd.Series([wmean, flux_med, normed_flux, normed_median_flux])\n\n# Arrays version + optimizations\ndef compute_all_aggregated_features_v3(df):\n    flux = df['flux'].values\n    flux_err = df['flux_err'].values\n    # Compute weighted mean\n    b_s = np.power(flux / flux_err, 2)\n    a_s = flux * b_s\n    wmean = np.sum(a_s) / np.sum(b_s)\n\n    flux_med = np.median(flux)\n    # Compute normed flux\n    normed_flux = (np.max(flux) - np.min(flux)) / wmean\n\n    # normed_median_flux\n    normed_median_flux = np.median(np.abs(flux - flux_med) / wmean)\n    \n    return pd.Series([wmean, flux_med, normed_flux, normed_median_flux])\n\nt1 = time.time()\nz1 = groups.apply(compute_all_aggregated_features)\nt2 = time.time()\nz2 = groups.apply(compute_all_aggregated_features_v2)\nt3 = time.time()\nz3 = groups.apply(compute_all_aggregated_features_v3)\nt4 = time.time()\nprint('pandas.Series: {0:.3f} s'.format(t2-t1))\nprint('arrays: {0:.3f} s'.format(t3-t2))\nprint('arrays+optimizations: {0:.3f} s'.format(t4-t3))\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c27e3d745f58688e4b85084727564821476abd52"},"cell_type":"markdown","source":"With the **apply** function it's clearly more efficient to use arrays than series."},{"metadata":{"trusted":true,"_uuid":"51adbadcd1b5d2a7f963e6e8d42c8b9ff954bc12"},"cell_type":"code","source":"# Check results\nfor col in z1.columns:\n    a = np.abs(z1[col] - z2[col])\n    b = np.abs(z1[col] - z3[col])\n    print(col, np.max(a), np.max(b))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5c629bb69889e9f642877580130b65e8c927501a"},"cell_type":"markdown","source":"### Aggregations vs apply"},{"metadata":{"trusted":true,"_uuid":"6ffbf8fd9f88e25ac2373216d1f62e2eef5c66c9"},"cell_type":"code","source":"def get_initial_aggregations():\n    return {\n        'flux': ['min', 'max', 'mean', 'median', 'std']\n    }\n\ndef get_initial_aggregations_v2(df):\n    flux = df['flux'].values\n    return pd.Series([np.min(flux), np.max(flux), np.mean(flux), np.median(flux), np.std(flux, ddof=1)])\n\nt1 = time.time()\nz1 = groups.agg(get_initial_aggregations())\nt2 = time.time()\nz2 = groups.apply(get_initial_aggregations_v2)\nt3 = time.time()\nprint('aggregations: {0:.3f} s'.format(t2-t1))\nprint('apply with arrays: {0:.3f} s'.format(t3-t2))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"88d32680edf7617ceb7faf022955d37b488ce1de"},"cell_type":"markdown","source":"For statistics on series, the best option is using aggregations"},{"metadata":{"trusted":true,"_uuid":"155a90f8efd9d015877e8694c04adb1118c0d1dc"},"cell_type":"code","source":"# Check results\nz1.columns = ['_'.join([i, j]) for i, j in z1.columns]\nz2.columns = z1.columns\nfor col in z1.columns:\n    a = np.abs(z1[col] - z2[col])\n    print(col, np.max(a))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"83cb161064c3fc841ee77b9f0cbf9a0e592b47c5"},"cell_type":"markdown","source":""}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}