{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Thanks for https://www.kaggle.com/tqa236/kalman-filter-hyperparameter-search-with-bo for pandas tricks I'm new to.","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom scipy.ndimage import gaussian_filter1d\nfrom scipy.interpolate import interp1d","metadata":{"execution":{"iopub.status.busy":"2021-07-28T20:44:58.023840Z","iopub.execute_input":"2021-07-28T20:44:58.024396Z","iopub.status.idle":"2021-07-28T20:44:58.028764Z","shell.execute_reply.started":"2021-07-28T20:44:58.024364Z","shell.execute_reply":"2021-07-28T20:44:58.027877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def apply_gauss_smoothing(df, params):\n    SZ_1 = params['sz_1']\n    SZ_2 = params['sz_2']\n    SZ_CRIT = params['sz_crit']    \n    \n    unique_paths = df[['collectionName', 'phoneName']].drop_duplicates().to_numpy()\n    for collection, phone in unique_paths:\n        cond = np.logical_and(df['collectionName'] == collection, df['phoneName'] == phone)\n        data = df[cond][['latDeg', 'lngDeg']].to_numpy()\n                \n        lat_g1 = gaussian_filter1d(data[:, 0], np.sqrt(SZ_1))\n        lon_g1 = gaussian_filter1d(data[:, 1], np.sqrt(SZ_1))\n        lat_g2 = gaussian_filter1d(data[:, 0], np.sqrt(SZ_2))\n        lon_g2 = gaussian_filter1d(data[:, 1], np.sqrt(SZ_2))\n\n        lat_dif = data[1:,0] - data[:-1,0]\n        lon_dif = data[1:,1] - data[:-1,1]\n\n        lat_crit = np.append(np.abs(gaussian_filter1d(lat_dif, np.sqrt(SZ_CRIT)) / (1e-9 + gaussian_filter1d(np.abs(lat_dif), np.sqrt(SZ_CRIT)))),[0])\n        lon_crit = np.append(np.abs(gaussian_filter1d(lon_dif, np.sqrt(SZ_CRIT)) / (1e-9 + gaussian_filter1d(np.abs(lon_dif), np.sqrt(SZ_CRIT)))),[0])           \n            \n        df.loc[cond, 'latDeg'] = lat_g1 * lat_crit + lat_g2 * (1.0 - lat_crit)\n        df.loc[cond, 'lngDeg'] = lon_g1 * lon_crit + lon_g2 * (1.0 - lon_crit)    \n                       \n    return df","metadata":{"execution":{"iopub.status.busy":"2021-07-28T20:44:58.030095Z","iopub.execute_input":"2021-07-28T20:44:58.030607Z","iopub.status.idle":"2021-07-28T20:44:58.046425Z","shell.execute_reply.started":"2021-07-28T20:44:58.030575Z","shell.execute_reply":"2021-07-28T20:44:58.044872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mean_with_other_phones(df):\n    collections_list = df[['collectionName']].drop_duplicates().to_numpy()\n\n    for collection in collections_list:\n        phone_list = df[df['collectionName'].to_list() == collection][['phoneName']].drop_duplicates().to_numpy()\n\n        phone_data = {}\n        corrections = {}\n        for phone in phone_list:\n            cond = np.logical_and(df['collectionName'] == collection[0], df['phoneName'] == phone[0]).to_list()\n            phone_data[phone[0]] = df[cond][['millisSinceGpsEpoch', 'latDeg', 'lngDeg']].to_numpy()\n\n        for current in phone_data:\n            correction = np.ones(phone_data[current].shape, dtype=np.float)\n            correction[:,1:] = phone_data[current][:,1:]\n            \n            # Telephones data don't complitely match by time, so - interpolate.\n            for other in phone_data:\n                if other == current:\n                    continue\n\n                loc = interp1d(phone_data[other][:,0], \n                               phone_data[other][:,1:], \n                               axis=0, \n                               kind='linear', \n                               copy=False, \n                               bounds_error=None, \n                               fill_value='extrapolate', \n                               assume_sorted=True)\n                \n                start_idx = 0\n                stop_idx = 0\n                for idx, val in enumerate(phone_data[current][:,0]):\n                    if val < phone_data[other][0,0]:\n                        start_idx = idx\n                    if val < phone_data[other][-1,0]:\n                        stop_idx = idx\n\n                if stop_idx - start_idx > 0:\n                    correction[start_idx:stop_idx,0] += 1\n                    correction[start_idx:stop_idx,1:] += loc(phone_data[current][start_idx:stop_idx,0])                    \n\n            correction[:,1] /= correction[:,0]\n            correction[:,2] /= correction[:,0]\n            \n            corrections[current] = correction.copy()\n        \n        for phone in phone_list:\n            cond = np.logical_and(df['collectionName'] == collection[0], df['phoneName'] == phone[0]).to_list()\n            \n            df.loc[cond, ['latDeg', 'lngDeg']] = corrections[phone[0]][:,1:]            \n            \n    return df","metadata":{"execution":{"iopub.status.busy":"2021-07-28T20:44:58.048555Z","iopub.execute_input":"2021-07-28T20:44:58.048868Z","iopub.status.idle":"2021-07-28T20:44:58.065138Z","shell.execute_reply.started":"2021-07-28T20:44:58.048838Z","shell.execute_reply":"2021-07-28T20:44:58.063928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_base = pd.read_csv('../input/google-smartphone-decimeter-challenge/baseline_locations_test.csv')\nsub = pd.read_csv('../input/google-smartphone-decimeter-challenge/sample_submission.csv')\n\nsmoothed_baseline = apply_gauss_smoothing(test_base, {'sz_1' : 0.85, 'sz_2' : 5.65, 'sz_crit' : 1.5})\nsmoothed_baseline = mean_with_other_phones(smoothed_baseline)\n\nsub = sub.assign( latDeg=smoothed_baseline.latDeg, lngDeg=smoothed_baseline.lngDeg )\nsub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-28T20:44:58.066522Z","iopub.execute_input":"2021-07-28T20:44:58.066994Z","iopub.status.idle":"2021-07-28T20:45:06.482821Z","shell.execute_reply.started":"2021-07-28T20:44:58.066960Z","shell.execute_reply":"2021-07-28T20:45:06.481919Z"},"trusted":true},"execution_count":null,"outputs":[]}]}