{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"In this sample notebook it is demonstrated how to apply Kalman Filter to improve the baseline slightly\n\nThe notebook is based on https://www.kaggle.com/jpmiller/baseline-from-host-data","metadata":{}},{"cell_type":"markdown","source":"## ensure you have everything you need","metadata":{}},{"cell_type":"code","source":"!pip install simdkalman","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Please read the documentation if you would like to learn more about this implementation of kf: https://simdkalman.readthedocs.io/en/latest/","metadata":{}},{"cell_type":"code","source":"from pathlib import Path\nimport numpy as np\nimport pandas as pd\nimport simdkalman\nfrom tqdm.notebook import tqdm","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## define kf model\n","metadata":{}},{"cell_type":"code","source":"T = 0.83\nstate_transition = np.array([[1, 0, T, 0, 0.5 * T ** 2, 0], [0, 1, 0, T, 0, 0.5 * T ** 2], [0, 0, 1, 0, T, 0],\n                             [0, 0, 0, 1, 0, T], [0, 0, 0, 0, 1, 0], [0, 0, 0, 0, 0, 1]])\nprocess_noise = np.diag([1e-5, 1e-5, 5e-6, 5e-6, 1e-6, 1e-6]) + np.ones((6, 6)) * 1e-9\nobservation_model = np.array([[1, 0, 0, 0, 0, 0], [0, 1, 0, 0, 0, 0]])\nobservation_noise = np.diag([5e-5, 5e-5]) + np.ones((2, 2)) * 1e-9\n\nkf = simdkalman.KalmanFilter(\n        state_transition = state_transition,\n        process_noise = process_noise,\n        observation_model = observation_model,\n        observation_noise = observation_noise)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def apply_kf_smoothing(df, kf_=kf):\n    unique_paths = df[['collectionName', 'phoneName']].drop_duplicates().to_numpy()\n    for collection, phone in tqdm(unique_paths):\n        cond = np.logical_and(df['collectionName'] == collection, df['phoneName'] == phone)\n        data = df[cond][['latDeg', 'lngDeg']].to_numpy()\n        data = data.reshape(1, len(data), 2)\n        smoothed = kf_.smooth(data)\n        df.loc[cond, 'latDeg'] = smoothed.states.mean[0, :, 0]\n        df.loc[cond, 'lngDeg'] = smoothed.states.mean[0, :, 1]\n    return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## evaluate train error","metadata":{}},{"cell_type":"code","source":"data_path = Path(\"../input/google-smartphone-decimeter-challenge\")\n\ntruths = (data_path / 'train').rglob('ground_truth.csv')\n    # returns a generator\n\ndf_list = []\ncols = ['collectionName', 'phoneName', 'millisSinceGpsEpoch', 'latDeg', 'lngDeg']\n\nfor t in tqdm(truths, total=73):\n    df_phone = pd.read_csv(t, usecols=cols)  \n    df_list.append(df_phone)\ndf_truth = pd.concat(df_list, ignore_index=True)\n\ndf_basepreds_kf = apply_kf_smoothing(pd.read_csv('../input/google-smartphone-decimeter-challenge/baseline_locations_train.csv', usecols=cols))\ndf_all = df_truth.merge(df_basepreds_kf, how='inner', on=cols[:3], suffixes=('_truth', '_basepred'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# simplified haversine distance\ndef calc_haversine(lat1, lon1, lat2, lon2):\n    \"\"\"Calculates the great circle distance between two points\n    on the earth. Inputs are array-like and specified in decimal degrees.\n    \"\"\"\n    lat1, lon1, lat2, lon2 = map(np.radians, [lat1, lon1, lat2, lon2])\n    dlat = lat2 - lat1\n    dlon = lon2 - lon1\n    a = np.sin(dlat/2.0)**2 + np.cos(lat1) * np.cos(lat2) * np.sin(dlon/2.0)**2\n\n    c = 2 * np.arcsin(a**0.5)\n    dist = 6_367_000 * c\n    return dist","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all['dist'] = calc_haversine(df_all.latDeg_truth, df_all.lngDeg_truth, \n    df_all.latDeg_basepred, df_all.lngDeg_basepred)\n\nprint(f'mean error on train dataset: {df_all.dist.mean():.5f}m - slightly better than the baseline, but still a lot of improvements are needed')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## prepare a submission based on the sample submission","metadata":{}},{"cell_type":"code","source":"test_base = pd.read_csv(\n    '../input/google-smartphone-decimeter-challenge/baseline_locations_test.csv')\n\nsub = pd.read_csv('../input/google-smartphone-decimeter-challenge/sample_submission.csv')\n\nkf_smoothed_baseline = apply_kf_smoothing(test_base)\nsub = sub.assign(\n    latDeg = kf_smoothed_baseline.latDeg,\n    lngDeg = kf_smoothed_baseline.lngDeg\n)\nsub.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}