{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# import library\nimport os\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom matplotlib_venn import venn2, venn2_circles\nimport seaborn as sns\nfrom tqdm.notebook import tqdm\nimport pathlib\nimport plotly\nimport plotly.express as px","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-10T00:16:25.264665Z","iopub.execute_input":"2021-06-10T00:16:25.265081Z","iopub.status.idle":"2021-06-10T00:16:27.391125Z","shell.execute_reply.started":"2021-06-10T00:16:25.265001Z","shell.execute_reply":"2021-06-10T00:16:27.390128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# utils","metadata":{"_kg_hide-input":true}},{"cell_type":"code","source":"def calc_haversine(lat1, lon1, lat2, lon2):\n    \"\"\"Calculates the great circle distance between two points\n    on the earth. Inputs are array-like and specified in decimal degrees.\n    \"\"\"\n    RADIUS = 6_367_000\n    lat1, lon1, lat2, lon2 = map(np.radians, [lat1, lon1, lat2, lon2])\n    dlat = lat2 - lat1\n    dlon = lon2 - lon1\n    a = np.sin(dlat/2)**2 + \\\n        np.cos(lat1) * np.cos(lat2) * np.sin(dlon/2)**2\n    dist = 2 * RADIUS * np.arcsin(a**0.5)\n    return dist","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-10T00:16:27.392399Z","iopub.execute_input":"2021-06-10T00:16:27.392657Z","iopub.status.idle":"2021-06-10T00:16:27.399368Z","shell.execute_reply.started":"2021-06-10T00:16:27.392632Z","shell.execute_reply":"2021-06-10T00:16:27.398383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize_trafic(df, center, zoom=9):\n    fig = px.scatter_mapbox(df,\n                            \n                            # Here, plotly gets, (x,y) coordinates\n                            lat=\"latDeg\",\n                            lon=\"lngDeg\",\n                            \n                            #Here, plotly detects color of series\n                            color=\"phoneName\",\n                            labels=\"phoneName\",\n                            \n                            zoom=zoom,\n                            center=center,\n                            height=600,\n                            width=800)\n    fig.update_layout(mapbox_style='stamen-terrain')\n    fig.update_layout(margin={\"r\": 0, \"t\": 0, \"l\": 0, \"b\": 0})\n    fig.update_layout(title_text=\"GPS trafic\")\n    fig.show()\n    \ndef visualize_collection(df, collection):\n    target_df = df[df['collectionName']==collection].copy()\n    lat_center = target_df['latDeg'].mean()\n    lng_center = target_df['lngDeg'].mean()\n    center = {\"lat\":lat_center, \"lon\":lng_center}\n    \n    visualize_trafic(target_df, center)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-10T00:16:27.402384Z","iopub.execute_input":"2021-06-10T00:16:27.402933Z","iopub.status.idle":"2021-06-10T00:16:27.412743Z","shell.execute_reply.started":"2021-06-10T00:16:27.402873Z","shell.execute_reply":"2021-06-10T00:16:27.411599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_distance_diff(df):\n    df['latDeg_prev'] = df['latDeg'].shift(1)\n    df['latDeg_next'] = df['latDeg'].shift(-1)\n    df['lngDeg_prev'] = df['lngDeg'].shift(1)\n    df['lngDeg_next'] = df['lngDeg'].shift(-1)\n    df['phone_prev'] = df['phone'].shift(1)\n    df['phone_next'] = df['phone'].shift(-1)\n    \n    df['latDeg_gt_prev'] = df['latDeg_gt'].shift(1)\n    df['latDeg_gt_next'] = df['latDeg_gt'].shift(-1)\n    df['lngDeg_gt_prev'] = df['lngDeg_gt'].shift(1)\n    df['lngDeg_gt_next'] = df['lngDeg_gt'].shift(-1)    \n    \n    df['latDeg_prev_diff'] = df['latDeg'] - df['latDeg_prev']\n    df['latDeg_next_diff'] = df['latDeg_next'] - df['latDeg']\n    df['latDeg_gt_prev_diff'] = df['latDeg_gt'] - df['latDeg_gt_prev']\n    df['latDeg_gt_next_diff'] = df['latDeg_gt_next'] - df['latDeg_gt']\n    \n    df['lngDeg_prev_diff'] = df['lngDeg'] - df['lngDeg_prev']\n    df['lngDeg_next_diff'] = df['lngDeg_next'] - df['lngDeg']\n    df['lngDeg_gt_prev_diff'] = df['lngDeg_gt'] - df['lngDeg_gt_prev']\n    df['lngDeg_gt_next_diff'] = df['lngDeg_gt_next'] - df['lngDeg_gt']\n    \n    df['dist_prev'] = calc_haversine(df['latDeg'], df['lngDeg'], df['latDeg_prev'], df['lngDeg_prev'])\n    df['dist_next'] = calc_haversine(df['latDeg'], df['lngDeg'], df['latDeg_next'], df['lngDeg_next'])\n    \n    df['dist_gt_prev'] = calc_haversine(df['latDeg_gt'], df['lngDeg_gt'], df['latDeg_gt_prev'], df['lngDeg_gt_prev'])\n    df['dist_gt_next'] = calc_haversine(df['latDeg_gt'], df['lngDeg_gt'], df['latDeg_gt_next'], df['lngDeg_gt_next'])\n    \n    df.loc[df['phone']!=df['phone_prev'], ['latDeg_prev', 'lngDeg_prev', 'dist_prev', 'latDeg_gt_prev', 'lngDeg_gt_prev', 'dist_gt_prev', \n                                           'latDeg_prev_diff', 'latDeg_gt_prev_diff', 'lngDeg_prev_diff', 'lngDeg_gt_prev_diff']] = np.nan\n    \n    df.loc[df['phone']!=df['phone_next'], ['latDeg_next', 'lngDeg_next', 'dist_next', 'latDeg_gt_next', 'lngDeg_gt_next', 'dist_gt_next',\n                                           'latDeg_next_diff', 'latDeg_gt_next_diff', 'lngDeg_next_diff', 'lngDeg_gt_next_diff']] = np.nan\n    \n    return df","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-10T00:16:27.415013Z","iopub.execute_input":"2021-06-10T00:16:27.415530Z","iopub.status.idle":"2021-06-10T00:16:27.429693Z","shell.execute_reply.started":"2021-06-10T00:16:27.415488Z","shell.execute_reply":"2021-06-10T00:16:27.429029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# directory setting\nINPUT = '../input/google-smartphone-decimeter-challenge'","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2021-06-10T00:16:27.430774Z","iopub.execute_input":"2021-06-10T00:16:27.431148Z","iopub.status.idle":"2021-06-10T00:16:27.444534Z","shell.execute_reply.started":"2021-06-10T00:16:27.431121Z","shell.execute_reply":"2021-06-10T00:16:27.443844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(INPUT + '/' + 'baseline_locations_train.csv')\ntest = pd.read_csv(INPUT + '/' + 'baseline_locations_test.csv')\nsample_sub = pd.read_csv(INPUT + '/' + 'sample_submission.csv')","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2021-06-10T00:16:27.445578Z","iopub.execute_input":"2021-06-10T00:16:27.445984Z","iopub.status.idle":"2021-06-10T00:16:28.131012Z","shell.execute_reply.started":"2021-06-10T00:16:27.445944Z","shell.execute_reply":"2021-06-10T00:16:28.130028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ground_truth\np = pathlib.Path(INPUT)\ngt_files = list(p.glob('train/*/*/ground_truth.csv'))\n\ngts = []\nfor gt_file in gt_files:\n    gts.append(pd.read_csv(gt_file))\nground_truth = pd.concat(gts)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T00:16:28.132505Z","iopub.execute_input":"2021-06-10T00:16:28.132887Z","iopub.status.idle":"2021-06-10T00:16:29.221283Z","shell.execute_reply.started":"2021-06-10T00:16:28.132845Z","shell.execute_reply":"2021-06-10T00:16:29.220599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"# preparing data for viz\ntmp1 = ground_truth.copy()\ntmp1['phone'] = tmp1['collectionName'] + '_' + tmp1['phoneName']\ntmp1['phoneName'] = tmp1['phoneName'] + '_GT'\ntmp = train.append(tmp1)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T00:16:29.222663Z","iopub.execute_input":"2021-06-10T00:16:29.223008Z","iopub.status.idle":"2021-06-10T00:16:29.374583Z","shell.execute_reply.started":"2021-06-10T00:16:29.222982Z","shell.execute_reply":"2021-06-10T00:16:29.373866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_trafic(tmp[tmp['phone']=='2020-05-14-US-MTV-1_Pixel4XLModded'],\n                 center={\"lat\":37.6458, \"lon\":-122.4056}, zoom=19)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T00:16:29.375691Z","iopub.execute_input":"2021-06-10T00:16:29.376190Z","iopub.status.idle":"2021-06-10T00:16:30.417736Z","shell.execute_reply.started":"2021-06-10T00:16:29.376159Z","shell.execute_reply":"2021-06-10T00:16:30.416802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_trafic(tmp[tmp['phone']=='2020-06-04-US-MTV-1_Pixel4'],\n                 center={\"lat\":37.41634, \"lon\":-122.0805}, zoom=19)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T00:16:30.419197Z","iopub.execute_input":"2021-06-10T00:16:30.419582Z","iopub.status.idle":"2021-06-10T00:16:30.528701Z","shell.execute_reply.started":"2021-06-10T00:16:30.419553Z","shell.execute_reply":"2021-06-10T00:16:30.528095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As you can see in the graph above, the baseline coordinates seem to vary widely,  \neven though the car is stopped at the start or goal and the ground_truth coordinates have not changed.","metadata":{}},{"cell_type":"code","source":"ground_truth = ground_truth.rename(columns={'latDeg':'latDeg_gt', 'lngDeg':'lngDeg_gt', 'heightAboveWgs84EllipsoidM':'heightAboveWgs84EllipsoidM_gt'})\ntrain = train.merge(ground_truth, on=['collectionName', 'phoneName', 'millisSinceGpsEpoch'], how='inner')\ntrain['dist_err'] = calc_haversine(train['latDeg_gt'], train['lngDeg_gt'], train['latDeg'], train['lngDeg'])\ntrain = add_distance_diff(train)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T00:16:31.184510Z","iopub.execute_input":"2021-06-10T00:16:31.185004Z","iopub.status.idle":"2021-06-10T00:16:31.490831Z","shell.execute_reply.started":"2021-06-10T00:16:31.184971Z","shell.execute_reply":"2021-06-10T00:16:31.489886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['speedMps'].hist()\nplt.title('Distribution of \"speedMps\"')","metadata":{"execution":{"iopub.status.busy":"2021-06-10T00:18:19.709608Z","iopub.execute_input":"2021-06-10T00:18:19.709984Z","iopub.status.idle":"2021-06-10T00:18:19.869081Z","shell.execute_reply.started":"2021-06-10T00:18:19.709927Z","shell.execute_reply":"2021-06-10T00:18:19.868238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The speed of the train data is held by ground_truth.  \nLooking at this distribution, we can see that there are many records with speed = 0.","metadata":{}},{"cell_type":"code","source":"train.loc[train['speedMps']==0.0,'speed0'] = 'speed = 0'\ntrain.loc[train['speedMps']>0.0,'speed0'] = 'speed > 0'","metadata":{"execution":{"iopub.status.busy":"2021-06-10T00:20:41.013005Z","iopub.execute_input":"2021-06-10T00:20:41.013494Z","iopub.status.idle":"2021-06-10T00:20:41.042211Z","shell.execute_reply.started":"2021-06-10T00:20:41.013437Z","shell.execute_reply":"2021-06-10T00:20:41.041214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.boxplot(x='speed0', y='dist_err', data=train, showfliers = False)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T00:20:41.328187Z","iopub.execute_input":"2021-06-10T00:20:41.328652Z","iopub.status.idle":"2021-06-10T00:20:41.558123Z","shell.execute_reply.started":"2021-06-10T00:20:41.328608Z","shell.execute_reply":"2021-06-10T00:20:41.557430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We just looked at a few examples, but even when looking at the entire train data,  \nthe error appears to be large when the car is stopped.","metadata":{}},{"cell_type":"code","source":"def visualize_err_move_dist(df, phone, reject_outlier=True):\n    '''\n    visualize baseline error and relative move distance\n    '''\n    fig, axes = plt.subplots(figsize=(20, 10), nrows=2,sharex=True)\n    df = df[df['phone']==phone]\n    if reject_outlier:\n        th = (df['dist_err'].std() * 3) + df['dist_err'].mean()\n        df = df[df['dist_err']<th]\n    \n    axes[0].plot(df['millisSinceGpsEpoch'], df['dist_err'], label='err(baseline)')\n    axes[1].plot(df['millisSinceGpsEpoch'], df['speedMps'], label='speedMps')\n    axes[1].plot(df['millisSinceGpsEpoch'], df['dist_prev'], label='move dist(baseline)')\n    axes[1].plot(df['millisSinceGpsEpoch'], df['dist_gt_prev'], label='move dist(ground_truth)')\n    axes[0].legend(loc='upper right')\n    axes[1].legend(loc='upper right')\n    axes[0].grid(color='g', linestyle=':', linewidth=0.3)\n    axes[1].grid(color='g', linestyle=':', linewidth=0.3)\n    fig.suptitle(phone, fontsize=16)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T00:34:40.395012Z","iopub.execute_input":"2021-06-10T00:34:40.395489Z","iopub.status.idle":"2021-06-10T00:34:40.403751Z","shell.execute_reply.started":"2021-06-10T00:34:40.395458Z","shell.execute_reply":"2021-06-10T00:34:40.403112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_err_move_dist(train, '2020-05-14-US-MTV-1_Pixel4XLModded')","metadata":{"execution":{"iopub.status.busy":"2021-06-10T00:34:40.787551Z","iopub.execute_input":"2021-06-10T00:34:40.787878Z","iopub.status.idle":"2021-06-10T00:34:41.219359Z","shell.execute_reply.started":"2021-06-10T00:34:40.787850Z","shell.execute_reply":"2021-06-10T00:34:41.218457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's check the time series of baseline error and move distance  \nfor the example we just checked on the map.  \n\nAs you can see in the graph below,  \nthe baseline is moving a lot while it is actually stopped at the start and end points.  \n\nI am not sure about the cause,  \nbut I think this approach to reduce the error may be effective.\n\n(supplement)  \nmove_dist(ground_truth) and speedMps are almost identical.   \nIs the speed being calculated based on the coordinates?","metadata":{}},{"cell_type":"markdown","source":"View all phone results below","metadata":{}},{"cell_type":"code","source":"phones = train['phone'].unique()\nfor phone in phones:\n    visualize_err_move_dist(train, phone)","metadata":{"execution":{"iopub.status.busy":"2021-06-09T21:22:27.054895Z","iopub.execute_input":"2021-06-09T21:22:27.055383Z","iopub.status.idle":"2021-06-09T21:23:00.647596Z","shell.execute_reply.started":"2021-06-09T21:22:27.055345Z","shell.execute_reply":"2021-06-09T21:23:00.646725Z"},"trusted":true},"execution_count":null,"outputs":[]}]}