{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-06T21:46:29.104636Z","iopub.execute_input":"2021-07-06T21:46:29.105328Z","iopub.status.idle":"2021-07-06T21:46:30.360248Z","shell.execute_reply.started":"2021-07-06T21:46:29.105268Z","shell.execute_reply":"2021-07-06T21:46:30.35909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainfile = pd.read_csv(\"../input/google-smartphone-decimeter-challenge/baseline_locations_train.csv\")\ntrainfile","metadata":{"execution":{"iopub.status.busy":"2021-07-06T21:46:30.362159Z","iopub.execute_input":"2021-07-06T21:46:30.362531Z","iopub.status.idle":"2021-07-06T21:46:30.748173Z","shell.execute_reply.started":"2021-07-06T21:46:30.362499Z","shell.execute_reply":"2021-07-06T21:46:30.746968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testfile = pd.read_csv(\"../input/google-smartphone-decimeter-challenge/baseline_locations_test.csv\")\ntestfile.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-06T21:46:30.75035Z","iopub.execute_input":"2021-07-06T21:46:30.750668Z","iopub.status.idle":"2021-07-06T21:46:30.998121Z","shell.execute_reply.started":"2021-07-06T21:46:30.750636Z","shell.execute_reply":"2021-07-06T21:46:30.997133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(\"../input/google-smartphone-decimeter-challenge/sample_submission.csv\")\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-06T21:46:31.458164Z","iopub.execute_input":"2021-07-06T21:46:31.458541Z","iopub.status.idle":"2021-07-06T21:46:31.637258Z","shell.execute_reply.started":"2021-07-06T21:46:31.458506Z","shell.execute_reply":"2021-07-06T21:46:31.636281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for file in [trainfile, testfile, submission]:\n    print(file.shape)","metadata":{"execution":{"iopub.status.busy":"2021-07-06T21:46:32.594306Z","iopub.execute_input":"2021-07-06T21:46:32.594683Z","iopub.status.idle":"2021-07-06T21:46:32.601297Z","shell.execute_reply.started":"2021-07-06T21:46:32.594649Z","shell.execute_reply":"2021-07-06T21:46:32.599964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainfile['phone'].unique()[9] # The tenth trip","metadata":{"execution":{"iopub.status.busy":"2021-07-06T21:47:15.220052Z","iopub.execute_input":"2021-07-06T21:47:15.220442Z","iopub.status.idle":"2021-07-06T21:47:15.241766Z","shell.execute_reply.started":"2021-07-06T21:47:15.220412Z","shell.execute_reply":"2021-07-06T21:47:15.240206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainfile['phone'].unique()[19] # The tenth trip","metadata":{"execution":{"iopub.status.busy":"2021-07-06T21:47:43.288422Z","iopub.execute_input":"2021-07-06T21:47:43.288863Z","iopub.status.idle":"2021-07-06T21:47:43.312581Z","shell.execute_reply.started":"2021-07-06T21:47:43.288827Z","shell.execute_reply":"2021-07-06T21:47:43.311297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's look at a particular cellphone user in the training data\nphone_example = trainfile.loc[0, 'phone']\nprint(\"Phone:\", phone_example)\nsubset = trainfile[trainfile['phone'] == phone_example].copy()\nprint(\"Subset size:\", subset.shape)\nsubset.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-24T21:12:05.49478Z","iopub.execute_input":"2021-06-24T21:12:05.495327Z","iopub.status.idle":"2021-06-24T21:12:05.54238Z","shell.execute_reply.started":"2021-06-24T21:12:05.495272Z","shell.execute_reply":"2021-06-24T21:12:05.541333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install gwpy","metadata":{"execution":{"iopub.status.busy":"2021-06-24T21:02:58.645256Z","iopub.execute_input":"2021-06-24T21:02:58.645667Z","iopub.status.idle":"2021-06-24T21:03:18.376205Z","shell.execute_reply.started":"2021-06-24T21:02:58.645587Z","shell.execute_reply":"2021-06-24T21:03:18.375179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from gwpy.time import tconvert\nsubset['datetime'] = (subset['millisSinceGpsEpoch'] / 1000).apply(tconvert)\nsubset.head(20)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T21:38:21.152684Z","iopub.execute_input":"2021-06-21T21:38:21.153139Z","iopub.status.idle":"2021-06-21T21:38:22.145895Z","shell.execute_reply.started":"2021-06-21T21:38:21.153048Z","shell.execute_reply":"2021-06-21T21:38:22.14482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import folium\nfolium.__version__","metadata":{"execution":{"iopub.status.busy":"2021-06-21T21:39:38.248761Z","iopub.execute_input":"2021-06-21T21:39:38.249132Z","iopub.status.idle":"2021-06-21T21:39:38.2551Z","shell.execute_reply.started":"2021-06-21T21:39:38.249095Z","shell.execute_reply":"2021-06-21T21:39:38.253844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_map = folium.Map(location=[37.42, -122.09], zoom_start=10)\n\nsample_freq = 100\n\ncount = 0\nfor row in subset.index:\n    if count % sample_freq == 0:\n        latitude = subset.loc[row, 'latDeg']\n        longitude = subset.loc[row, 'lngDeg']\n        message = str(row) + \", \" + str(subset.loc[row, 'datetime'])\n        point = folium.Marker(location=[latitude, longitude], popup=message)\n        point.add_to(path_map)\n    count += 1\n    \npath_map","metadata":{"execution":{"iopub.status.busy":"2021-06-21T21:40:33.382904Z","iopub.execute_input":"2021-06-21T21:40:33.38359Z","iopub.status.idle":"2021-06-21T21:40:33.432992Z","shell.execute_reply.started":"2021-06-21T21:40:33.383533Z","shell.execute_reply":"2021-06-21T21:40:33.431844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Additional data about this user\n!tree /kaggle/input/google-smartphone-decimeter-challenge/train/2020-05-14-US-MTV-1/Pixel4/","metadata":{"execution":{"iopub.status.busy":"2021-06-21T20:26:54.279896Z","iopub.execute_input":"2021-06-21T20:26:54.280267Z","iopub.status.idle":"2021-06-21T20:26:55.066474Z","shell.execute_reply.started":"2021-06-21T20:26:54.280236Z","shell.execute_reply":"2021-06-21T20:26:55.065403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '/kaggle/input/google-smartphone-decimeter-challenge/train/2020-05-14-US-MTV-1/Pixel4/'\nsubset_ground_truth = pd.read_csv(path + 'ground_truth.csv')\nsubset_ground_truth.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-21T21:46:14.030113Z","iopub.execute_input":"2021-06-21T21:46:14.03053Z","iopub.status.idle":"2021-06-21T21:46:14.065095Z","shell.execute_reply.started":"2021-06-21T21:46:14.030493Z","shell.execute_reply":"2021-06-21T21:46:14.063776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# How much difference is there?\n# Multiply the degrees of separation of longitude and latitude by 111,139 to get the corresponding linear distances in meters.\n# https://sciencing.com/convert-latitude-longtitude-feet-2724.html\nsubset['latDiff'] = (subset['latDeg'] - subset_ground_truth['latDeg']) * 1111390 # centimeters\nsubset['lngDiff'] = (subset['lngDeg'] - subset_ground_truth['lngDeg']) * 1111390 \nsubset['Diff'] = (subset['latDiff'] ** 2 + subset['lngDiff'] ** 2) ** 0.5\nsubset.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-21T20:42:27.391112Z","iopub.execute_input":"2021-06-21T20:42:27.391728Z","iopub.status.idle":"2021-06-21T20:42:27.410438Z","shell.execute_reply.started":"2021-06-21T20:42:27.391693Z","shell.execute_reply":"2021-06-21T20:42:27.409543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subset['Diff'].hist(bins=50)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T20:43:05.350621Z","iopub.execute_input":"2021-06-21T20:43:05.351087Z","iopub.status.idle":"2021-06-21T20:43:05.783405Z","shell.execute_reply.started":"2021-06-21T20:43:05.351056Z","shell.execute_reply":"2021-06-21T20:43:05.782488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"threshold = 50\noutliers = subset[subset['Diff'] > threshold]\noutliers","metadata":{"execution":{"iopub.status.busy":"2021-06-21T21:50:09.626517Z","iopub.execute_input":"2021-06-21T21:50:09.627041Z","iopub.status.idle":"2021-06-21T21:50:09.649394Z","shell.execute_reply.started":"2021-06-21T21:50:09.627002Z","shell.execute_reply":"2021-06-21T21:50:09.6485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_map = folium.Map(location=[37.42, -122.09], zoom_start=10)\n\nsample_freq = 100\n\ncount = 0\nfor row in subset.index:\n    if count % sample_freq == 0:\n        latitude = subset_ground_truth.loc[row, 'latDeg']\n        longitude = subset_ground_truth.loc[row, 'lngDeg']\n        message = str(row)\n        point = folium.Marker(location=[latitude, longitude], popup=message)\n        point.add_to(path_map)\n    count += 1\n    \npath_map","metadata":{"execution":{"iopub.status.busy":"2021-06-21T21:51:06.378627Z","iopub.execute_input":"2021-06-21T21:51:06.379185Z","iopub.status.idle":"2021-06-21T21:51:06.421183Z","shell.execute_reply.started":"2021-06-21T21:51:06.379129Z","shell.execute_reply":"2021-06-21T21:51:06.420339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_map = folium.Map(location=[37.42, -122.09], zoom_start=10)\n\nfor row in outliers.index:\n    latitude = subset.loc[row, 'latDeg']\n    longitude = subset.loc[row, 'lngDeg']\n    message = str(row) + \" - data\"\n    point = folium.Marker(location=[latitude, longitude], popup=message)\n    point.add_to(path_map)\n\n    latitude = subset_ground_truth.loc[row, 'latDeg']\n    longitude = subset_ground_truth.loc[row, 'lngDeg']\n    message = str(row) + \" - ground truth\"\n    point = folium.Marker(location=[latitude, longitude], popup=message)\n    point.add_to(path_map)\n    \npath_map","metadata":{"execution":{"iopub.status.busy":"2021-06-21T21:51:38.245673Z","iopub.execute_input":"2021-06-21T21:51:38.246078Z","iopub.status.idle":"2021-06-21T21:51:38.281761Z","shell.execute_reply.started":"2021-06-21T21:51:38.24604Z","shell.execute_reply":"2021-06-21T21:51:38.280788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fname = path + 'Pixel4_GnssLog.txt'\nlines_to_display = 30\nline_count = 0\nwith open(fname, 'r') as file:\n    for line in file:\n        print(line)\n        line_count += 1\n        if line_count > lines_to_display:\n            break","metadata":{"execution":{"iopub.status.busy":"2021-06-21T20:03:38.591742Z","iopub.execute_input":"2021-06-21T20:03:38.592134Z","iopub.status.idle":"2021-06-21T20:03:38.600694Z","shell.execute_reply.started":"2021-06-21T20:03:38.592105Z","shell.execute_reply":"2021-06-21T20:03:38.599722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subset_derived = pd.read_csv(path + 'Pixel4_derived.csv')\nsubset_derived.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-21T20:03:38.405632Z","iopub.execute_input":"2021-06-21T20:03:38.40599Z","iopub.status.idle":"2021-06-21T20:03:38.590287Z","shell.execute_reply.started":"2021-06-21T20:03:38.405964Z","shell.execute_reply":"2021-06-21T20:03:38.589552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fname = path + 'supplemental/Pixel4_GnssLog.20o'\nlines_to_display = 30\nline_count = 0\nwith open(fname, 'r') as file:\n    for line in file:\n        print(line)\n        line_count += 1\n        if line_count > lines_to_display:\n            break","metadata":{"execution":{"iopub.status.busy":"2021-06-21T20:27:40.38363Z","iopub.execute_input":"2021-06-21T20:27:40.383988Z","iopub.status.idle":"2021-06-21T20:27:40.414262Z","shell.execute_reply.started":"2021-06-21T20:27:40.383954Z","shell.execute_reply":"2021-06-21T20:27:40.413408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fname = path + 'supplemental/SPAN_Pixel4_10Hz.nmea'\nlines_to_display = 30\nline_count = 0\nwith open(fname, 'r') as file:\n    for line in file:\n        print(line)\n        line_count += 1\n        if line_count > lines_to_display:\n            break","metadata":{"execution":{"iopub.status.busy":"2021-06-21T20:28:12.365032Z","iopub.execute_input":"2021-06-21T20:28:12.365366Z","iopub.status.idle":"2021-06-21T20:28:12.389466Z","shell.execute_reply.started":"2021-06-21T20:28:12.365335Z","shell.execute_reply":"2021-06-21T20:28:12.388601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Worst-case submission\nsubmission[['latDeg', 'lngDeg']] = testfile[['latDeg', 'lngDeg']]\nsubmission.to_csv(\"worst_case_submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T20:35:34.081896Z","iopub.execute_input":"2021-06-21T20:35:34.082435Z","iopub.status.idle":"2021-06-21T20:35:34.425337Z","shell.execute_reply.started":"2021-06-21T20:35:34.082391Z","shell.execute_reply":"2021-06-21T20:35:34.424539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-21T20:34:53.591049Z","iopub.execute_input":"2021-06-21T20:34:53.591421Z","iopub.status.idle":"2021-06-21T20:34:53.605695Z","shell.execute_reply.started":"2021-06-21T20:34:53.591387Z","shell.execute_reply":"2021-06-21T20:34:53.604367Z"},"trusted":true},"execution_count":null,"outputs":[]}]}