{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from pathlib import Path\nimport pandas as pd\nfrom tqdm.notebook import tqdm\nimport numpy as np\n","metadata":{"execution":{"iopub.status.busy":"2021-07-05T08:54:59.295735Z","iopub.execute_input":"2021-07-05T08:54:59.296181Z","iopub.status.idle":"2021-07-05T08:54:59.307938Z","shell.execute_reply.started":"2021-07-05T08:54:59.296118Z","shell.execute_reply":"2021-07-05T08:54:59.307054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# helper function to load GNSS logs as dataframe","metadata":{}},{"cell_type":"code","source":"def gnss_log_to_raw_dataframe(path):\n    path_split = str(path).split('/')\n    collection_name = path_split[-3]\n    phone = path_split[-2]\n    \n    gnss_section_names = {'Raw','UncalAccel', 'UncalGyro', 'UncalMag', 'Fix', 'Status', 'OrientationDeg'}\n    with open(path) as f_open:\n        datalines = f_open.readlines()\n\n    datas = {k: [] for k in gnss_section_names}\n    gnss_map = {k: [] for k in gnss_section_names}\n    for dataline in datalines:\n        is_header = dataline.startswith('#')\n        dataline = dataline.strip('#').strip().split(',')\n        # skip over notes, version numbers, etc\n        if is_header and dataline[0] in gnss_section_names:\n            try:\n                gnss_map[dataline[0]] = dataline[1:]\n            except:\n                pass\n        elif not is_header:\n            try:\n                datas[dataline[0]].append(dataline[1:])\n            except:\n                pass\n    results = dict()\n    for k, v in datas.items():\n        results[k] = pd.DataFrame(v, columns=gnss_map[k])\n    # pandas doesn't properly infer types from these lists by default\n    for k, df in results.items():\n        for col in df.columns:\n            if col == 'CodeType':\n                continue\n            try:\n                results[k][col] = pd.to_numeric(results[k][col])\n            except:\n                pass\n    raw_df = results['Raw']       \n    raw_df['collectionName'] = collection_name\n    raw_df['phoneName'] = phone\n    \n    return raw_df","metadata":{"execution":{"iopub.status.busy":"2021-07-05T08:54:59.312731Z","iopub.execute_input":"2021-07-05T08:54:59.313280Z","iopub.status.idle":"2021-07-05T08:54:59.326132Z","shell.execute_reply.started":"2021-07-05T08:54:59.313234Z","shell.execute_reply":"2021-07-05T08:54:59.325019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"# reload df and groundtruth\nn_files=73\ndatapath = Path('../input/google-smartphone-decimeter-challenge/')\n\nground_truths = (datapath / \"train\").rglob(\"ground_truth.csv\")\nderived_files = (datapath / \"train\").rglob(\"*_derived.csv\")\nraw_files = (datapath / \"train\").rglob(\"*_GnssLog.txt\")\n\ndf_gt = pd.concat([pd.read_csv(filepath) for filepath in tqdm(ground_truths, total=n_files, desc=\"Reading ground truth data\")], ignore_index=True)\ndf_raw_train = pd.concat([gnss_log_to_raw_dataframe(filepath) for filepath in tqdm(raw_files, total=n_files, desc=\"Reading raw data\")], ignore_index=True)\ndf_derived = pd.concat([pd.read_csv(filepath) for filepath in tqdm(derived_files, total=n_files, desc=\"Loading data\")], ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-05T08:58:16.888141Z","iopub.execute_input":"2021-07-05T08:58:16.888567Z","iopub.status.idle":"2021-07-05T09:15:27.250889Z","shell.execute_reply.started":"2021-07-05T08:58:16.888528Z","shell.execute_reply":"2021-07-05T09:15:27.249959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Apply tip 1","metadata":{}},{"cell_type":"code","source":"df_raw_train['MillisSinceGpsEpoch'] = np.floor( (df_raw_train['TimeNanos'] - df_raw_train['FullBiasNanos']) / 1000000.0).astype(int)\n\ndf_derived_corrected = pd.DataFrame() \nfor indexes, subdf in df_derived.groupby(['collectionName', 'phoneName']):\n    df_raw_sub = df_raw_train[(df_raw_train['collectionName']==indexes[0])&(df_raw_train['phoneName']==indexes[1])]\n    \n    # Change each value in df_derived['MillisSinceGpsEpoch'] to be the prior epoch.\n    raw_timestamps = df_raw_sub['MillisSinceGpsEpoch'].unique()\n    derived_timestamps = subdf['millisSinceGpsEpoch'].unique()\n\n    # The timestamps in derived are one epoch ahead. We need to map each epoch\n    # in derived to the prior one (in Raw).\n    indexes = np.searchsorted(raw_timestamps, derived_timestamps)\n    from_t_to_fix_derived = dict(zip(derived_timestamps, raw_timestamps[indexes-1]))\n    subdf['millisSinceGpsEpoch'] = np.array(list(map(lambda v: from_t_to_fix_derived[v], subdf['millisSinceGpsEpoch'])))\n\n    df_derived_corrected = pd.concat([df_derived_corrected, subdf])\n    \n\n","metadata":{"execution":{"iopub.status.busy":"2021-07-05T09:17:42.432100Z","iopub.execute_input":"2021-07-05T09:17:42.432708Z","iopub.status.idle":"2021-07-05T09:19:59.679150Z","shell.execute_reply.started":"2021-07-05T09:17:42.432673Z","shell.execute_reply":"2021-07-05T09:19:59.677886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Apply tip 5","metadata":{}},{"cell_type":"code","source":"delta_millis = df_derived_corrected['millisSinceGpsEpoch'] - df_derived_corrected['receivedSvTimeInGpsNanos'] / 1e6\nwhere_good_signals = (delta_millis > 0) & (delta_millis < 300)\ndf_derived_corrected_filtered = df_derived_corrected[where_good_signals].copy()\n","metadata":{"execution":{"iopub.status.busy":"2021-07-05T09:19:59.680989Z","iopub.execute_input":"2021-07-05T09:19:59.681335Z","iopub.status.idle":"2021-07-05T09:20:01.005191Z","shell.execute_reply.started":"2021-07-05T09:19:59.681300Z","shell.execute_reply":"2021-07-05T09:20:01.004080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_derived_corrected.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-05T09:20:01.006771Z","iopub.execute_input":"2021-07-05T09:20:01.007075Z","iopub.status.idle":"2021-07-05T09:20:01.015678Z","shell.execute_reply.started":"2021-07-05T09:20:01.007047Z","shell.execute_reply":"2021-07-05T09:20:01.014379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_derived_corrected_filtered.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-05T09:20:01.017684Z","iopub.execute_input":"2021-07-05T09:20:01.018135Z","iopub.status.idle":"2021-07-05T09:20:01.032338Z","shell.execute_reply.started":"2021-07-05T09:20:01.018087Z","shell.execute_reply":"2021-07-05T09:20:01.031532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}