{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook creates a time lag table for each path in the test data.  \nThe time lag is obtained from subtracting timestamp (column No.0) from the values of below:\n<br/>\n<br/>\n- the path with any ibeacon data: column No.9 of \"TYPE_BEACON\"  \n  When the timestamp is subtracted, these values will be the same for each row.  \n  https://www.kaggle.com/jiweiliu/fix-the-timestamps-of-test-data-using-dask\n  <br/>  \n  <br/>  \n- the path without ibeacon data: column No.6 of \"TYPE_WIFI\" (\"lastseen_ts\")  \n  \"lastseen_ts minus timestamp\" varies for each row, so the maximum value (closest to zero) is adopted as the time lag to whole path. Also consider 200 milliseconds as the offset.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import sys\nimport numpy as np\nimport pandas as pd\nimport gc, glob, time, pickle\nfrom tqdm import tqdm\n\nimport matplotlib.pyplot as plt\nplt.style.use('seaborn-deep')\nplt.style.use('seaborn-darkgrid')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\"test_site_dict.pkl\" in the below cell is  the dictionary consists of wifi bssid and the ibeacon MAC address in the test data, and created with [this notebook](https://www.kaggle.com/horsek/ilnpre1-create-testsitedict).","metadata":{}},{"cell_type":"code","source":"input_dir = '../input/indoor-location-navigation'\n\nwith open('../input/iln-dataset/test_site_dict.pkl', 'rb') as f:\n    test_site_dict = pickle.load(f)\n\nfloor_map = {\"B2\":-2, \"B1\":-1,\n             \"F1\":0, \"F2\":1, \"F3\":2, \"F4\":3, \"F5\":4,\n             \"F6\":5, \"F7\":6, \"F8\":7, \"F9\":8,\n             \"1F\":0, \"2F\":1, \"3F\":2, \"4F\":3, \"5F\":4,\n             \"6F\":5, \"7F\":6, \"8F\":7, \"9F\":8}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nModify the host's code \"read_data_file\" function in \"io_f.py\"\nfor dealing with the malformed data etc.\n'''\n\nfrom dataclasses import dataclass\n\n@dataclass\nclass ReadData:\n    acce: np.ndarray\n    acce_uncali: np.ndarray\n    gyro: np.ndarray\n    gyro_uncali: np.ndarray\n    magn: np.ndarray\n    magn_uncali: np.ndarray\n    ahrs: np.ndarray\n    wifi: np.ndarray\n    ibeacon: np.ndarray\n    waypoint: np.ndarray\n\ndef split_list_as_req(line_data):\n    redo = False\n    data_BU = []\n    header_list = [i for i, itm in enumerate(line_data) if 'TYPE_' in itm]\n    if len(header_list) > 1:\n        data_BU = [line_data[header_list[1]-1][-13:]] + line_data[header_list[1]:]\n        line_data[header_list[1]-1] = line_data[header_list[1]-1][:-13]\n        line_data = line_data[:header_list[1]]\n        redo = True\n    return redo, line_data, data_BU\n\ndef read_data_file(data_filename):\n    acce = []\n    acce_uncali = []\n    gyro = []\n    gyro_uncali = []\n    magn = []\n    magn_uncali = []\n    ahrs = []\n    wifi = []\n    ibeacon = []\n    waypoint = []\n\n    with open(data_filename, 'r', encoding='utf-8') as file:\n        lines = file.readlines()\n\n    i = 0\n    redo = False\n    while i < len(lines):\n        if not redo:\n            line_data = lines[i]\n            line_data = line_data.strip()\n            if not line_data or line_data[0] == '#':\n                i += 1\n                continue\n            line_data = line_data.split('\\t')\n        else:\n            line_data = data_BU\n            redo = False\n\n        redo, line_data, data_BU = split_list_as_req(line_data)\n    \n        if line_data[1] == 'TYPE_ACCELEROMETER':\n            try:\n                acce.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            except ValueError:\n                print(data_filename)\n                print(line_data)\n\n        elif line_data[1] == 'TYPE_ACCELEROMETER_UNCALIBRATED':\n            acce_uncali.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n\n        elif line_data[1] == 'TYPE_GYROSCOPE':\n            gyro.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n\n        elif line_data[1] == 'TYPE_GYROSCOPE_UNCALIBRATED':\n            gyro_uncali.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n\n        elif line_data[1] == 'TYPE_MAGNETIC_FIELD':\n            magn.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n\n        elif line_data[1] == 'TYPE_MAGNETIC_FIELD_UNCALIBRATED':\n            magn_uncali.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n\n        elif line_data[1] == 'TYPE_ROTATION_VECTOR':\n            ahrs.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n\n        elif line_data[1] == 'TYPE_WIFI':\n            sys_ts = line_data[0]\n            ssid = line_data[2]\n            bssid = line_data[3]\n            rssi = line_data[4]\n            lastseen_ts = line_data[6]\n            frequency = line_data[5]\n            wifi_data = [sys_ts, ssid, bssid, rssi, lastseen_ts, frequency]\n            wifi.append(wifi_data)\n\n        elif line_data[1] == 'TYPE_BEACON':\n            ts = line_data[0]\n            uuid = line_data[2]\n            major = line_data[3]\n            minor = line_data[4]\n            rssi = line_data[6]\n            txpow = line_data[5]\n            distance = line_data[7]\n            mac = line_data[8]\n            if len(line_data)>9:\n                ts_copy = line_data[9]\n            else:\n                ts_copy = ts\n            ibeacon_data = [ts, '_'.join([uuid, major, minor]), rssi,\n                            txpow, distance, mac, ts_copy]\n            ibeacon.append(ibeacon_data)\n\n        elif line_data[1] == 'TYPE_WAYPOINT':\n            waypoint.append([int(line_data[0]), float(line_data[2]), float(line_data[3])])\n\n        if not redo:\n            i += 1\n            \n    acce = np.array(acce)\n    acce_uncali = np.array(acce_uncali)\n    gyro = np.array(gyro)\n    gyro_uncali = np.array(gyro_uncali)\n    magn = np.array(magn)\n    magn_uncali = np.array(magn_uncali)\n    ahrs = np.array(ahrs)\n    wifi = np.array(wifi)\n    ibeacon = np.array(ibeacon)\n    waypoint = np.array(waypoint)\n\n    return ReadData(acce, acce_uncali, gyro, gyro_uncali, magn, magn_uncali, ahrs, wifi, ibeacon, waypoint)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"''' Retrieve the Site ID from txt file '''\ndef SiteID(txt):\n    p1 = txt[1].find('SiteID:')+7\n    p2 = txt[1].find('\\tSiteName:')\n    assert p1!=-1+7 and p2!=-1, 'SiteID not found'\n    return txt[1][p1:p2]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('\\n=== Read Path Data ===')\ntest_files = sorted(glob.glob(f'{input_dir}/test/*.txt'))\nTestPathData,TestSiteName = {},{}\ntime.sleep(1)\nfor path_file_ in tqdm(test_files):\n    path_name = path_file_.split('/')[-1].replace('.txt','')\n    TestPathData[path_name] = read_data_file(path_file_)\n    \n    with open(path_file_, 'r', encoding=\"utf-8\") as f:\n        txt = f.readlines()\n    TestSiteName[path_name] = SiteID(txt)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"''' Create a time lag table (1) '''\ntime_lag_summary = {}\ntime.sleep(1)\nfor PathName, PathData in tqdm(TestPathData.items()):\n    site = TestSiteName[PathName]\n    \n    ''' wifi '''\n    df_wifi = pd.DataFrame(PathData.wifi,\n                           columns=['sys_ts','ssid','bssid','rssi',\n                                    'lastseen_ts','frequency'])\n    df_wifi['sys_ts']=df_wifi['sys_ts'].astype(np.int64)\n    df_wifi['lastseen_ts']=df_wifi['lastseen_ts'].astype(np.int64)\n    time_lag_wifi = (df_wifi['lastseen_ts']-df_wifi['sys_ts']+200).max()\n    \n    ''' beacon '''\n    if PathData.ibeacon.shape[0]==0:\n        time_lag_beac = [np.nan]\n    else:\n        df_beac = pd.DataFrame(PathData.ibeacon,\n                               columns=['ts','uuid_maj_min','rssi',\n                                        'txpow','distance','mac','ts_copy'])\n        df_beac['ts']=df_beac['ts'].astype(np.int64)\n        df_beac['ts_copy']=df_beac['ts_copy'].astype(np.int64)\n        time_lag_beac = (df_beac['ts_copy']-df_beac['ts']).unique().tolist()\n    \n    time_lag_summary[PathName] = [time_lag_wifi, time_lag_beac,\n                                  time_lag_wifi-time_lag_beac[0]]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"''' Create a time lag table (2) '''\ndf_summary = pd.DataFrame(time_lag_summary).T\ndf_summary.columns = ['time_lag_wifi','time_lag_beac','diff']\ndf_summary = df_summary.astype({'diff':float})\ndf_summary['beac_unique'] = df_summary['time_lag_beac'].apply(lambda x: len(x))\ndf_summary['time_lag']=df_summary.apply(lambda x:\n                                        x['time_lag_beac'][0]\n                                        if x['time_lag_beac']!=[np.nan]\n                                        else x['time_lag_wifi'], axis=1)\ndf_summary","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_summary['beac_unique'].unique()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The number of candidate time lags values obtained from beacon data is 1 for all paths.","metadata":{}},{"cell_type":"code","source":"df_summary['diff'].describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The median of \"diff\" values is almost zero, so 200 ms of offset value seems to be proper.","metadata":{}},{"cell_type":"code","source":"df_summary[['time_lag']].to_csv('test_ts_lag.csv')","metadata":{},"execution_count":null,"outputs":[]}]}