{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"In addition to [fix timestamps of test set](https://www.kaggle.com/jiweiliu/fix-the-timestamps-of-test-data-using-dask) by [jiwei Liu](https://www.kaggle.com/jiweiliu).\n\nAbout 5% of test files lack of Ibeacon features, and we can use wifi last_timestamp to fix it.\n\nThen all data in test set have a real_timestamp.","metadata":{}},{"cell_type":"markdown","source":"However I have few idea about the sequential relationship in this competation. Could anyone teach me something? ","metadata":{}},{"cell_type":"markdown","source":"Many people noticed the test data has *fake* timestamps as discussed in this [thread](https://www.kaggle.com/c/indoor-location-navigation/discussion/218074). Depending on how the timestamp is used, this could be a big deal for your model. In my case, my RNN model's LB score is improved by 0.4 with fixing test data's timestamp only.\n\nIn this notebook, I will:\n* modify the `read_data_file` function from the host's github to read last timestamp of `ibeacon`\n* calculate the `gap` between the real timestamp and the `fake` timestamp from `ibeacon`. \n* use `dask` to recover the real timestamp of the test data in parallel with the `gap`.","metadata":{}},{"cell_type":"code","source":"! ls ../input/indoor-location-navigation","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from glob import glob\nimport os\nimport sys\n\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom dask.distributed import wait","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SENSORS = ['acce','acce_uncali','gyro',\n           'gyro_uncali','magn','magn_uncali','ahrs']\n\nNFEAS = {\n    'acce': 3,\n    'acce_uncali': 3,\n    'gyro': 3,\n    'gyro_uncali': 3,\n    'magn': 3,\n    'magn_uncali': 3,\n    'ahrs': 3,\n    'wifi': 1,\n    'ibeacon': 1,\n    'waypoint': 3\n}\n\nACOLS = ['timestamp','x','y','z']\n        \nFIELDS = {\n    'acce': ACOLS,\n    'acce_uncali': ACOLS,\n    'gyro': ACOLS,\n    'gyro_uncali': ACOLS,\n    'magn': ACOLS,\n    'magn_uncali': ACOLS,\n    'ahrs': ACOLS,\n    'wifi': ['timestamp','ssid','bssid','rssi','last_timestamp'],\n    'ibeacon': ['timestamp','code','rssi','last_timestamp'],\n    'waypoint': ['timestamp','x','y']\n}\n\ndef to_frame(data, col):\n    cols = FIELDS[col]\n    is_dummy = False\n    if data.shape[0]>0:\n        df = pd.DataFrame(data, columns=cols)\n    else:\n        df = create_dummy_df(cols)\n        is_dummy = True\n    for col in df.columns:\n        if 'timestamp' in col:\n            df[col] = df[col].astype('int64')\n    return df, is_dummy\n\ndef create_dummy_df(cols):\n    df = pd.DataFrame()\n    for col in cols:\n        df[col] = [0]\n        if col in ['ssid','bssid']:\n            df[col] = df[col].map(str)\n    return df","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from dataclasses import dataclass\n\n@dataclass\nclass ReadData:\n    acce: np.ndarray\n    acce_uncali: np.ndarray\n    gyro: np.ndarray\n    gyro_uncali: np.ndarray\n    magn: np.ndarray\n    magn_uncali: np.ndarray\n    ahrs: np.ndarray\n    wifi: np.ndarray\n    ibeacon: np.ndarray\n    waypoint: np.ndarray\n\n\ndef read_data_file(data_filename):\n    acce = []\n    acce_uncali = []\n    gyro = []\n    gyro_uncali = []\n    magn = []\n    magn_uncali = []\n    ahrs = []\n    wifi = []\n    ibeacon = []\n    waypoint = []\n\n    with open(data_filename, 'r', encoding='utf-8') as file:\n        lines = file.readlines()\n\n    for line_data in lines:\n        line_data = line_data.strip()\n        if not line_data or line_data[0] == '#':\n            continue\n\n        line_data = line_data.split('\\t')\n\n        if line_data[1] == 'TYPE_ACCELEROMETER':\n            acce.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n\n        if line_data[1] == 'TYPE_ACCELEROMETER_UNCALIBRATED':\n            acce_uncali.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n\n        if line_data[1] == 'TYPE_GYROSCOPE':\n            gyro.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n\n        if line_data[1] == 'TYPE_GYROSCOPE_UNCALIBRATED':\n            gyro_uncali.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n\n        if line_data[1] == 'TYPE_MAGNETIC_FIELD':\n            magn.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n\n        if line_data[1] == 'TYPE_MAGNETIC_FIELD_UNCALIBRATED':\n            magn_uncali.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n\n        if line_data[1] == 'TYPE_ROTATION_VECTOR':\n            if len(line_data)>=5:\n                ahrs.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n\n        if line_data[1] == 'TYPE_WIFI':\n            sys_ts = line_data[0]\n            ssid = line_data[2]\n            bssid = line_data[3]\n            rssi = line_data[4]\n            lastseen_ts = line_data[6]\n            wifi_data = [sys_ts, ssid, bssid, rssi, lastseen_ts]\n            wifi.append(wifi_data)\n            continue\n\n        if line_data[1] == 'TYPE_BEACON':\n            ts = line_data[0]\n            uuid = line_data[2]\n            major = line_data[3]\n            minor = line_data[4]\n            rssi = line_data[6]\n            lastts = line_data[-1]\n            ibeacon_data = [ts, '_'.join([uuid, major, minor]), rssi, lastts]\n            ibeacon.append(ibeacon_data)\n            continue\n\n        if line_data[1] == 'TYPE_WAYPOINT':\n            waypoint.append([int(line_data[0]), float(line_data[2]), float(line_data[3])])\n\n    acce = np.array(acce)\n    acce_uncali = np.array(acce_uncali)\n    gyro = np.array(gyro)\n    gyro_uncali = np.array(gyro_uncali)\n    magn = np.array(magn)\n    magn_uncali = np.array(magn_uncali)\n    ahrs = np.array(ahrs)\n    wifi = np.array(wifi)\n    ibeacon = np.array(ibeacon)\n    waypoint = np.array(waypoint)\n\n    return ReadData(acce, acce_uncali, gyro, gyro_uncali, magn, magn_uncali, ahrs, wifi, ibeacon, waypoint)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The main changes made are these two lines:\n```\nlastts = line_data[-1] # last timestamp\nibeacon_data = [ts, '_'.join([uuid, major, minor]), rssi, lastts]\n```","metadata":{}},{"cell_type":"code","source":"def get_test_dfs(PATH, test_files):\n    dtest = get_test_df(PATH)\n    buildings = set(dtest['building'].values.tolist())\n    dws = {}\n    ntest_files = []\n    for fname in tqdm(test_files):\n#       path即文件名\n        path = fname.split('/')[-1].split('.')[0]\n        mask = dtest['path'] == path\n        dws[fname] = dtest.loc[mask, ['timestamp','x','y','floor','building','site_path_timestamp']].copy().reset_index(drop=True)\n        ntest_files.append(fname)\n    return dws\n\ndef get_test_df(PATH):\n    dtest = pd.read_csv(f'{PATH}/sample_submission.csv')\n    dtest['building'] = dtest['site_path_timestamp'].apply(lambda x: x.split('_')[0])\n    dtest['path'] = dtest['site_path_timestamp'].apply(lambda x: x.split('_')[1])\n    dtest['timestamp'] = dtest['site_path_timestamp'].apply(lambda x: x.split('_')[2])\n    dtest['timestamp'] = dtest['timestamp'].astype('int64')\n    dtest = dtest.sort_values(['path','timestamp']).reset_index(drop=True)\n    return dtest\n\ndef get_time_gap(name):\n    data = read_data_file(name)\n    db,no_ibeacon = to_frame(data.ibeacon,'ibeacon')\n    gap = db['last_timestamp'] - db['timestamp']\n    assert gap.unique().shape[0]==1\n    return gap.values[0],no_ibeacon\n\ndef fix_timestamp_test(df, gap):\n    df['real_timestamp'] = df['timestamp'] + gap\n    return df","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import dask\nfrom dask.distributed import Client, wait, LocalCluster","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# set n_workers to number of cores\nclient = Client(n_workers=2, \n                threads_per_worker=1)\nclient","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Read data","metadata":{}},{"cell_type":"code","source":"PATH = '../input/indoor-location-navigation'\n#train_files = glob(f'{PATH}/train/*/*/*.txt')\ndtest = get_test_df(PATH)\ntest_sites = dtest['building'].unique()\ntrain_files = []\nfor i in test_sites:\n    train_files.extend(glob(f'{PATH}/train/{i}/*/*.txt'))\ntest_files = glob(f'{PATH}/test/*.txt')\nlen(train_files),len(test_files)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dfs = get_test_dfs(PATH, test_files)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"`test_dfs` is a dictionary which maps the file path to its waypoint dataframe.","metadata":{}},{"cell_type":"markdown","source":"### How to recover the real timestamp\n\nIn the [webinar](https://youtu.be/xt3OzMC-XMU?t=690), the host mentioned that for `ibeacon`, the `timestamp` and the `last_timestamp` are the same timestamps. We can verify this claim by checking the training ibeacon data. ","metadata":{}},{"cell_type":"code","source":"fname = train_files[0]\ndata = read_data_file(fname)\ndb,no_ibeacon = to_frame(data.ibeacon,'ibeacon')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(db['timestamp']==db['last_timestamp']).all()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I also checked every other train files. The claim is true for all of them. Next, let's look at one test ibeacon data. ","metadata":{}},{"cell_type":"code","source":"fname = test_files[0]\ndata = read_data_file(fname)\ndb,no_ibeacon = to_frame(data.ibeacon,'ibeacon')\ndb.head()","metadata":{"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The `timestamp` and the `last_timestamp` are obviously different. But if we look closely, the gap between them are actually constant.","metadata":{}},{"cell_type":"code","source":"db['gap'] = db['last_timestamp'] - db['timestamp']\ndb['gap'].unique()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Hence, an intuitive guess is this `gap` is artificially introduced when preparing test data and we could use this `gap` to fix timestamps of `waypoints`, `wifi`, etc.","metadata":{}},{"cell_type":"markdown","source":"#### Fix one test waypoint","metadata":{}},{"cell_type":"code","source":"fname = test_files[0]\ngap,no_ibeacon = get_time_gap(fname)\ndf = fix_timestamp_test(test_dfs[fname], gap)\ndf[['timestamp','real_timestamp','site_path_timestamp']]","metadata":{"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Fix all test waypoints using DASK","metadata":{}},{"cell_type":"code","source":"%%time\nfutures = []\nfor fname in tqdm(test_files, total=len(test_files)):\n    f = client.submit(get_time_gap,fname)\n    futures.append(f)\n\nfutures2 = []\nno_ibeacon_list = []\nfor f,fname in tqdm(zip(futures, test_files), total=len(test_files)):\n    gap,no_ibeacon = f.result()\n    no_ibeacon_list.append(no_ibeacon)\n    f = client.submit(fix_timestamp_test, test_dfs[fname], gap)\n    futures2.append(f)\n    \nfixed_test_dfs = {}\nfor f,fname in tqdm(zip(futures2, test_files), total=len(test_files)):\n    fixed_test_dfs[fname] = f.result()\n    \nfix_summary = pd.DataFrame({'file':test_files, 'no_ibeacon':no_ibeacon_list})\nfix_summary.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fix_summary['no_ibeacon'].mean()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fixed_test_dfs[test_files[0]][['timestamp','real_timestamp']].head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fixed_test_dfs[test_files[4]][['timestamp','real_timestamp']].head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**There are about 5% of test files without ibeacon data so these files still have incorrect timestamps. How to fix these data is the next question. Hopefully the host could respoind to this issue.**","metadata":{}},{"cell_type":"markdown","source":"### Use Wifi last_timestamp to fix test files timestamp without ibeacon","metadata":{}},{"cell_type":"markdown","source":"compare with ibeacon, wifi_timestamp is less accurate","metadata":{}},{"cell_type":"code","source":"for no_ibeacon in fix_summary.loc[fix_summary['no_ibeacon']==True].index:\n    fname = test_files[no_ibeacon]\n    data = read_data_file(fname)\n    cols = FIELDS['wifi']\n    db = pd.DataFrame(data.wifi, columns=cols)\n    for col in db.columns:\n        if 'timestamp' in col:\n            db[col] = db[col].astype('int64')\n\n    nearest_time_list = []\n    nearest_real_time_list = []\n\n    for test_t in fixed_test_dfs[fname]['timestamp']:\n        dists = []\n        for t in db['timestamp'].unique():\n            dists.append((t - test_t))\n        nearest_index = np.argmin(abs(np.array(dists)))\n        nearest_time_list.append(db['timestamp'].unique()[nearest_index])\n\n    difference = [nearest_time_list[i] - list(fixed_test_dfs[fname]['timestamp'])[i] for i in range(len(nearest_time_list))]\n    for i in range(len(nearest_time_list)):\n        nearest_real_time_list.append(np.max(db[db['timestamp'] == nearest_time_list[i]]['last_timestamp']))\n    fixed_test_dfs[fname]['real_timestamp'] = np.array([nearest_real_time_list[i]-difference[i] for i in range(len(nearest_real_time_list))])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The result of using WIFI to fix timestamps seems to be OK.","metadata":{}},{"cell_type":"code","source":"fixed_test_dfs[test_files[4]]['real_timestamp'] - fixed_test_dfs[test_files[4]]['timestamp'] - 1574135710654","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Before fix**","metadata":{}},{"cell_type":"code","source":"fname = test_files[4]\ntest_dfs[fname].head()[['timestamp','site_path_timestamp']]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**After fix**","metadata":{}},{"cell_type":"code","source":"fixed_test_dfs[fname].head()[['timestamp','real_timestamp','site_path_timestamp']]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_fix_test_dfs = pd.concat([fixed_test_dfs[test_files[i]] for i in range(len(test_files))])\nall_fix_test_dfs = all_fix_test_dfs.set_index('site_path_timestamp')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_fix_test_dfs['real_timestamp'].sort_index().to_csv('fix_test_timestamp.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"You can use the same method to fix test data `wifi` dataframes.","metadata":{}}]}