{"cells":[{"metadata":{},"cell_type":"markdown","source":"Many people noticed the test data has *fake* timestamps as discussed in this [thread](https://www.kaggle.com/c/indoor-location-navigation/discussion/218074). Depending on how the timestamp is used, this could be a big deal for your model. In my case, my RNN model's LB score is improved by 0.4 with fixing test data's timestamp only.\n\nIn this notebook, I will:\n* modify the `read_data_file` function from the host's github to read last timestamp of `ibeacon`\n* calculate the `gap` between the real timestamp and the `fake` timestamp from `ibeacon`. \n* use `dask` to recover the real timestamp of the test data in parallel with the `gap`."},{"metadata":{"trusted":true},"cell_type":"code","source":"! ls ../input/indoor-location-navigation","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"from glob import glob\nimport os\nimport sys\n\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\nfrom sklearn.preprocessing import LabelEncoder\nfrom glob import glob\nfrom dask.distributed import wait\n\nSENSORS = ['acce','acce_uncali','gyro',\n           'gyro_uncali','magn','magn_uncali','ahrs']\n\nNFEAS = {\n    'acce': 3,\n    'acce_uncali': 3,\n    'gyro': 3,\n    'gyro_uncali': 3,\n    'magn': 3,\n    'magn_uncali': 3,\n    'ahrs': 3,\n    'wifi': 1,\n    'ibeacon': 1,\n    'waypoint': 3\n}\n\nACOLS = ['timestamp','x','y','z']\n        \nFIELDS = {\n    'acce': ACOLS,\n    'acce_uncali': ACOLS,\n    'gyro': ACOLS,\n    'gyro_uncali': ACOLS,\n    'magn': ACOLS,\n    'magn_uncali': ACOLS,\n    'ahrs': ACOLS,\n    'wifi': ['timestamp','ssid','bssid','rssi','last_timestamp'],\n    'ibeacon': ['timestamp','code','rssi','last_timestamp'],\n    'waypoint': ['timestamp','x','y']\n}\n\ndef to_frame(data, col):\n    cols = FIELDS[col]\n    is_dummy = False\n    if data.shape[0]>0:\n        df = pd.DataFrame(data, columns=cols)\n    else:\n        df = create_dummy_df(cols)\n        is_dummy = True\n    for col in df.columns:\n        if 'timestamp' in col:\n            df[col] = df[col].astype('int64')\n    return df, is_dummy\n\ndef create_dummy_df(cols):\n    df = pd.DataFrame()\n    for col in cols:\n        df[col] = [0]\n        if col in ['ssid','bssid']:\n            df[col] = df[col].map(str)\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"from dataclasses import dataclass\n\nimport numpy as np\n\n\n@dataclass\nclass ReadData:\n    acce: np.ndarray\n    acce_uncali: np.ndarray\n    gyro: np.ndarray\n    gyro_uncali: np.ndarray\n    magn: np.ndarray\n    magn_uncali: np.ndarray\n    ahrs: np.ndarray\n    wifi: np.ndarray\n    ibeacon: np.ndarray\n    waypoint: np.ndarray\n\n\ndef read_data_file(data_filename):\n    acce = []\n    acce_uncali = []\n    gyro = []\n    gyro_uncali = []\n    magn = []\n    magn_uncali = []\n    ahrs = []\n    wifi = []\n    ibeacon = []\n    waypoint = []\n\n    with open(data_filename, 'r', encoding='utf-8') as file:\n        lines = file.readlines()\n\n    for line_data in lines:\n        line_data = line_data.strip()\n        if not line_data or line_data[0] == '#':\n            continue\n\n        line_data = line_data.split('\\t')\n\n        if line_data[1] == 'TYPE_ACCELEROMETER':\n            acce.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n\n        if line_data[1] == 'TYPE_ACCELEROMETER_UNCALIBRATED':\n            acce_uncali.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n\n        if line_data[1] == 'TYPE_GYROSCOPE':\n            gyro.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n\n        if line_data[1] == 'TYPE_GYROSCOPE_UNCALIBRATED':\n            gyro_uncali.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n\n        if line_data[1] == 'TYPE_MAGNETIC_FIELD':\n            magn.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n\n        if line_data[1] == 'TYPE_MAGNETIC_FIELD_UNCALIBRATED':\n            magn_uncali.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n\n        if line_data[1] == 'TYPE_ROTATION_VECTOR':\n            if len(line_data)>=5:\n                ahrs.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n\n        if line_data[1] == 'TYPE_WIFI':\n            sys_ts = line_data[0]\n            ssid = line_data[2]\n            bssid = line_data[3]\n            rssi = line_data[4]\n            lastseen_ts = line_data[6]\n            wifi_data = [sys_ts, ssid, bssid, rssi, lastseen_ts]\n            wifi.append(wifi_data)\n            continue\n\n        if line_data[1] == 'TYPE_BEACON':\n            ts = line_data[0]\n            uuid = line_data[2]\n            major = line_data[3]\n            minor = line_data[4]\n            rssi = line_data[6]\n            lastts = line_data[-1]\n            ibeacon_data = [ts, '_'.join([uuid, major, minor]), rssi, lastts]\n            ibeacon.append(ibeacon_data)\n            continue\n\n        if line_data[1] == 'TYPE_WAYPOINT':\n            waypoint.append([int(line_data[0]), float(line_data[2]), float(line_data[3])])\n\n    acce = np.array(acce)\n    acce_uncali = np.array(acce_uncali)\n    gyro = np.array(gyro)\n    gyro_uncali = np.array(gyro_uncali)\n    magn = np.array(magn)\n    magn_uncali = np.array(magn_uncali)\n    ahrs = np.array(ahrs)\n    wifi = np.array(wifi)\n    ibeacon = np.array(ibeacon)\n    waypoint = np.array(waypoint)\n\n    return ReadData(acce, acce_uncali, gyro, gyro_uncali, magn, magn_uncali, ahrs, wifi, ibeacon, waypoint)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The main changes made are these two lines:\n```\nlastts = line_data[-1] # last timestamp\nibeacon_data = [ts, '_'.join([uuid, major, minor]), rssi, lastts]\n```"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def get_test_dfs(PATH, test_files):\n    dtest = get_test_df(PATH)\n    buildings = set(dtest['building'].values.tolist())\n    dws = {}\n    ntest_files = []\n    for fname in tqdm(test_files):\n        path = fname.split('/')[-1].split('.')[0]\n        mask = dtest['path'] == path\n        dws[fname] = dtest.loc[mask, ['timestamp','x','y','floor','building','site_path_timestamp']].copy().reset_index(drop=True)\n        ntest_files.append(fname)\n    return dws\n\ndef get_test_df(PATH):\n    dtest = pd.read_csv(f'{PATH}/sample_submission.csv')\n    dtest['building'] = dtest['site_path_timestamp'].apply(lambda x: x.split('_')[0])\n    dtest['path'] = dtest['site_path_timestamp'].apply(lambda x: x.split('_')[1])\n    dtest['timestamp'] = dtest['site_path_timestamp'].apply(lambda x: x.split('_')[2])\n    dtest['timestamp'] = dtest['timestamp'].astype('int64')\n    dtest = dtest.sort_values(['path','timestamp']).reset_index(drop=True)\n    return dtest\n\ndef get_time_gap(name):\n    data = read_data_file(name)\n    db,no_ibeacon = to_frame(data.ibeacon,'ibeacon')\n    gap = db['last_timestamp'] - db['timestamp']\n    assert gap.unique().shape[0]==1\n    return gap.values[0],no_ibeacon\n\ndef fix_timestamp_test(df, gap):\n    df['real_timestamp'] = df['timestamp'] + gap\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import dask\nfrom dask.distributed import Client, wait, LocalCluster","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# set n_workers to number of cores\nclient = Client(n_workers=2, \n                threads_per_worker=1)\nclient","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Read data"},{"metadata":{"trusted":true},"cell_type":"code","source":"PATH = '../input/indoor-location-navigation'\n#train_files = glob(f'{PATH}/train/*/*/*.txt')\ndtest = get_test_df(PATH)\ntest_sites = dtest['building'].unique()\ntrain_files = []\nfor i in test_sites:\n    train_files.extend(glob(f'{PATH}/train/{i}/*/*.txt'))\ntest_files = glob(f'{PATH}/test/*.txt')\nlen(train_files),len(test_files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dfs = get_test_dfs(PATH, test_files)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"`test_dfs` is a dictionary which maps the file path to its waypoint dataframe."},{"metadata":{},"cell_type":"markdown","source":"### How to recover the real timestamp\n\nIn the [webinar](https://youtu.be/xt3OzMC-XMU?t=690), the host mentioned that for `ibeacon`, the `timestamp` and the `last_timestamp` are the same timestamps. We can verify this claim by checking the training ibeacon data. "},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = train_files[4]\ndata = read_data_file(fname)\ndb,no_ibeacon = to_frame(data.ibeacon,'ibeacon')\ndb.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"(db['timestamp']==db['last_timestamp']).all()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I also checked every other train files. The claim is true for all of them. Next, let's look at one test ibeacon data. "},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = test_files[0]\ndata = read_data_file(fname)\ndb,no_ibeacon = to_frame(data.ibeacon,'ibeacon')\ndb.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The `timestamp` and the `last_timestamp` are obviously different. But if we look closely, the gap between them are actually constant."},{"metadata":{"trusted":true},"cell_type":"code","source":"db['gap'] = db['last_timestamp'] - db['timestamp']\ndb['gap'].unique()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Hence, an intuitive guess is this `gap` is artificially introduced when preparing test data and we could use this `gap` to fix timestamps of `waypoints`, `wifi`, etc."},{"metadata":{},"cell_type":"markdown","source":"#### Fix one test waypoint"},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = test_files[0]\ngap,no_ibeacon = get_time_gap(fname)\ndf = fix_timestamp_test(test_dfs[fname], gap)\ndf[['timestamp','real_timestamp','site_path_timestamp']]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Fix all test waypoints using DASK"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfutures = []\nfor fname in tqdm(test_files, total=len(test_files)):\n    f = client.submit(get_time_gap,fname)\n    futures.append(f)\n\nfutures2 = []\nno_ibeacon_list = []\nfor f,fname in tqdm(zip(futures, test_files), total=len(test_files)):\n    gap,no_ibeacon = f.result()\n    no_ibeacon_list.append(no_ibeacon)\n    f = client.submit(fix_timestamp_test, test_dfs[fname], gap)\n    futures2.append(f)\n    \nfixed_test_dfs = {}\nfor f,fname in tqdm(zip(futures2, test_files), total=len(test_files)):\n    fixed_test_dfs[fname] = f.result()\n    \nfix_summary = pd.DataFrame({'file':test_files, 'no_ibeacon':no_ibeacon_list})\nfix_summary.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fix_summary['no_ibeacon'].mean()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**There are about 5% of test files without ibeacon data so these files still have incorrect timestamps. How to fix these data is the next question. Hopefully the host could respoind to this issue.**"},{"metadata":{},"cell_type":"markdown","source":"**Before fix**"},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = test_files[1]\ntest_dfs[fname].head()[['timestamp','site_path_timestamp']]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**After fix**"},{"metadata":{"trusted":true},"cell_type":"code","source":"fixed_test_dfs[fname].head()[['timestamp','real_timestamp','site_path_timestamp']]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"You can use the same method to fix test data `wifi` dataframes."}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}