{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook improve [previous score using leak](https://www.kaggle.com/tomooinubushi/postprocessing-based-on-leakage)(LB: 4.718 -> 4.683).","metadata":{}},{"cell_type":"markdown","source":"This notebook aims to improve score considering device id and time differences between paths.\nBasic idea is described in [chris's great discussion](https://www.kaggle.com/c/indoor-location-navigation/discussion/234543).\n\n> 3. Device leakage\n>\n> This is the data leak that I don't think has been talked about on this forum before.\n>\n> If you look at the uncalibrated sensor information, you might see a row like:\n>\n> 0000000000276 TYPE_MAGNETIC_FIELD_UNCALIBRATED -106.762695 35.142517 -355.44434 -83.41217 16.18042 -325.82092 3\n>\n> Those last 3 float values (before the accuracy) are device specific. Since the same devices are used for the training and test paths, it SEEMS like this is a huge leak - because combining with leak #1, I can almost exactly specify what device was recording which path at what time. (and so exactly specify the test floors)\n\nThanks to his discussion, I found good postprocess based on leakage considering device id.\n\nRespecting to chris' attitude to open leaks, I also publish this notebook.","metadata":{}},{"cell_type":"markdown","source":"In this notebook, each device id is defined by sum of magn biases(x, y, and z), which are recorded in sensor data (TYPE_MAGNETIC_FIELD_UNCALIBRATED).\n\nIn some path, however, magn bias is not recorded.\n\nIn such case, calculate the difference betwenn calibrated and uncalibrated and we define it as device ID.\n\nAlthough the bias is not exactly same along a path, there is not so large difference.","metadata":{}},{"cell_type":"code","source":"import os\nimport glob\nimport numpy as np\nimport pandas as pd \nimport math\nimport sys\n\nfrom dataclasses import dataclass\n\nimport warnings # Supress warnings \nwarnings.filterwarnings('ignore')\nfrom matplotlib import pyplot as plt\n\nfrom tqdm.notebook import tqdm\n\nimport json\nfrom datetime import datetime\nimport glob","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@dataclass\nclass ReadData:\n    acce: np.ndarray\n    acce_uncali: np.ndarray\n    gyro: np.ndarray\n    gyro_uncali: np.ndarray\n    magn: np.ndarray\n    magn_uncali: np.ndarray\n    ahrs: np.ndarray\n    wifi: np.ndarray\n    ibeacon: np.ndarray\n    waypoint: np.ndarray\n    starttime: int\n    endtime: int\n\ndef read_data_file(data_filename):\n    acce = []\n    acce_uncali = []\n    gyro = []\n    gyro_uncali = []\n    magn = []\n    magn_uncali = []\n    ahrs = []\n    wifi = []\n    ibeacon = []\n    waypoint = []\n    starttime = -1\n    endtime = -1\n\n    with open(data_filename, 'r', encoding='utf-8') as file:\n        lines = file.readlines()\n\n    for line_data in lines:\n        line_data = line_data.strip()\n        if not line_data: continue\n        if len(line_data) > 20: \n            if line_data[2:7] == 'start':\n                if len(line_data.split(':')) > 1: starttime = int(line_data.split(':')[1])\n                else: starttime = int(line_data.split('\\t')[2])\n                continue   \n            if line_data[2:5] == 'end':\n                if len(line_data.split(':')) > 1: endtime = int(line_data.split(':')[1])\n                else: endtime = int(line_data.split('\\t')[2])\n                continue \n\n        if line_data[0] == '#': continue\n\n        line_data = line_data.split('\\t')\n        if line_data[1] == 'TYPE_WAYPOINT':\n            waypoint.append([int(line_data[0]), float(line_data[2]), float(line_data[3])])\n            continue\n       \n        if line_data[1] == 'TYPE_ACCELEROMETER':\n            acce.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n        \n        if line_data[1] == 'TYPE_ACCELEROMETER_UNCALIBRATED':\n            acce_uncali.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n        \n        if line_data[1] == 'TYPE_GYROSCOPE':\n            gyro.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n\n        if line_data[1] == 'TYPE_GYROSCOPE_UNCALIBRATED':\n            gyro_uncali.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n        \n        if line_data[1] == 'TYPE_MAGNETIC_FIELD':\n            magn.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n\n        if line_data[1] == 'TYPE_MAGNETIC_FIELD_UNCALIBRATED':\n            if len(line_data) > 7:\n                magn_uncali.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4]), float(line_data[5]), float(line_data[6]), float(line_data[7])])\n            else:\n                magn_uncali.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4]), 0, 0, 0])\n            continue\n\n        if line_data[1] == 'TYPE_ROTATION_VECTOR':\n            ahrs.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n\n        if line_data[1] == 'TYPE_WIFI':\n            sys_ts = line_data[0]\n            ssid = line_data[2]\n            bssid = line_data[3]\n            rssi = line_data[4]\n            lastseen_ts = line_data[6]\n            wifi_data = [sys_ts, ssid, bssid, rssi, lastseen_ts]\n            wifi.append(wifi_data)\n            continue\n\n        if line_data[1] == 'TYPE_BEACON':\n            ts = line_data[0]\n            uuid = line_data[2]\n            major = line_data[3]\n            minor = line_data[4]\n            rssi = line_data[6]\n            dist = line_data[7]\n            last_seen_timestamp = line_data[9]\n            ibeacon_data = [ts, '_'.join([uuid, major, minor]), rssi, dist,last_seen_timestamp]\n            ibeacon.append(ibeacon_data)\n            continue\n        \n    \n    acce = np.array(acce)\n    acce_uncali = np.array(acce_uncali)\n    gyro = np.array(gyro)\n    gyro_uncali = np.array(gyro_uncali)\n    magn = np.array(magn)\n    magn_uncali = np.array(magn_uncali)\n    ahrs = np.array(ahrs)\n    wifi = np.array(wifi)\n    ibeacon = np.array(ibeacon)\n    waypoint = np.array(waypoint)\n    \n    return ReadData(acce, acce_uncali, gyro, gyro_uncali, magn, magn_uncali, ahrs, wifi, ibeacon, waypoint, starttime, endtime)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# great public notebook\ndf_sample = pd.read_csv('../input/3-3-g6-indoor-navigation-snap-to-grid/submission_snap_to_grid.csv')\ndf_sample[\"site_id\"] = df_sample[\"site_path_timestamp\"].apply(lambda x:x.split('_')[0])\ndf_sample[\"path_id\"] = df_sample[\"site_path_timestamp\"].apply(lambda x:x.split('_')[1])\ndf_sample[\"timestamp\"] = df_sample[\"site_path_timestamp\"].apply(lambda x:x.split('_')[2]).astype(int)\nlist_site = df_sample[\"site_id\"].unique()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make train data for each site\ndf_leak = pd.DataFrame()\nfor site_id in tqdm(list_site):\n    print(site_id)\n    ## train\n    list_train_files = glob.glob(f\"../input/indoor-location-navigation/train/{site_id}/**/*.txt\", recursive = True)\n    l_pid = []\n    l_sts = []\n    l_ets = []\n    l_swpx = []\n    l_swpy = []\n    l_ewpx = []\n    l_ewpy = []\n    l_d = []\n    for filename in tqdm(list_train_files):\n        path_id = filename.split(\".\")[2].split(\"/\")[6]\n        try: df_all = read_data_file(filename)\n        except:continue\n        \n        # get start and end timestamp\n        sts = df_all.starttime\n        ets = df_all.endtime\n        \n        df_waypoint = pd.DataFrame(df_all.waypoint)\n        df_waypoint.columns = ['timestamp', 'waypoint_x','waypoint_y']\n        df_waypoint[\"timestamp\"] = (df_waypoint[\"timestamp\"]).astype(float)\n        \n        # search start and end waypoints of each path\n        swpx = df_waypoint.query(\"timestamp == @df_waypoint.timestamp.min()\")[\"waypoint_x\"].iloc[0]\n        swpy = df_waypoint.query(\"timestamp == @df_waypoint.timestamp.min()\")[\"waypoint_y\"].iloc[0]\n        ewpx = df_waypoint.query(\"timestamp == @df_waypoint.timestamp.max()\")[\"waypoint_x\"].iloc[0]\n        ewpy = df_waypoint.query(\"timestamp == @df_waypoint.timestamp.max()\")[\"waypoint_y\"].iloc[0]\n\n        # search device id besed on magn bias\n        d = sum(df_all.magn_uncali[0,4:7])\n        if d == 0:d = sum(df_all.magn_uncali[0,1:4] - df_all.magn[0,1:4])\n        d = round(d,2)\n\n        l_pid += [path_id]\n        l_sts += [sts]\n        l_ets += [ets]\n        l_swpx += [swpx]\n        l_swpy += [swpy]\n        l_ewpx += [ewpx]\n        l_ewpy += [ewpy]\n        l_d += [d]        \n    df_mart_train = pd.DataFrame(data={\"path_id\": l_pid,\n                                       \"start_time\": l_sts, \"end_time\": l_ets,\n                                       \"start_waypoint_x\": l_swpx, \"start_waypoint_y\": l_swpy,\n                                       \"end_waypoint_x\": l_ewpx, \"end_waypoint_y\": l_ewpy,\n                                       \"device\": l_d},\n                                 columns=[\"path_id\",\"start_time\",\"end_time\",\"start_waypoint_x\",\"start_waypoint_y\",\n                                          \"end_waypoint_x\", \"end_waypoint_y\", \"device\"])\n\n    l_pid = []\n    l_sts = []\n    l_ets = []\n    l_swpx = []\n    l_swpy = []\n    l_ewpx = []\n    l_ewpy = []\n    l_d = []\n    \n    ## test\n    df_sample_site = df_sample.query(\"site_id == @site_id\")\n    df_sample_site[\"timestamp\"] = df_sample_site[\"timestamp\"].astype(float)\n    list_path = df_sample_site[\"path_id\"].unique()\n    for path_id in tqdm(list_path):\n        df_sample_path = df_sample_site.query(\"path_id == @path_id\")\n        filename = f\"../input/indoor-location-navigation/test/{path_id}.txt\"\n        df_all = read_data_file(filename)\n        df_wifi = pd.DataFrame(df_all.wifi)\n        df_wifi.columns = ['timestamp', 'ssid', 'bssid', 'rssi', 'last_seen_timestamp']\n        df_wifi[\"timestamp\"] = (df_wifi[\"timestamp\"]).astype(float)\n        df_wifi[\"last_seen_timestamp\"] = (df_wifi[\"last_seen_timestamp\"]).astype(float)  \n        \n        df_ibeacon = pd.DataFrame(df_all.ibeacon)\n        # retrieve raw timestamp\n        if len(df_ibeacon) > 0:\n            df_ibeacon.columns = ['timestamp', 'uuid', 'rssi', 'dist','last_seen_timestamp']\n            df_ibeacon[\"timestamp\"] = (df_ibeacon[\"timestamp\"]).astype(float)\n            df_ibeacon[\"last_seen_timestamp\"] = (df_ibeacon[\"last_seen_timestamp\"]).astype(float)\n            time_diff = df_ibeacon.loc[0,\"last_seen_timestamp\"]-df_ibeacon.loc[0,\"timestamp\"]\n        else:\n            time_diff = (df_wifi[\"last_seen_timestamp\"] - df_wifi[\"timestamp\"]).max()\n            \n        # search device id besed on magn bias\n        d = sum(df_all.magn_uncali[0,4:7])\n        if d == 0:d = sum(df_all.magn_uncali[0,1:4] - df_all.magn[0,1:4])\n        d = round(d,2)\n        \n        sts = df_all.starttime + time_diff\n        ets = df_all.endtime + time_diff\n        swpx = np.nan;swpy = np.nan; ewpx = np.nan; ewpy = np.nan;floor = np.nan\n        # x and y\n        df_start = df_mart_train.query(\"device == @d and start_time > @ets - 2000 and start_time < @ets + 10000\").sort_values(\"start_time\").reset_index()\n        df_end = df_mart_train.query(\"device == @d and end_time < @sts + 2000 and end_time > @sts - 10000\").sort_values(\"end_time\",ascending=False).reset_index()\n        if len(df_start) > 0:\n            ewpx = df_start.iloc[0][\"start_waypoint_x\"]\n            ewpy = df_start.iloc[0][\"start_waypoint_y\"]\n        if len(df_end) > 0:\n            swpx = df_end.iloc[0][\"end_waypoint_x\"]\n            swpy = df_end.iloc[0][\"end_waypoint_y\"]\n\n        l_pid += [path_id]\n        l_sts += [sts]\n        l_ets += [ets]\n        l_swpx += [swpx]\n        l_swpy += [swpy]\n        l_ewpx += [ewpx]\n        l_ewpy += [ewpy]\n        l_d += [d]    \n      \n    df_mart_test = pd.DataFrame(data={\"path_id\": l_pid,\n                                       \"start_time\": df_all.starttime, \"end_time\": df_all.endtime,\n                                       \"start_waypoint_x\": l_swpx, \"start_waypoint_y\": l_swpy,\n                                       \"end_waypoint_x\": l_ewpx, \"end_waypoint_y\": l_ewpy,\n                                       \"device\": l_d},\n                                 columns=[\"path_id\",\"start_time\",\"end_time\",\"start_waypoint_x\",\"start_waypoint_y\",\n                                          \"end_waypoint_x\", \"end_waypoint_y\",\"device\"])\n    \n    df_leak = df_leak.append(df_mart_test)\n    \n    # calculate time difference and waypoint difference\n    df_tr = df_mart_train.sort_values([\"device\",\"start_time\"]).reset_index(drop=True)\n    df_tr[\"time_diff\"] = df_tr[\"start_time\"] - df_tr.groupby(\"device\").shift(1)[\"end_time\"]\n    df_tr[\"x_diff\"] = df_tr[\"start_waypoint_x\"] - df_tr.groupby(\"device\").shift(1)[\"end_waypoint_x\"]\n    df_tr[\"y_diff\"] = df_tr[\"start_waypoint_y\"] - df_tr.groupby(\"device\").shift(1)[\"end_waypoint_y\"]\n    \n    # visualize relationship between time and waypoint differences \n    fig, axes = plt.subplots(ncols=2, nrows=1, figsize=(12, 6))\n    _ = df_tr.query(\"time_diff > -2000 and time_diff < 1000*60\").plot.scatter(x=\"time_diff\",y=\"x_diff\",ax=axes[0])\n    _ = df_tr.query(\"time_diff > -2000 and time_diff < 1000*60\").plot.scatter(x=\"time_diff\",y=\"y_diff\",ax=axes[1])\n\n    display(fig)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As shown in above, the greater the time difference, the greater the difference in waypoints.","metadata":{}},{"cell_type":"code","source":"# apply leakage waypoint\ndf_sub = df_sample.copy()\nlist_path = df_sub[\"path_id\"].unique()\nfor path_id in tqdm(list_path):\n    df_sub_path = df_sub.query(\"path_id == @path_id\")\n    start_idx = df_sub.loc[df_sub[\"path_id\"] == path_id].index.min()\n    end_idx = df_sub.loc[df_sub[\"path_id\"] == path_id].index.max()\n    start_x = df_sub_path.at[start_idx,\"x\"]\n    start_y = df_sub_path.at[start_idx,\"y\"]\n    end_x = df_sub_path.at[end_idx,\"x\"]\n    end_y = df_sub_path.at[end_idx,\"y\"]\n    start_x_leak = df_leak.query(\"path_id == @path_id\")[\"start_waypoint_x\"].iloc[0]\n    start_y_leak = df_leak.query(\"path_id == @path_id\")[\"start_waypoint_y\"].iloc[0]\n    end_x_leak = df_leak.query(\"path_id == @path_id\")[\"end_waypoint_x\"].iloc[0]\n    end_y_leak = df_leak.query(\"path_id == @path_id\")[\"end_waypoint_y\"].iloc[0]\n    if not np.isnan(start_x_leak):\n        df_sub.at[start_idx,\"x\"] = start_x_leak\n        df_sub.at[start_idx,\"y\"] = start_y_leak\n    if not np.isnan(end_x_leak):\n        df_sub.at[end_idx,\"x\"] = end_x_leak\n        df_sub.at[end_idx,\"y\"] = end_y_leak","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub.drop([\"site_id\",\"path_id\",\"timestamp\"],axis=1).to_csv(\"submission.csv\",index = False)\ndf_sub","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}