{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Metadata"},{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings \nwarnings.filterwarnings('ignore')\n\nimport os \nimport time \nimport math \nimport pickle as pkl \nfrom datetime import datetime\nimport json\nimport pprint\nimport glob\nimport imageio\nimport numpy as np\nimport pandas as pd\npd.set_option('display.max_columns', 10000)\nimport seaborn as sns \nnp.set_printoptions(suppress=True)\nimport matplotlib.pyplot as plt \nimport plotly.graph_objs as go\nfrom PIL import Image\nfrom tqdm.notebook import tqdm\n\nsample_building = \"../input/indoor-location-navigation/metadata/5a0546857ecc773753327266/\"\n\ndef plot_all_floors(path_to_building):\n    plt.figure(figsize=(16, 10))\n    floor_paths = glob.glob(os.path.join(path_to_building, \"*/floor_image.png\"))\n    for ind, floor_path in enumerate(floor_paths):\n        w = math.ceil(len(floor_paths) / 2)\n        h = math.ceil(len(floor_paths) / w)\n        plt.subplot(h, w, ind + 1)\n        image = imageio.imread(floor_path)  \n        plt.imshow(image)\n        plt.axis(\"off\")\n        plt.title(floor_path.split(\"/\")[-2], fontsize=16)\n    plt.show()\n    \ndef read_floor_info(path_to_building): \n    json_paths = glob.glob(os.path.join(path_to_building, \"*/floor_info.json\"))\n    for ind, json_path in enumerate(json_paths): \n        with open(json_path, \"r\") as f: \n            info = json.load(f)\n        print('for {} floor in building {}'.format(json_path.split('/')[-2], sample_building))\n        pprint.pprint(info)\n\n\n## meta data related\n\nplot_all_floors(sample_building)\nread_floor_info(sample_building)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## for train/test data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# copy from https://github.com/location-competition/indoor-location-competition-20/blob/master/io_f.py\nfrom dataclasses import dataclass\n\n@dataclass\nclass ReadData:\n    acce: np.ndarray\n    acce_uncali: np.ndarray\n    gyro: np.ndarray\n    gyro_uncali: np.ndarray\n    magn: np.ndarray\n    magn_uncali: np.ndarray\n    ahrs: np.ndarray\n    wifi: np.ndarray\n    ibeacon: np.ndarray\n    waypoint: np.ndarray\n\n\ndef read_data_file(data_filename):\n    acce = []\n    acce_uncali = []\n    gyro = []\n    gyro_uncali = []\n    magn = []\n    magn_uncali = []\n    ahrs = []\n    wifi = []\n    ibeacon = []\n    waypoint = []\n\n    with open(data_filename, 'r', encoding='utf-8') as file:\n        lines = file.readlines()\n\n    for line_data in lines:\n        line_data = line_data.strip()\n        if not line_data or line_data[0] == '#':\n            continue\n\n        line_data = line_data.split('\\t')\n\n        if line_data[1] == 'TYPE_WAYPOINT':\n            waypoint.append([int(line_data[0]), float(line_data[2]), float(line_data[3])])\n            continue\n       \n        if line_data[1] == 'TYPE_ACCELEROMETER':\n            acce.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n        \n        if line_data[1] == 'TYPE_ACCELEROMETER_UNCALIBRATED':\n            acce_uncali.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n        \n        if line_data[1] == 'TYPE_GYROSCOPE':\n            gyro.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n\n        if line_data[1] == 'TYPE_GYROSCOPE_UNCALIBRATED':\n            gyro_uncali.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n        \n        if line_data[1] == 'TYPE_MAGNETIC_FIELD':\n            magn.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n\n        if line_data[1] == 'TYPE_MAGNETIC_FIELD_UNCALIBRATED':\n            magn_uncali.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n\n        if line_data[1] == 'TYPE_ROTATION_VECTOR':\n            ahrs.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            continue\n\n        if line_data[1] == 'TYPE_WIFI':\n            sys_ts = line_data[0]\n            ssid = line_data[2]\n            bssid = line_data[3]\n            rssi = line_data[4]\n            lastseen_ts = line_data[6]\n            wifi_data = [sys_ts, ssid, bssid, rssi, lastseen_ts]\n            wifi.append(wifi_data)\n            continue\n\n        if line_data[1] == 'TYPE_BEACON':\n            ts = line_data[0]\n            uuid = line_data[2]\n            major = line_data[3]\n            minor = line_data[4]\n            rssi = line_data[6]\n            ibeacon_data = [ts, '_'.join([uuid, major, minor]), rssi]\n            ibeacon.append(ibeacon_data)\n            continue\n        \n    \n    acce = np.array(acce)\n    acce_uncali = np.array(acce_uncali)\n    gyro = np.array(gyro)\n    gyro_uncali = np.array(gyro_uncali)\n    magn = np.array(magn)\n    magn_uncali = np.array(magn_uncali)\n    ahrs = np.array(ahrs)\n    wifi = np.array(wifi)\n    ibeacon = np.array(ibeacon)\n    waypoint = np.array(waypoint)\n    \n    return ReadData(acce, acce_uncali, gyro, gyro_uncali, magn, magn_uncali, ahrs, wifi, ibeacon, waypoint)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def visualize_trajectory(trajectory, floor_plan_filename, width_meter, height_meter, title=None, mode='lines + markers + text', show=False):\n    \"\"\"\n    Copied from from https://github.com/location-competition/indoor-location-competition-20/blob/master/visualize_f.py\n\n    \"\"\"\n    fig = go.Figure()\n\n    # add trajectory\n    size_list = [6] * trajectory.shape[0]\n    size_list[0] = 10\n    size_list[-1] = 10\n\n    color_list = ['rgba(4, 174, 4, 0.5)'] * trajectory.shape[0]\n    color_list[0] = 'rgba(12, 5, 235, 1)'\n    color_list[-1] = 'rgba(235, 5, 5, 1)'\n\n    position_count = {}\n    text_list = []\n    for i in range(trajectory.shape[0]):\n        if str(trajectory[i]) in position_count:\n            position_count[str(trajectory[i])] += 1\n        else:\n            position_count[str(trajectory[i])] = 0\n        text_list.append('        ' * position_count[str(trajectory[i])] + f'{i}')\n    text_list[0] = 'Start 0'\n    text_list[-1] = f'End {trajectory.shape[0] - 1}'\n\n    fig.add_trace(\n        go.Scattergl(\n            x=trajectory[:, 0],\n            y=trajectory[:, 1],\n            mode=mode,\n            marker=dict(size=size_list, color=color_list),\n            line=dict(shape='linear', color='lightgrey', width=3, dash='dash'),\n            text=text_list,\n            textposition=\"top center\",\n            name='trajectory',\n        ))\n\n    # add floor plan\n    floor_plan = Image.open(floor_plan_filename)\n    fig.update_layout(images=[\n        go.layout.Image(\n            source=floor_plan,\n            xref=\"x\",\n            yref=\"y\",\n            x=0,\n            y=height_meter,\n            sizex=width_meter,\n            sizey=height_meter,\n            sizing=\"contain\",\n            opacity=1,\n            layer=\"below\",\n        )\n    ])\n\n    # configure\n    fig.update_xaxes(autorange=False, range=[0, width_meter])\n    fig.update_yaxes(autorange=False, range=[0, height_meter], scaleanchor=\"x\", scaleratio=1)\n    fig.update_layout(\n        title=go.layout.Title(\n            text=title or \"No title.\",\n            xref=\"paper\",\n            x=0,\n        ),\n        autosize=True,\n        width=800,\n        height=  800 * height_meter / width_meter,\n        template=\"plotly_white\",\n    )\n\n    if show:\n        fig.show()\n\n    return fig\n\ndef visualize_train_trajectory(path):\n    \"\"\"\n    Edited from \n    https://www.kaggle.com/ihelon/indoor-location-exploratory-data-analysis\n    \"\"\"\n    _id, floor = path.split(\"/\")[:2]\n    \n    train_floor_data = read_data_file(f\"../input/indoor-location-navigation/train/{path}\")\n    with open(f\"../input/indoor-location-navigation/metadata/{_id}/{floor}/floor_info.json\") as f:\n        train_floor_info = json.load(f)\n\n    return visualize_trajectory(\n        train_floor_data.waypoint[:, 1:3], \n        f\"../input/indoor-location-navigation/metadata/{_id}/{floor}/floor_image.png\",\n        train_floor_info[\"map_info\"][\"width\"], \n        train_floor_info[\"map_info\"][\"height\"],\n        f\"Visualization of {path}\"\n    )\n\ndef visualize_approx_trajectory(path, points):\n    \"\"\"\n    Edited from \n    https://www.kaggle.com/ihelon/indoor-location-exploratory-data-analysis\n    \"\"\"\n    _id, floor = path.split(\"/\")[:2]\n    \n    train_floor_data = read_data_file(f\"../input/indoor-location-navigation/train/{path}\")\n    with open(f\"../input/indoor-location-navigation/metadata/{_id}/{floor}/floor_info.json\") as f:\n        train_floor_info = json.load(f)\n\n    return visualize_trajectory(\n        points, \n        f\"../input/indoor-location-navigation/metadata/{_id}/{floor}/floor_image.png\",\n        train_floor_info[\"map_info\"][\"width\"], \n        train_floor_info[\"map_info\"][\"height\"],\n        f\"Visualization of {path}\"\n    )\n\nvisualize_train_trajectory(\"5a0546857ecc773753327266/F2/5dccf516c04f060006e6e3c9.txt\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# cat /kaggle/input/indoor-location-navigation/train/5a0546857ecc773753327266/B1/5e15730aa280850006f3d005.txt | head -5\n# cat /kaggle/input/indoor-location-navigation/train/5a0546857ecc773753327266/B1/5e15730aa280850006f3d005.txt | tail -5\n\nsample_building_train = \"../input/indoor-location-navigation/train/5a0546857ecc773753327266/F2/5dccf516c04f060006e6e3c9.txt\"\nsample_building_test = \"/kaggle/input/indoor-location-navigation/test/046cfa46be49fc10834815c6.txt\"\n        \ntrain_sample = read_data_file(sample_building_train)\ntest_sample = read_data_file(sample_building_test)\n\nimu = np.concatenate((train_sample.acce, train_sample.acce_uncali[:, 1:], train_sample.gyro[:, 1:], \n                          train_sample.gyro_uncali[:, 1:], train_sample.magn[:, 1:], train_sample.magn_uncali[:, 1:], train_sample.ahrs[:, 1:]), axis=1)\nimu_df = pd.DataFrame(imu)\nimu_df.columns = ['timestamp', 'acce_x','acce_y', 'acce_z','acce_uncali_x','acce_uncali_y', 'acce_uncali_z',\n              'gyro_x','gyro_y', 'gyro_z','gyro_uncali_x','gyro_uncali_y', 'gyro_uncali_z',\n              'magn_x','magn_y', 'magn_z','magn_uncali_x','magn_uncali_y', 'magn_uncali_z',\n              'ahrs_x','ahrs_y', 'ahrs_z']\ndisplay(imu_df.head().style.set_caption('imu'))\n\nwaypoint_df = pd.DataFrame(train_sample.waypoint)\nwaypoint_df.columns = ['timestamp', 'X', 'Y']\ndisplay(waypoint_df.style.set_caption('waypoint')); ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display(imu_df.iloc[:, 1:].describe())\n\n# check the differece between calibrated and uncalibrated columns \ncali_columns = [col for col in imu_df.columns[1:] if 'uncali' not in col and 'ahrs' not in col]\nuncali_columns = [col for col in imu_df.columns[1:] if 'uncali' in col and 'ahrs' not in col]\nprint('cali columns', cali_columns)\nprint('uncali_columns', uncali_columns)\na = imu_df[cali_columns]\nb = imu_df[uncali_columns]\nb.columns = a.columns \ndiff = a - b \ndiff.columns = [col + '_diff' for col in diff.columns]\ndisplay(diff.describe().style.set_caption('difference'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = 1573713056850\nend_time = 1573713091483\n\ndef plot_imu_signals(col):\n    fig, ax = plt.subplots(nrows=3, ncols=1, figsize=(14, 9))\n\n    sns.lineplot(x=imu_df.timestamp, y=imu_df[f\"{col}_uncali_x\"], ax=ax[0], label = 'uncali', color='orange')\n    sns.lineplot(x=imu_df.timestamp, y=imu_df[f\"{col}_uncali_y\"], ax=ax[1], label = 'uncali', color='orange')\n    sns.lineplot(x=imu_df.timestamp, y=imu_df[f\"{col}_uncali_z\"], ax=ax[2], label = 'uncali', color='orange')\n\n    sns.lineplot(x=imu_df.timestamp, y=imu_df[f\"{col}_x\"], ax=ax[0], label='cali', color='cornflowerblue')\n    sns.lineplot(x=imu_df.timestamp, y=imu_df[f\"{col}_y\"], ax=ax[1], label='cali', color='cornflowerblue')\n    sns.lineplot(x=imu_df.timestamp, y=imu_df[f\"{col}_z\"], ax=ax[2], label='cali', color='cornflowerblue')\n    \n    ax[0].set_ylabel(f\"{col}_x \\n(calib./uncalib.)\")\n    ax[1].set_ylabel(f\"{col}_y \\n(calib./uncalib.)\")\n    ax[2].set_ylabel(f\"{col}_z \\n(calib./uncalib.)\")\n\n    for i in range(3):\n        ax[i].set_xlim([start_time, end_time])\n    plt.tight_layout()\n    plt.show()\n    \nplot_imu_signals('acce')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_imu_signals('gyro')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_imu_signals('magn')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def calc_from_acce(timestamp, acce, p_0):\n    df = pd.DataFrame({'timestamp' : timestamp, 'acceleration' : acce})\n    df['timestamp_ms'] = df['timestamp'].apply(lambda x: datetime.fromtimestamp(x/1000.0))\n    df['timedelta_ms'] = df['timestamp_ms'].diff()\n    df['timedelta_s'] = df['timedelta_ms'].apply(lambda x: x.total_seconds()).fillna(0)\n    df['velocity'] = (df['acceleration']*df['timedelta_s']).cumsum()\n    df['position'] = p_0 + (df['velocity']*df['timedelta_s']).cumsum()\n\n    return df[['timestamp', 'timestamp_ms', 'timedelta_s', 'position', 'velocity', 'acceleration']]\n\ndf_from_acce_x = calc_from_acce(imu_df.timestamp, imu_df.acce_x, waypoint_df.X.iloc[0])\ndf_from_acce_y = calc_from_acce(imu_df.timestamp, imu_df.acce_y, waypoint_df.Y.iloc[0])\n\n# print(os.path.basename(sample_building_train))\n# visualize_train_trajectory('/'.join(sample_building_train.split('/')[-3:]))\nvisualize_approx_trajectory('/'.join(sample_building_train.split('/')[-3:]), np.concatenate((df_from_acce_x.position.values.reshape(-1, 1), df_from_acce_y.position.values.reshape(-1, 1)), axis=1)[::50])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport os.path as osp \n\nsite_count = 0 \nroot = \"/kaggle/input/indoor-location-navigation/train\"\nsite_list, floor_list, traj_list = [], [], []\nfor site in os.listdir(root): \n    if not osp.isdir(osp.join(root, site)): \n        continue \n    site_count += 1\n    traj_count = 0 \n    num_floors = len(os.listdir(osp.join(root, site)))\n    for floor in os.listdir(osp.join(root, site)): \n        traj_count += len(os.listdir(osp.join(root, site, floor)))\n    traj_list.append(traj_count)\n    floor_list.append(num_floors)\n    site_list.append(site)\n    # print('for site {}, there are {} floors and {} trajectories in total'.format(site, num_floors, traj_count))\n\nprint('{} sites in total'.format(site_count))\n\nfloor_traj = pd.DataFrame()\nfloor_traj['site'] = site_list\nfloor_traj['floor'] = floor_list \nfloor_traj['traj'] = traj_list\nfloor_traj = floor_traj.set_index('site')\n\ndisplay(floor_traj.describe())\nfig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2, figsize=(14, 8))\nsns.distplot(floor_traj['floor'], ax=ax1)\nsns.distplot(floor_traj['traj'], ax=ax2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## first we build a mapping from sites to trajectories "},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission = pd.read_csv(\"../input/indoor-location-navigation/sample_submission.csv\", index_col=0)\nprint('{} rows in sample submission'.format(sample_submission.shape[0]))\n\nsite_path_timestamps = sample_submission.index.values\n\nsplits = [item.split('_') for item in site_path_timestamps]\nsites = [split[0] for split in splits]\ntrajs = [split[1] for split in splits]\ntimestamps = [split[2] for split in splits]\nprint(\"{} unique sites with {} trajectories\".format(len(np.unique(sites)), len(np.unique(trajs))))\n\ntraj2site = {}\nfor traj, site in zip(trajs, sites): \n    if traj not in traj2site: \n        traj2site[traj] = site\n    else: \n        assert traj2site[traj] == site, \"One trajectory can not correspond to two sites\"\n        \nprint('mapping from trajectories to sites built, with {} items'.format(len(traj2site.keys())))\n\n# only 24 sites in the training site are involved \nusites = np.unique(sites)\ntrain_sites = floor_traj.index.values.tolist()\nindices = []\n\nfor usite in usites: \n    assert usite in train_sites, \"Site {} can not be found in training set\".format(usite)\n    index = train_sites.index(usite)\n    indices.append(index)\n    \nmasked_floor_traj = floor_traj.iloc[indices, :]\ndisplay(masked_floor_traj.describe().style.set_caption('in consideration'))\nfig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2, figsize=(14, 8))\nsns.distplot(masked_floor_traj['floor'], ax=ax1)\nsns.distplot(masked_floor_traj['traj'], ax=ax2); ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## obtain the floor names(for later one-hot embedding)"},{"metadata":{"trusted":true},"cell_type":"code","source":"import copy \n\ndef build_floorname2floorindex(floornames):\n    new_floornames = []\n    for floorname in floornames: \n        if floorname[0] <= 'Z' and floorname[0] >= 'A': \n            new_floornames.append(floorname)\n        else: \n            new_floornames.append(floorname[::-1])\n    f_floornames = [floorname for floorname in new_floornames if 'F' in floorname]\n    b_floornames = [floorname for floorname in new_floornames if 'B' in floorname]\n    min_index = 0 - len(b_floornames)\n    \n    new_floornames = (sorted(b_floornames, reverse=True) + sorted(f_floornames)) if min_index < 0 else sorted(f_floornames)\n    print(new_floornames)\n    mapping = {name: i for i, name in enumerate(new_floornames)}\n    out_mapping = copy.deepcopy(mapping)\n    for k, v in mapping.items(): \n        out_mapping[k[::-1]] = v\n    print(out_mapping)\n    return out_mapping, min_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.random.seed(2021)\nimport pickle as pkl \nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.ensemble import GradientBoostingRegressor, GradientBoostingClassifier, RandomForestClassifier, RandomForestRegressor\nfrom sklearn.metrics import mean_squared_error\n\ndownsample_factor = 5\nsite_loc_models = {}\nsite_floor_models = {}\nsite_shifts = {}\n\ntest_root = \"/kaggle/input/indoor-location-navigation/test/\"\n\nsample_submission = pd.read_csv(\"/kaggle/input/indoor-location-navigation/sample_submission.csv\", index_col=0)\n\nids = sample_submission.index.values.tolist()\nsample_sites = [_id.split('_')[0] for _id in ids]\nsample_trajs = [_id.split('_')[1] for _id in ids]\nsample_timestamps = [_id.split('_')[2] for _id in ids]\n\n\ndef estimate_loc_every_timestamp(timestamps, waypoint_timestamps, waypoints): \n    estimated_locs = []\n    start, end = waypoint_timestamps[0], waypoint_timestamps[-1]\n    interval = end - start \n    dist = np.linalg.norm(waypoints[0] - waypoints[-1])\n    # m / ms \n    vel = dist / interval \n    # actually we can assume that the surveyor walks with a constant speed \n    for timestamp in timestamps: \n        if timestamp <= start: \n            loc = waypoints[0]\n        elif timestamp >= end: \n            loc = waypoints[-1]\n        else: \n            e = None\n            for end_index in range(len(waypoint_timestamps)): \n                if waypoint_timestamps[end_index] > timestamp: \n                    e = end_index\n                    break \n            s = e - 1 \n            assert s >= 0 \n            loc = waypoints[s] + (timestamp - waypoint_timestamps[s]) * (waypoints[e] - waypoints[s]) / (waypoint_timestamps[e] - waypoint_timestamps[s])\n        estimated_locs.append(loc)\n    \n    estimated_locs = np.stack(estimated_locs, axis=0)\n    return estimated_locs\n    \nresume_id = 0 \nif osp.exists(\"checkpoint.pkl\"): \n    with open(\"checkpoint.pkl\", \"rb\") as f: \n        checkpoint = pkl.load(f)\n    resume_id = checkpoint['resume_id']\n    site_loc_models = checkpoint['loc_model']\n    site_floor_models = checkpoint['cls_model']\n    site_shifts = checkpoint['shifts']\n    print(\"load checkpoint with resume id {}\".format(resume_id))\n    \nfor site_id, usite in enumerate(usites): \n    # if site_id < resume_id: \n    #     continue \n    row_indices = np.nonzero(np.array(sample_sites) == usite)[0]\n    # print(row_indices)\n    print('--- * processing the {} site {} starts * ---'.format(site_id, usite))\n    # train a model for a site\n    # we have to first generaete the training data \n    usite_root = osp.join(root, usite)\n    num_floors = masked_floor_traj[masked_floor_traj.index == usite]['floor'].iloc[0]\n    X_train, Y_train_cls, Y_train_reg = [], [], []\n    floorname2floorindex, shift = build_floorname2floorindex(os.listdir(usite_root))\n    for floor_name in os.listdir(usite_root): \n        floor_label = floorname2floorindex[floor_name]\n        skip_count = 0 \n        for traj in tqdm(os.listdir(osp.join(usite_root, floor_name))): \n            traj_path = osp.join(usite_root, floor_name, traj)\n            data = read_data_file(traj_path)\n            # post-process data into imu & waypoint \n            if data.acce.shape != data.acce_uncali.shape: \n                # print('skip trajectory {}'.format(traj))\n                skip_count += 1\n                continue\n            # print(downsample_factor)\n            imu = np.concatenate((data.acce, data.acce_uncali[:, 1:], data.gyro[:, 1:], \n                          data.gyro_uncali[:, 1:], data.magn[:, 1:], data.magn_uncali[:, 1:], data.ahrs[:, 1:]), axis=1)[::downsample_factor]\n            imu_df = pd.DataFrame(imu)\n            imu_df.columns = ['timestamp', 'acce_x','acce_y', 'acce_z','acce_uncali_x','acce_uncali_y', 'acce_uncali_z',\n                          'gyro_x','gyro_y', 'gyro_z','gyro_uncali_x','gyro_uncali_y', 'gyro_uncali_z',\n                          'magn_x','magn_y', 'magn_z','magn_uncali_x','magn_uncali_y', 'magn_uncali_z',\n                          'ahrs_x','ahrs_y', 'ahrs_z']\n            waypoint_df = pd.DataFrame(data.waypoint)\n            waypoint_df.columns = ['timestamp', 'waypoint_x', 'waypoint_y']\n            waypoints = np.stack((waypoint_df.waypoint_x, waypoint_df.waypoint_y), axis=1)\n            estimated_locs = estimate_loc_every_timestamp(imu_df.timestamp.values.tolist(), waypoint_df.timestamp.values.tolist(), waypoints)\n            # print(estimated_locs)\n            imu_df['loc_x'] = estimated_locs[:, 0]\n            imu_df['loc_y'] = estimated_locs[:, 1]\n            train_columns = [col for col in imu_df.columns if 'magn' in col]\n            traj_X_train = imu_df[train_columns].values\n            traj_Y_train_cls = np.zeros((traj_X_train.shape[0], num_floors))\n            traj_Y_train_cls[:, floor_label] = 1 \n            traj_Y_train_reg = imu_df[['loc_x', 'loc_y']].values\n            # print(traj_X_train.shape, traj_Y_train_cls.shape, traj_Y_train_reg.shape)\n            X_train.append(traj_X_train); Y_train_cls.append(traj_Y_train_cls); Y_train_reg.append(traj_Y_train_reg)\n            # visualize_approx_trajectory(osp.join(usite, floor_name, traj), estimated_locs)\n    X_train = np.concatenate(X_train, axis=0); Y_train_cls = np.concatenate(Y_train_cls, axis=0); Y_train_reg = np.concatenate(Y_train_reg, axis=0)\n    shuffle_indices = np.arange(len(X_train))\n    np.random.shuffle(shuffle_indices)\n    X_train = X_train[shuffle_indices]; Y_train_cls = Y_train_cls[shuffle_indices]; Y_train_reg = Y_train_reg[shuffle_indices]\n    print(X_train.shape, Y_train_cls.shape, Y_train_reg.shape)\n    \n    start = time.time()\n    cls_model = RandomForestClassifier()\n    reg_model = RandomForestRegressor()\n    val_ratio = 0.25 \n    val_num = int(len(X_train) * val_ratio)\n    x_train, y_train_cls, y_train_reg = X_train[val_num:], Y_train_cls[val_num:], Y_train_reg[val_num:]\n    x_val, y_val_cls, y_val_reg = X_train[:val_num], Y_train_cls[:val_num], Y_train_reg[:val_num]\n    cls_model.fit(x_train, y_train_cls)\n    end = time.time()\n    print('training on classifier finished in {}s'.format(end - start))\n    cls_score = cls_model.score(x_val, y_val_cls)\n    print('score on the validation set for classification is {:.4f}'.format(cls_score))\n    start = time.time()\n    reg_model.fit(x_train, y_train_reg)\n    reg_score = reg_model.score(x_val, y_val_reg)\n    y_val_reg_pred = reg_model.predict(x_val)\n    rmse = mean_squared_error(y_val_reg_pred, y_val_reg, squared=False)\n    end = time.time()\n    print('training on regressor finished in {}s'.format(end - start))\n    print('score/rmse on the validation set for regression is {:.4f}({:.4F})'.format(reg_score, rmse))\n    # site_floor_models[usite] = cls_model\n    # site_loc_models[usite] = reg_model\n    # site_shifts[usite] = shift \n    # with open(\"checkpoint.pkl\".format(site_id), \"wb\") as f: \n    #     pkl.dump({\"resume_id\": site_id + 1, \"cls_model\": site_floor_models, \"loc_model\": site_loc_models, \"shifts\": site_shifts}, f)\n    print('--- * processing site {} ends * ---'.format(usite))\n    for row_index in row_indices: \n        traj = sample_trajs[row_index]\n        test_data = read_data_file(osp.join(test_root, traj + \".txt\"))\n        timestamp = sample_timestamps[row_index]\n        ref_timestamps = test_data.acce[:, 0]\n        # find nearest timestamp\n        index = np.abs(ref_timestamps - int(timestamp)).argsort()[0]\n        # we have already obtained index \n        imu = np.concatenate((test_data.acce, test_data.acce_uncali[:, 1:], test_data.gyro[:, 1:], \n                          test_data.gyro_uncali[:, 1:], test_data.magn[:, 1:], test_data.magn_uncali[:, 1:], test_data.ahrs[:, 1:]), axis=1)\n        imu_df = pd.DataFrame(imu)\n        imu_df.columns = ['timestamp', 'acce_x','acce_y', 'acce_z','acce_uncali_x','acce_uncali_y', 'acce_uncali_z',\n                      'gyro_x','gyro_y', 'gyro_z','gyro_uncali_x','gyro_uncali_y', 'gyro_uncali_z',\n                      'magn_x','magn_y', 'magn_z','magn_uncali_x','magn_uncali_y', 'magn_uncali_z',\n                      'ahrs_x','ahrs_y', 'ahrs_z']\n        test_columns = [col for col in imu_df.columns if 'mag' in col]\n        inputs = imu_df.iloc[index][test_columns].values.reshape(1, -1)\n        loc = reg_model.predict(inputs).reshape(-1)\n        floor_pred = cls_model.predict(inputs).argmax() + shift\n\n        sample_submission.iloc[row_index, 0] = int(floor_pred) \n        sample_submission.iloc[row_index, 1] = loc[0]\n        sample_submission.iloc[row_index, 2] = loc[1]\n    del X_train, Y_train_cls, Y_train_reg\n    del cls_model, reg_model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display(sample_submission.style.set_caption('final submission'))\nsample_submission.to_csv(\"submission_v1.csv\") ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test_root = \"/kaggle/input/indoor-location-navigation/test/\"\n\n# sample_submission = pd.read_csv(\"/kaggle/input/indoor-location-navigation/sample_submission.csv\", index_col=0)\n# print(sample_submission.columns)\n# floor_predictions = []\n# xs_predictions = []\n# ys_predictions = []\n\n# ids = sample_submission.index.values.tolist()\n# sites = [_id.split('_')[0] for _id in ids]\n# trajs = [_id.split('_')[1] for _id in ids]\n# timestamps = [_id.split('_')[2] for _id in ids]\n\n# for i, (site, traj, timestamp) in enumerate(zip(sites, trajs, timestamps)): \n#     loc_model = site_loc_models[site]\n#     cls_model = site_floor_models[site]\n#     shift = site_shifts[site]\n#     # read trajectory \n#     test_data = read_data_file(osp.join(test_root, traj + \".txt\"))\n#     ref_timestamps = test_data.timestamp.values\n#     # find nearest timestamp \n#     index = np.abs(ref_timestamps - timestamp).argsort()[0]\n#     # we have already obtained index \n#     test_columns = [col for col in test_data.columns if 'mag' in col]\n#     inputs = test_data.iloc[index][test_columns].values.reshape(1, -1)\n#     loc = loc_model.predict(inputs).reshape(-1)\n#     floor = cls_model.predict(inputs) + shift \n#     sample_submission.iloc[i].floor = floor \n#     sample_submission.iloc[i].x = loc[0]\n#     sample_submission.iloc[i].y = loc[1]\n    \n# display(sample_submission.style.set_caption('final submission'))\n# sample_submission.to_csv(\"submission_v1.csv\") ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}