{"cells":[{"metadata":{},"cell_type":"markdown","source":"### What is this?\nThis notebook takes your raw submission (= no post processing yet) as input and shows the following for each sites and floors.\n\n- Your raw submission in the floor map on the left.\n- Cost Minimization post processing result in the floor map on the middle.\n- Snap to Grid Post Processing result followed by  Cost Minimization post processing on the right.\n\n\nBy having them in the same place I hope you'll be able to see how much improvement you would potentially have!\nI appreciate your feedback :)\n\n### References\n- [indoor - Post-processing by Cost Minimization](https://www.kaggle.com/saitodevel01/indoor-post-processing-by-cost-minimization) by @saitodevel01.\n- [Indoor Navigation - Snap to Grid\" Post Processing](https://www.kaggle.com/robikscube/indoor-navigation-snap-to-grid-post-processing) by @robikscube.\n- [LSTM by Keras with Unified Wi-Fi Feats](https://www.kaggle.com/kokitanisaka/lstm-by-keras-with-unified-wi-fi-feats) by @kokitanisaka.\n"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# Helper Functions\nimport pandas as pd\nimport numpy as np\n\nimport json\nimport matplotlib.pylab as plt\n\ndef split_col(df):\n    df = pd.concat([\n        df['site_path_timestamp'].str.split('_', expand=True) \\\n        .rename(columns={0:'site',\n                         1:'path',\n                         2:'timestamp'}),\n        df\n    ], axis=1).copy()\n    return df\n\nfloor_map = {\"B2\":-2, \"B1\":-1, \"F1\":0, \"F2\": 1, \"F3\":2,\n             \"F4\":3, \"F5\":4, \"F6\":5, \"F7\":6,\"F8\":7,\"F9\":8,\n             \"1F\":0, \"2F\":1, \"3F\":2, \"4F\":3, \"5F\":4, \"6F\":5,\n             \"7F\":6, \"8F\": 7, \"9F\":8}\n\n\ndef plot_preds(\n    ax,\n    context_text,\n    site,\n    floorNo,\n    sub=None,\n    true_locs=None,\n    base=\"../input/indoor-location-navigation\",\n    show_train=True,\n    show_preds=True,\n    fix_labels=True,\n    map_floor=None\n):\n    \"\"\"\n    Plots predictions on floorplan map.\n    \n    map_floor : use a different floor's map\n    \"\"\"\n    if map_floor is None:\n        map_floor = floorNo\n    # Prepare width_meter & height_meter (taken from the .json file)\n    floor_plan_filename = f\"{base}/metadata/{site}/{map_floor}/floor_image.png\"\n    json_plan_filename = f\"{base}/metadata/{site}/{map_floor}/floor_info.json\"\n    with open(json_plan_filename) as json_file:\n        json_data = json.load(json_file)\n\n    width_meter = json_data[\"map_info\"][\"width\"]\n    height_meter = json_data[\"map_info\"][\"height\"]\n\n    floor_img = plt.imread(f\"{base}/metadata/{site}/{map_floor}/floor_image.png\")\n\n\n    ax.imshow(floor_img)\n\n    if show_train:\n        true_locs = true_locs.query('site == @site and floorNo == @map_floor').copy()\n        true_locs[\"x_\"] = true_locs[\"x\"] * floor_img.shape[0] / height_meter\n        true_locs[\"y_\"] = (\n            true_locs[\"y\"] * -1 * floor_img.shape[1] / width_meter\n        ) + floor_img.shape[0]\n        true_locs.query(\"site == @site and floorNo == @map_floor\").groupby(\"path\").plot(\n            x=\"x_\",\n            y=\"y_\",\n            style=\"+\",\n            ax=ax,\n            label=\"train waypoint location\",\n            color=\"grey\",\n            alpha=0.5,\n        )\n\n    if show_preds:\n        sub = sub.query('site == @site and floorNo == @floorNo').copy()\n        sub[\"x_\"] = sub[\"x\"] * floor_img.shape[0] / height_meter\n        sub[\"y_\"] = (\n            sub[\"y\"] * -1 * floor_img.shape[1] / width_meter\n        ) + floor_img.shape[0]\n        for path, path_data in sub.query(\n            \"site == @site and floorNo == @floorNo\"\n        ).groupby(\"path\"):\n            path_data.plot(\n                x=\"x_\",\n                y=\"y_\",\n                style=\".-\",\n                ax=ax,\n                title=f\"{context_text} - {site} - floor - {floorNo}\",\n                alpha=1,\n                label=path,\n            )\n    if fix_labels:\n        handles, labels = ax.get_legend_handles_labels()\n        by_label = dict(zip(labels, handles))\n        ax.legend(\n            by_label.values(), by_label.keys(), loc=\"center left\", bbox_to_anchor=(1, 0.5)\n        )\n    return","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"!git clone --depth 1 https://github.com/location-competition/indoor-location-competition-20 indoor_location_competition_20\n!rm -rf indoor_location_competition_20/data","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"import multiprocessing\nfrom typing import Any, List, Tuple, Union\n\nimport numpy as np\nimport pandas as pd\nimport scipy.interpolate\nimport scipy.sparse\nfrom tqdm import tqdm\nfrom scipy.spatial.distance import cdist\nimport indoor_location_competition_20.compute_f as compute_f\nfrom indoor_location_competition_20.io_f import read_data_file\n\n\n# Credit @obikscube: https://www.kaggle.com/robikscube/indoor-navigation-snap-to-grid-post-processing\ndef post_process_snap_to_grid(sub_df: pd.DataFrame, train_waypoints, threshold: int) -> pd.DataFrame:\n    \"\"\"\n    Snap to grid if within a threshold.\n\n    x, y are the predicted points.\n    x_, y_ are the closest grid points.\n    _x_, _y_ are the new predictions after post processing.\n    \"\"\"\n    sub_df = split_col(sub_df)\n    sub_df = sub_process(sub_df, train_waypoints)\n\n    sub_df = add_xy(sub_df)\n    train_waypoints = add_xy(train_waypoints)\n\n    ds = []\n    for (site, myfloor), d in sub_df.groupby(['site','floor']):\n        true_floor_locs = train_waypoints.loc[(train_waypoints['floor'] == myfloor) &\n                                            (train_waypoints['site'] == site)] \\\n            .reset_index(drop=True)\n        if len(true_floor_locs) == 0:\n            print(f'Skipping {site} {myfloor}')\n            continue\n        d['matched_point'] = [closest_point(x, list(true_floor_locs['xy'])) for x in d['xy']]\n        d['x_'] = d['matched_point'].apply(lambda x: x[0])\n        d['y_'] = d['matched_point'].apply(lambda x: x[1])\n        ds.append(d)\n\n    sub_df = pd.concat(ds) # type: ignore\n\n    # Calculate the distances\n    sub_df['dist'] = np.sqrt((sub_df.x - sub_df.x_) **\n                             2 + (sub_df.y - sub_df.y_)**2)\n\n    sub_df['_x_'] = sub_df['x']\n    sub_df['_y_'] = sub_df['y']\n    sub_df.loc[sub_df['dist'] < threshold,\n               '_x_'] = sub_df.loc[sub_df['dist'] < threshold]['x_']  # type: ignore\n    sub_df.loc[sub_df['dist'] < threshold,\n               '_y_'] = sub_df.loc[sub_df['dist'] < threshold]['y_']  # type: ignore\n\n    sub_df = sub_df[['site_path_timestamp', 'floor', '_x_', '_y_',\n                     'site', 'path', 'floorNo']].rename(columns={'_x_': 'x', '_y_': 'y'})\n    return sub_df\n\n\ndef split_col(df: pd.DataFrame) -> pd.DataFrame:\n    df = pd.concat([\n        df['site_path_timestamp'].str.split('_', expand=True).rename(columns={0: 'site',\n                                                                              1: 'path',\n                                                                              2: 'timestamp'}),\n        df\n    ], axis=1)  # type: ignore\n    return df\n\ndef add_xy(df: pd.DataFrame) -> pd.DataFrame:\n    df['xy'] = [(x, y) for x,y in zip(df['x'], df['y'])]\n    return df\n\ndef closest_point(point, points):\n    \"\"\" Find closest point from a list of points. \"\"\"\n    return points[cdist([point], points).argmin()]\n\ndef sub_process(sub, train_waypoints):\n    sub = split_col(sub[['site_path_timestamp','floor','x','y']]).copy()\n    sub = sub.merge(train_waypoints[['site','floorNo','floor']].drop_duplicates(), how='left')\n    return sub\n\n# credit @saitodevel01: https://www.kaggle.com/saitodevel01/indoor-post-processing-by-cost-minimization\ndef post_process_correct_path(sub_df: pd.DataFrame) -> pd.DataFrame:\n    tmp = sub_df['site_path_timestamp'].apply(lambda s : pd.Series(s.split('_')))\n    sub_df['site'] = tmp[0]\n    sub_df['path'] = tmp[1]\n    sub_df['timestamp'] = tmp[2].astype(float)\n\n    processes = multiprocessing.cpu_count()\n    with multiprocessing.Pool(processes=processes) as pool:\n        dfs = pool.imap_unordered(correct_path, sub_df.groupby('path'))\n        dfs = tqdm(dfs)\n        dfs = list(dfs)\n    processed_df = pd.concat(dfs).sort_values('site_path_timestamp')\n    return processed_df # type: ignore\n\ndef compute_rel_positions(acce_datas, ahrs_datas):\n    step_timestamps, step_indexs, step_acce_max_mins = compute_f.compute_steps(acce_datas)\n    headings = compute_f.compute_headings(ahrs_datas)\n    stride_lengths = compute_f.compute_stride_length(step_acce_max_mins)\n    step_headings = compute_f.compute_step_heading(step_timestamps, headings)\n    rel_positions = compute_f.compute_rel_positions(stride_lengths, step_headings)\n    return rel_positions\n\ndef correct_path(args):\n    path, path_df = args\n\n    T_ref  = path_df['timestamp'].values\n    xy_hat = path_df[['x', 'y']].values\n\n    example = read_data_file(f'/kaggle/input/indoor-location-navigation/test/{path}.txt')\n    rel_positions = compute_rel_positions(example.acce, example.ahrs)\n    if T_ref[-1] > rel_positions[-1, 0]:\n        rel_positions = [np.array([[0, 0, 0]]), rel_positions, np.array([[T_ref[-1], 0, 0]])]\n    else:\n        rel_positions = [np.array([[0, 0, 0]]), rel_positions]\n    rel_positions = np.concatenate(rel_positions)\n\n    T_rel = rel_positions[:, 0]\n    delta_xy_hat = np.diff(scipy.interpolate.interp1d(T_rel, np.cumsum(rel_positions[:, 1:3], axis=0), axis=0)(T_ref), axis=0)\n\n    N = xy_hat.shape[0]\n    delta_t = np.diff(T_ref)\n    alpha = (8.1)**(-2) * np.ones(N)\n    beta  = (0.3 + 0.3 * 1e-3 * delta_t)**(-2)\n    A = scipy.sparse.spdiags(alpha, [0], N, N)\n    B = scipy.sparse.spdiags( beta, [0], N-1, N-1)\n    D = scipy.sparse.spdiags(np.stack([-np.ones(N), np.ones(N)]), [0, 1], N-1, N)\n\n    Q = A + (D.T @ B @ D)\n    c = (A @ xy_hat) + (D.T @ (B @ delta_xy_hat))\n    xy_star = scipy.sparse.linalg.spsolve(Q, c)\n\n    return pd.DataFrame({\n        'site_path_timestamp' : path_df['site_path_timestamp'],\n        'floor' : path_df['floor'],\n        'x' : xy_star[:, 0],\n        'y' : xy_star[:, 1],\n    })","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"train_waypoints = pd.read_csv('/kaggle/input/indoor-location-train-waypoints/train_waypoints.csv')\nsub_df = pd.read_csv('/kaggle/input/lstm-by-keras-with-unified-wifi-feats/submission.csv')\nprocessed_sub_df1 = post_process_correct_path(sub_df)\nprocessed_sub_df2 = post_process_snap_to_grid(processed_sub_df1, train_waypoints, threshold=8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df = sub_process(sub_df, train_waypoints)\nprocessed_sub_df1 = sub_process(processed_sub_df1, train_waypoints)\nprocessed_sub_df2 = sub_process(processed_sub_df2, train_waypoints)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from pathlib import Path\ndef generate_target_sites(sub_df):\n    return sorted(sub_df['site'].unique())\n\ndef generate_site_floors_dict(sub_df):\n    sites = generate_target_sites(sub_df)\n    site_floors_dict = {}\n    for site in sites:\n        site_path = Path('/kaggle/input/indoor-location-navigation/train') / site\n        site_floors_dict[site] = [path.name for path in site_path.glob('*')]\n    return site_floors_dict\n\nall_sites = generate_target_sites(sub_df)\nsite_floors_dict = generate_site_floors_dict(sub_df)\nsite_floors_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sites1 = all_sites[:8]\nsites2 = all_sites[8:16]\nsites3 = all_sites[16:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_sites(sites, sub1_df, sub2_df, sub3_df):\n    num_floors = 0\n    for site in sites:\n        num_floors += len(site_floors_dict[site])\n\n    fig, ax = plt.subplots(num_floors, 3, figsize=(36, 12 * num_floors))\n\n    idx = 0\n    for site in sites:\n        floors = site_floors_dict[site]\n\n        for floor in floors:\n            plot_preds(ax[idx][0], \"raw submission\", site, floor, sub1_df, train_waypoints, show_preds=True)\n            plot_preds(ax[idx][1], \"correct_path\", site, floor, sub2_df, train_waypoints, show_preds=True)\n            plot_preds(ax[idx][2], \"snap_to_grid\", site, floor, sub3_df, train_waypoints, show_preds=True)\n            idx += 1\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_sites(sites1, sub_df, processed_sub_df1, processed_sub_df2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_sites(sites2, sub_df, processed_sub_df1, processed_sub_df2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_sites(sites3, sub_df, processed_sub_df1, processed_sub_df2)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}