{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install nudged","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:51:27.205156Z","iopub.execute_input":"2021-06-13T13:51:27.205658Z","iopub.status.idle":"2021-06-13T13:51:33.581562Z","shell.execute_reply.started":"2021-06-13T13:51:27.205555Z","shell.execute_reply":"2021-06-13T13:51:33.580543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport scipy.stats as stats\nimport glob\nimport json\nimport psutil\nimport random\nimport os\nimport time\nimport sys\nimport math\nimport scipy.interpolate\nimport scipy.sparse\nfrom tqdm import tqdm\nfrom contextlib import contextmanager\nimport matplotlib.pylab as plt\nimport shapely\nimport matplotlib\nimport nudged\nimport matplotlib.pyplot as plt\nimport plotly.graph_objs as go\nimport pickle\nfrom shapely.geometry import shape, GeometryCollection, Polygon, MultiPolygon\nfrom shapely.affinity import affine_transform\nfrom PIL import Image, ImageOps\nfrom skimage.morphology import convex_hull_image\nfrom shapely.geometry import Point\nfrom sklearn.metrics import mean_squared_error, mean_absolute_error\nfrom scipy.spatial.distance import cdist\nfrom pathlib import Path","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2021-06-13T13:51:33.583789Z","iopub.execute_input":"2021-06-13T13:51:33.584234Z","iopub.status.idle":"2021-06-13T13:51:34.311501Z","shell.execute_reply.started":"2021-06-13T13:51:33.584181Z","shell.execute_reply":"2021-06-13T13:51:34.310417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Provided GitHub functions\n!cp -r ../input/indoor-location-competition-20-git/* ./\nimport io_f\nimport visualize_f \nimport compute_f\nimport main  ","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2021-06-13T13:51:34.313804Z","iopub.execute_input":"2021-06-13T13:51:34.314114Z","iopub.status.idle":"2021-06-13T13:51:35.069119Z","shell.execute_reply.started":"2021-06-13T13:51:34.314080Z","shell.execute_reply":"2021-06-13T13:51:35.067952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# FUNCTIONS","metadata":{}},{"cell_type":"markdown","source":"* ### Additional","metadata":{}},{"cell_type":"code","source":"floors_key_assign = {'1F':0, '2F':1, '3F':2, '4F':3, '5F':4, '6F':5, '7F':6, '8F':7, '9F':8, \n          'B':-1, 'B1':-1,'B2':-2, 'B3':-3, 'BF':-1, 'BM':-1, \n          'F1':0, 'F2':1, 'F3':2, 'F4':3, 'F5':4, 'F6':5,'F7':6, 'F8':7, 'F9':8, 'F10':9,\n          'G':0, 'LG1':-1, 'LG2':-2, \n          \"L1\":0,\"L2\":1,\"L3\":2,\"L4\":3,\"L5\":4,\"L6\":5,\"L7\":6,\"L8\":7,\"L9\":8,\"L10\":9,'L11':10, \n          'LM':0, 'M':0, 'P1':-1, 'P2':-2,\n          \"地下一层\":-1,\"一层\":0,\"二层\":1,\"三层\":2,\"四层\":3}\n\ndef get_site_floor_from_id(site, floor_id):\n    site_p = \"../input/indoor-location-navigation/train/\"+site\n    floor_p = \"/*\"\n    site_floor_names = []\n    floors_files = sorted(glob.glob(site_p + floor_p))\n    for f, floor in enumerate(floors_files):\n        floor_name = floor.split('/')[-1]\n        site_floor_names.append(floor_name)\n    for key_f in floors_key_assign:\n        if key_f in site_floor_names:\n            if floors_key_assign[key_f] == floor_id:\n                return key_f\n    return floor_id\n\ndef highlight_dif(x):\n    r = 'red'\n    g = 'green'\n\n    m1 = x['f'] != x['pred_f']\n    m2 = x['f'] == x['pred_f']\n    \n    m3 = x['x'] != x['pred_x']\n    m4 = x['x'] == x['pred_x']\n    \n    m5 = x['y'] != x['pred_y']\n    m6 = x['y'] == x['pred_y']\n\n    df1 = pd.DataFrame('color: ', index=x.index, columns=x.columns)\n    #rewrite values by boolean masks\n    df1['pred_f'] = np.where(m1, 'color: {}'.format(r), df1['pred_f'])\n    df1['pred_f'] = np.where(m2, 'color: {}'.format(g), df1['pred_f'])\n    \n    df1['pred_x'] = np.where(m3, 'color: {}'.format(r), df1['pred_x'])\n    df1['pred_x'] = np.where(m4, 'color: {}'.format(g), df1['pred_x'])\n    \n    df1['pred_y'] = np.where(m5, 'color: {}'.format(r), df1['pred_y'])\n    df1['pred_y'] = np.where(m6, 'color: {}'.format(g), df1['pred_y'])\n    return df1\n\ndef highlight_eval(x):\n    r = 'red'\n    g = 'green'\n    y = 'yellow'\n\n    m1 = x['floor_knn_acc'] > x['floor_fix_acc']\n    m2 = x['floor_knn_acc'] == x['floor_fix_acc']\n    m3 = x['floor_knn_acc'] < x['floor_fix_acc']\n    \n    m4 = x['wp_knn_mpe'] < x['wp_fix_mpe']\n    m5 = x['wp_knn_mpe'] == x['wp_fix_mpe']\n    m6 = x['wp_knn_mpe'] > x['wp_fix_mpe']\n    \n    m7 = x['wp_knn_acc'] > x['wp_fix_acc']\n    m8 = x['wp_knn_acc'] == x['wp_fix_acc']\n    m9 = x['wp_knn_acc'] < x['wp_fix_acc']\n    \n    m10 = x['wp_knn_rmse'] < x['wp_fix_rmse']\n    m11 = x['wp_knn_rmse'] == x['wp_fix_rmse']\n    m12 = x['wp_knn_rmse'] > x['wp_fix_rmse']\n    \n    \n    df1 = pd.DataFrame('color: ', index=x.index, columns=x.columns)\n    #rewrite values by boolean masks\n    df1['floor_fix_acc'] = np.where(m1, 'color: {}'.format(r), df1['floor_fix_acc'])\n    df1['floor_fix_acc'] = np.where(m2, 'color: {}'.format(y), df1['floor_fix_acc'])\n    df1['floor_fix_acc'] = np.where(m3, 'color: {}'.format(g), df1['floor_fix_acc'])\n    \n    df1['wp_fix_mpe'] = np.where(m4, 'color: {}'.format(r), df1['wp_fix_mpe'])\n    df1['wp_fix_mpe'] = np.where(m5, 'color: {}'.format(y), df1['wp_fix_mpe'])\n    df1['wp_fix_mpe'] = np.where(m6, 'color: {}'.format(g), df1['wp_fix_mpe'])\n    \n    df1['wp_fix_acc'] = np.where(m7, 'color: {}'.format(r), df1['wp_fix_acc'])\n    df1['wp_fix_acc'] = np.where(m8, 'color: {}'.format(y), df1['wp_fix_acc'])\n    df1['wp_fix_acc'] = np.where(m9, 'color: {}'.format(g), df1['wp_fix_acc'])\n    \n    df1['wp_fix_rmse'] = np.where(m10, 'color: {}'.format(r), df1['wp_fix_rmse'])\n    df1['wp_fix_rmse'] = np.where(m11, 'color: {}'.format(y), df1['wp_fix_rmse'])\n    df1['wp_fix_rmse'] = np.where(m12, 'color: {}'.format(g), df1['wp_fix_rmse'])\n    return df1","metadata":{"execution":{"iopub.status.busy":"2021-06-13T19:18:15.502508Z","iopub.execute_input":"2021-06-13T19:18:15.502927Z","iopub.status.idle":"2021-06-13T19:18:15.529857Z","shell.execute_reply.started":"2021-06-13T19:18:15.502890Z","shell.execute_reply":"2021-06-13T19:18:15.528663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* ### Plot on Map","metadata":{}},{"cell_type":"code","source":"def plot_floor(\n    site,\n    floorNo,\n    paths_wp_df=None,\n    grid_wp=None,\n    grid_gen=None,\n    display_corridors=True,\n    display_map=True,\n    w_size=8,\n    w_color='grey',\n    g_size=4,\n    g_color='grey',\n    \n    base=\"../input/indoor-location-navigation\",\n    base_pol = \"../input/indoor-location-navigation-scaled-geojson/scaled_geojson\",\n):\n    map_floor = floorNo\n    # Prepare width_meter & height_meter (taken from the .json file)\n    floor_plan_filename = f\"{base}/metadata/{site}/{map_floor}/floor_image.png\"\n    json_plan_filename = f\"{base}/metadata/{site}/{map_floor}/floor_info.json\"\n    with open(json_plan_filename) as json_file:\n        json_data = json.load(json_file)\n    width_meter = json_data[\"map_info\"][\"width\"]\n    height_meter = json_data[\"map_info\"][\"height\"]\n    floor_img = f\"{base}/metadata/{site}/{map_floor}/floor_image.png\"\n    \n    file = f\"{base_pol}/{site}/{map_floor}/shapely_geometry.pkl\"\n    with open(file, 'rb') as f:\n        geometry = pickle.load(f)\n    \n    m_opacity=0.8\n    mode='lines + markers'\n    \n    #Display\n    fig = go.Figure()\n    \n    if paths_wp_df is not None:\n        for path, path_data in paths_wp_df.groupby(\"path\"):\n            fig.add_trace(\n                go.Scattergl(\n                    x=path_data[\"x\"],\n                    y=path_data[\"y\"],\n                    mode=mode,\n                    marker_size=10,\n                    name=path,\n                    visible='legendonly'\n                ))\n\n    if grid_wp is not None:\n        fig.add_trace(\n            go.Scattergl(\n                name=\"Waypoints Grid\",\n                x=grid_wp[:, 0],\n                y=grid_wp[:, 1],\n                mode='markers',\n                opacity=m_opacity,\n                marker_symbol=3,\n                marker_color=w_color,\n                marker_size=w_size,\n                visible='legendonly'\n\n            ))\n    if grid_gen is not None:\n        fig.add_trace(\n            go.Scattergl(\n                name=\"Generated Grid\",\n                x=grid_gen[:, 0],\n                y=grid_gen[:, 1],\n                mode='markers',\n                opacity=m_opacity,\n                marker_symbol=3,\n                marker_color=g_color,\n                marker_size=g_size,\n                visible='legendonly'\n            ))\n    \n    if display_map:\n        floor_plan = Image.open(floor_img)\n        fig.update_layout(images=[\n            go.layout.Image(\n                source=floor_plan,\n                xref=\"x\",\n                yref=\"y\",\n                x=0,\n                y=height_meter,\n                sizex=width_meter,\n                sizey=height_meter,\n                sizing=\"contain\",\n                opacity=0.5,\n                layer=\"below\",\n            )\n        ])\n    \n    if display_corridors:\n        for coord in extract_coords_from_polygon(geometry):\n            x, y = coord\n            fig.add_trace(\n                go.Scattergl(\n                    x=x,\n                    y=y,\n                    opacity=0.6,\n                    text=None,\n                    marker_color='red',\n                    hoverinfo='skip',\n                    showlegend=False\n                ))\n    \n    # configure\n    fig.update_xaxes(autorange=False, range=[0, width_meter])\n    fig.update_yaxes(autorange=False, range=[0, height_meter], scaleanchor=\"x\", scaleratio=1)\n    fig.update_layout(\n        title=go.layout.Title(\n            text=\"Site: %s Floor: %s\" % (site, map_floor),\n            xref=\"paper\",\n            x=0,\n        ),\n        autosize=True,\n        width=800,\n        height=800 * height_meter / width_meter,\n        template=\"plotly_white\",\n    )\n\n    return fig","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2021-06-13T13:51:35.098638Z","iopub.execute_input":"2021-06-13T13:51:35.099100Z","iopub.status.idle":"2021-06-13T13:51:35.119129Z","shell.execute_reply.started":"2021-06-13T13:51:35.099054Z","shell.execute_reply":"2021-06-13T13:51:35.118294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* ### Generate Grid from Map Geometry","metadata":{}},{"cell_type":"code","source":"def generate_grid_from_map(stite_check, floor_check, step = 2):\n    base_pol = \"../input/indoor-location-navigation-scaled-geojson/scaled_geojson\"\n    file = f\"{base_pol}/{stite_check}/{floor_check}/shapely_geometry.pkl\"\n    with open(file, 'rb') as f:\n        corridor = pickle.load(f)\n    \n    x_min, y_min, x_max, y_max = get_bounding_box(corridor)\n\n    x_range = range(math.ceil(x_min), math.floor(x_max), step)\n    y_range = range(math.ceil(y_min), math.floor(y_max), step)\n\n    valid_coords = []\n\n    for x in x_range:\n        for y in y_range:\n            if Point(x, y).within(corridor):\n                valid_coords.append([x, y])\n\n    return np.array(valid_coords)\n\ndef get_bounding_box(shapes):\n    \"\"\"\n    To extract bounding box from Polygon\n    \"\"\"\n    x_min = 10000\n    y_min = 10000\n    x_max = 0\n    y_max = 0\n    \n    if type(shapes) == Polygon:\n            shapes = [shapes]\n    for shape in shapes:\n        x, y = shape.exterior.xy\n        x_min = min(min(x), x_min)\n        y_min = min(min(y), y_min)\n        x_max = max(max(x), x_max)\n        y_max = max(max(y), y_max)\n    return x_min, y_min, x_max, y_max\n\ndef extract_coords_from_polygon(polygon):\n    coords = []\n    if type(polygon) == MultiPolygon:\n        polygons = polygon.geoms\n    else:\n        polygons = [polygon]\n\n    for polygon in polygons:\n        x, y = polygon.exterior.xy\n        coords.append((np.array(x), np.array(y)))\n        for interior in polygon.interiors:\n            x, y = interior.xy\n            coords.append((np.array(x), np.array(y)))\n\n    return coords","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2021-06-13T13:51:35.120344Z","iopub.execute_input":"2021-06-13T13:51:35.120775Z","iopub.status.idle":"2021-06-13T13:51:35.137154Z","shell.execute_reply.started":"2021-06-13T13:51:35.120731Z","shell.execute_reply":"2021-06-13T13:51:35.136321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* ### Generate Grid from WP Paths","metadata":{}},{"cell_type":"code","source":"def generate_grid_from_wp(site:str=None,floor:str=None, start:int=None, stop:int=None):\n    site_p = \"../input/indoor-location-navigation/train/*\"\n    floor_p = \"/*\"\n    path_p = \"/*\"\n        \n    if (site is not None) & (floor is None):\n        site_p = \"../input/indoor-location-navigation/train/\" + site\n\n    if (site is not None) & (floor is not None):\n        site_p = \"../input/indoor-location-navigation/train/\" + site\n        floor_p = \"/\" + floor\n    \n    #Go through sites\n    sites = sorted(glob.glob(site_p))\n    \n    if (start is not None) & (stop is not None):\n        sites = sites[start:stop]\n    \n    #Create DF of all waypoints in whole site\n    main_wp_df = pd.DataFrame(columns = ['site', 'floor', 'x', 'y'])\n    for s, site in enumerate(sites):\n        #print site info\n        site_id = site.split('/')[-1]\n\n        #Create DF of all waypoints in whole site\n        site_wp_df = pd.DataFrame(columns = ['site', 'floor', 'x', 'y'])\n        total_path = 0\n\n        #Go through floors\n        floors = sorted(glob.glob(site + floor_p))  \n        for f, floor in enumerate(floors):\n            #print floor info\n            floor_id = floor.split('/')[-1]\n\n            #Create DF of all waypoints on each floor\n            wp_floor_df = pd.DataFrame(columns = ['site', 'floor', 'x', 'y'])\n\n            #Go through paths\n            paths = sorted(glob.glob(floor + path_p))   \n            for p, path in enumerate(paths):                \n                total_path += len(paths)\n\n                #print path info\n                path_id = path.split('/')[-1].split('.')[0]\n\n\n                #Exstract All Paths Waypoints\n                path_wp = io_f.read_data_file(path).waypoint           \n                for wp in path_wp:\n                    new_row = {'site':site_id, 'floor':floor_id, 'x':wp[1], 'y':wp[2]}\n                    wp_floor_df = wp_floor_df.append(new_row, ignore_index=True)\n\n                #Drop duplicate waypoints\n                wp_floor_df = wp_floor_df.drop_duplicates(subset=['x', 'y'], keep='last')\n\n            #load to site wp df\n            site_wp_df = site_wp_df.append(wp_floor_df, ignore_index=True)\n\n        main_wp_df = main_wp_df.append(site_wp_df, ignore_index=True)\n\n    site_floor_list = main_wp_df.groupby(['site', 'floor'],as_index=False).agg({'x': 'count'})    \n    if (site is not None) | (floor is not None):\n        return main_wp_df, site_floor_list\n    else:\n        return main_wp_df","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2021-06-13T13:51:35.138194Z","iopub.execute_input":"2021-06-13T13:51:35.138568Z","iopub.status.idle":"2021-06-13T13:51:35.155043Z","shell.execute_reply.started":"2021-06-13T13:51:35.138514Z","shell.execute_reply":"2021-06-13T13:51:35.154149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* ### Get All Path from Floor","metadata":{}},{"cell_type":"code","source":"def get_floor_paths(stite_check,floor_check):\n    base = '../input/indoor-location-navigation/train'\n    floor_paths_files = sorted(glob.glob(\"{}/{}/{}/*\".format(base,stite_check,floor_check)))\n    floor_paths_df = pd.DataFrame(columns = ['site', 'floor', 'path', 'timestamp', 'x', 'y'])\n    for floor_paths_file in floor_paths_files:\n        path_id = floor_paths_file.split('/')[-1].split('.')[0]\n        #Exstract All Paths Waypoints\n        path_wp = io_f.read_data_file(floor_paths_file).waypoint\n        path_wp_df = pd.DataFrame(columns = ['site', 'floor', 'path', 'timestamp', 'x', 'y'])\n        for wp in path_wp:\n                new_row = {'site':stite_check, 'floor':floor_check, 'path':path_id, 'timestamp':wp[0], 'x':wp[1], 'y':wp[2]}\n                path_wp_df = path_wp_df.append(new_row, ignore_index=True)\n\n        floor_paths_df = floor_paths_df.append(path_wp_df, ignore_index=True)\n    return (floor_paths_df)","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2021-06-13T13:51:35.156729Z","iopub.execute_input":"2021-06-13T13:51:35.157124Z","iopub.status.idle":"2021-06-13T13:51:35.170336Z","shell.execute_reply.started":"2021-06-13T13:51:35.157047Z","shell.execute_reply":"2021-06-13T13:51:35.169175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* ### Cost Minimisation","metadata":{}},{"cell_type":"code","source":"def compute_rel_positions(acce_datas, ahrs_datas, posi_datas):\n    step_timestamps, step_indexs, step_acce_max_mins = compute_f.compute_steps(acce_datas)\n    headings = compute_f.compute_headings(ahrs_datas)\n    stride_lengths = compute_f.compute_stride_length(step_acce_max_mins)\n    step_headings = compute_f.compute_step_heading(step_timestamps, headings)\n    rel_positions = compute_f.compute_rel_positions(stride_lengths, step_headings)\n    return rel_positions\n\ndef correct_path(site, path_df, alpha_mod=2, beta_mod=2):\n    site_p = \"../input/indoor-location-navigation/train/\"\n    path = path_df['path'].values[0]\n    floor = path_df['floor'].values[0]\n    T_ref  = path_df['timestamp'].values\n    xy_hat = path_df[['x', 'y']].values\n    \n    site_floor = get_site_floor_from_id(site,floor)\n    \n    example = io_f.read_data_file(f'{site_p}/{site}/{site_floor}/{path}.txt')\n    rel_positions = compute_rel_positions(example.acce, example.ahrs, path_df[['timestamp','x','y']].to_numpy())\n    '''   \n    posi_datas = pd.DataFrame(columns=['timestamp','x', 'y'])\n    posi_datas['timestamp'] = step_positions[:, 0]\n    posi_datas['x'] = step_positions[:, 1]\n    posi_datas['y'] = step_positions[:, 2]\n    posi_datas['floor'] = floor\n    posi_datas['path'] = path'''\n    \n    if T_ref[-1] > rel_positions[-1, 0]:\n        rel_positions = [np.array([[0, 0, 0]]), rel_positions, np.array([[T_ref[-1], 0, 0]])]\n    else:\n        rel_positions = [np.array([[0, 0, 0]]), rel_positions]\n    rel_positions = np.concatenate(rel_positions)\n    \n    T_rel = rel_positions[:, 0]\n    delta_xy_hat = np.diff(scipy.interpolate.interp1d(T_rel, np.cumsum(rel_positions[:, 1:3], axis=0), axis=0)(T_ref), axis=0)\n\n    N = xy_hat.shape[0]\n    delta_t = np.diff(T_ref)\n    alpha = (8.1)**(-alpha_mod) * np.ones(N)\n    beta  = (0.3 + 0.3 * 1e-3 * delta_t)**(-beta_mod)\n    A = scipy.sparse.spdiags(alpha, [0], N, N)\n    B = scipy.sparse.spdiags( beta, [0], N-1, N-1)\n    D = scipy.sparse.spdiags(np.stack([-np.ones(N), np.ones(N)]), [0, 1], N-1, N)\n\n    Q = A + (D.T @ B @ D)\n    c = (A @ xy_hat) + (D.T @ (B @ delta_xy_hat))\n    xy_star = scipy.sparse.linalg.spsolve(Q, c)\n    \n    path_df['x'] = xy_star[:, 0]\n    path_df['y'] = xy_star[:, 1]\n    \n    return path_df","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2021-06-13T13:51:35.173482Z","iopub.execute_input":"2021-06-13T13:51:35.173828Z","iopub.status.idle":"2021-06-13T13:51:35.192135Z","shell.execute_reply.started":"2021-06-13T13:51:35.173797Z","shell.execute_reply":"2021-06-13T13:51:35.190952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* ### Grid Generation","metadata":{}},{"cell_type":"code","source":"def generate_grid_for_floor(stite_check, floor_check, step, all_paths=False):\n    #collect all waypoints on floor\n    if all_paths:\n        floor_paths_df = get_floor_paths(stite_check,floor_check)\n    #grid from waypoints\n    floor_wp_grid_df, site_floor_list = generate_grid_from_wp(stite_check, floor_check)\n    floor_wp_grid = floor_wp_grid_df[['x','y']].to_numpy() #for ploting\n    #generate grid based on map\n    floor_map_grid = generate_grid_from_map(stite_check, floor_check, step)\n    #snap to grid\n    #fixed_floor_paths_df = snap_to_grid(floor_paths_df, floor_map_grid, 2)\n    if all_paths:\n        return floor_paths_df, floor_wp_grid, floor_map_grid\n    else:\n        return floor_wp_grid, floor_map_grid","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:51:35.194784Z","iopub.execute_input":"2021-06-13T13:51:35.195223Z","iopub.status.idle":"2021-06-13T13:51:35.206338Z","shell.execute_reply.started":"2021-06-13T13:51:35.195180Z","shell.execute_reply":"2021-06-13T13:51:35.205499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# POST-PROCESSING","metadata":{}},{"cell_type":"markdown","source":"### Snap to Grid","metadata":{}},{"cell_type":"code","source":"def snap_to_grid(wp, grid, threshold):\n    wp = find_closest_grid(wp, grid)\n    wp['dist'] = np.sqrt((wp.x-wp.x_)**2 + (wp.y-wp.y_)**2)\n    \n    wp = snap_points_to_grid(wp, threshold=threshold)\n    wp = wp[['site','path', 'floor', 'timestamp','_x_','_y_']].rename(columns={'_x_':'x', '_y_':'y'})\n    return wp\n\ndef add_xy(df):\n    df['xy'] = [(x, y) for x,y in zip(df['x'], df['y'])]\n    return df\ndef closest_point(point, points):\n    \"\"\" Find closest point from a list of points. \"\"\"\n    return points[cdist([point], points).argmin()]\ndef find_closest_grid(wp, grid):\n    grid_df = pd.DataFrame(grid, columns = ['x','y'])\n    wp = add_xy(wp)\n    grid_df = add_xy(grid_df)\n    ds = []\n    for (site, myfloor), d in wp.groupby(['site','floor']):\n        true_floor_locs = grid_df.reset_index(drop=True)\n        if len(true_floor_locs) == 0:\n            print(f'Skipping {site} {myfloor}')\n            continue\n        d['matched_point'] = [closest_point(x, list(true_floor_locs['xy'])) for x in d['xy']]\n        d['x_'] = d['matched_point'].apply(lambda x: x[0])\n        d['y_'] = d['matched_point'].apply(lambda x: x[1])\n        ds.append(d)\n\n    wp = pd.concat(ds)\n    \n    return(wp)\ndef snap_points_to_grid(sub, threshold):\n    \"\"\"\n    Snap to grid if within a threshold.\n    \n    x, y are the predicted points.\n    x_, y_ are the closest grid points.\n    _x_, _y_ are the new predictions after post processing.\n    \"\"\"\n    sub['_x_'] = sub['x']\n    sub['_y_'] = sub['y']\n    sub.loc[sub['dist'] < threshold, '_x_'] = sub.loc[sub['dist'] < threshold]['x_']\n    sub.loc[sub['dist'] < threshold, '_y_'] = sub.loc[sub['dist'] < threshold]['y_']\n    return sub.copy()","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2021-06-13T13:51:35.207677Z","iopub.execute_input":"2021-06-13T13:51:35.208082Z","iopub.status.idle":"2021-06-13T13:51:35.222915Z","shell.execute_reply.started":"2021-06-13T13:51:35.208040Z","shell.execute_reply":"2021-06-13T13:51:35.221670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Floor FIx","metadata":{}},{"cell_type":"code","source":"def fix_floor(site_df, paths_list, s, s_len):\n    floor_fixed_df = pd.DataFrame(columns=['path','timestamp','x','y','pred_x','pred_y','fixed_x','fixed_y','f','pred_f','fixed_f'])\n    \n    site_df = site_df.sort_values(['path', 'timestamp'])\n    \n    for p, path_id in enumerate(paths_list):\n        sel_path = site_df[(site_df.path == path_id)].reset_index(drop=True)\n        print (\"\\033[A                                                                                         \\033[A\", end=\"\\r\")\n        print(f\"Site: {s:02d}/{s_len} | Fixing Predicted Floors | Path: {p+1}/{len(paths_list)} ({path_id})\", end=\"\\r\")\n\n        temp = sel_path.groupby(['pred_f'])[\"pred_f\"].count().reset_index(name=\"count\")\n        if temp.shape[0] > 1:\n            pred_floor = temp.pred_f.iloc[temp['count'].idxmax()]\n        else:\n            pred_floor = sel_path.pred_f.head(1).values[0]\n\n        sel_path['fixed_f'] = pred_floor\n        floor_fixed_df = floor_fixed_df.append(sel_path, ignore_index=True)\n        \n    return floor_fixed_df","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2021-06-13T13:51:35.224447Z","iopub.execute_input":"2021-06-13T13:51:35.224928Z","iopub.status.idle":"2021-06-13T13:51:35.238597Z","shell.execute_reply.started":"2021-06-13T13:51:35.224891Z","shell.execute_reply":"2021-06-13T13:51:35.237511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## WP Fix","metadata":{}},{"cell_type":"code","source":"def wp_fix(site_id, site_df, floors_list, paths_list, s, s_len):\n    wp_fixed_df = pd.DataFrame(columns=['path','timestamp','x','y','pred_x','pred_y','fixed_x','fixed_y','f','pred_f'])\n    \n    site_df = site_df.sort_values(['f', 'path', 'timestamp'])\n    alpha, thresshold = 2, 5.5\n    betta = 0\n    \n    #Tunning Betta\n    floor_id = 0\n    floor_name = get_site_floor_from_id(site_id, floor_id)\n    if floor_name==0:\n        floor_id = 1\n        floor_name = get_site_floor_from_id(site_id, floor_id)\n        \n    print (\"\\033[A                                                                                                                       \\033[A\", end=\"\\r\")\n    print(f\"Site: {s+1:02d}/{s_len} | Fixing Predicted XY | Floor: Tunning Betta with {floor_name} | Gererating Grid \", end=\"\\r\")\n    floor_wp_grid, floor_map_grid = generate_grid_for_floor(site_id, floor_name, 2)\n    sel_floor = site_df[(site_df.pred_f == floor_id)].reset_index(drop=True)     \n\n    if betta == 0:\n        min_score = 50.0\n        for b in [3,4,5,6]:\n            betta_eval_df = pd.DataFrame(columns=['pred_x','fixed_x'])\n            for p, path_id in enumerate(paths_list):\n                print (\"\\033[A                                                                                                                                                       \\033[A\", end=\"\\r\")\n                print(f\"Site: {s+1:02d}/{s_len} | Fixing Predicted XY | Tunning Betta ({b}/{betta}-{min_score:.2f}) | Path: {p+1}/{len(paths_list)} ({path_id})\", end=\"\\r\")\n                sel_path = sel_floor[(sel_floor.path == path_id)].reset_index(drop=True)\n                if sel_path.empty == False:\n                    knn_path = sel_path[['path','timestamp', 'pred_x', 'pred_y', 'f']]\n                    knn_path = knn_path.rename({'pred_x': 'x', 'pred_y': 'y', 'f': 'floor'}, axis='columns').sort_values('timestamp')\n                    knn_path['site'] = site_id\n\n                    betta_eval = sel_path[['x','y']]\n                    knn_pdr_cm_path = correct_path(site_id, knn_path, alpha, b)\n                    knn_pdr_cm_wpgrid_path = snap_to_grid(knn_pdr_cm_path, floor_wp_grid, thresshold)\n                    betta_eval['fixed_x'] = knn_pdr_cm_wpgrid_path['x']\n                    betta_eval['fixed_y'] = knn_pdr_cm_wpgrid_path['y']\n                    betta_eval_df = betta_eval_df.append(betta_eval, ignore_index=True)\n\n            mpe = np.mean(np.sqrt(np.power(betta_eval_df.fixed_x - betta_eval_df.x, 2) + np.power(betta_eval_df.fixed_y - betta_eval_df.y, 2)))\n            acc = sum(betta_eval_df.x == betta_eval_df.fixed_x)/betta_eval_df.shape[0]\n            score = mpe + (1-acc)\n            if score < min_score:\n                min_score = score\n                betta = b      \n    #Fixing site floor by floor               \n    for f, floor_id in enumerate(floors_list):\n        floor_name = get_site_floor_from_id(site_id, floor_id)\n        print (\"\\033[A                                                                                                                       \\033[A\", end=\"\\r\")\n        print(f\"Site: {s+1:02d}/{s_len} | Fixing Predicted XY | Floor: {f+1}/{len(floors_list)} ({floor_name}) | Gererating Grid \", end=\"\\r\")\n        floor_wp_grid, floor_map_grid = generate_grid_for_floor(site_id, floor_name, 2)\n        sel_floor = site_df[(site_df.pred_f == floor_id)].reset_index(drop=True)\n        \n        for p, path_id in enumerate(paths_list):\n            print (\"\\033[A                                                                                                                                                       \\033[A\", end=\"\\r\")\n            print(f\"Site: {s+1:02d}/{s_len} | Fixing Predicted XY (A:{alpha},B:{betta}) | Floor: {f+1}/{len(floors_list)} ({floor_name}) | Path: {p+1}/{len(paths_list)} ({path_id})\", end=\"\\r\")\n            sel_path = sel_floor[(sel_floor.path == path_id)].reset_index(drop=True)\n\n            if sel_path.empty == False:\n                knn_path = sel_path[['path','timestamp', 'pred_x', 'pred_y', 'f']]\n                knn_path = knn_path.rename({'pred_x': 'x', 'pred_y': 'y', 'f': 'floor'}, axis='columns').sort_values('timestamp')\n                knn_path['site'] = site_id\n\n                knn_pdr_cm_path = correct_path(site_id, knn_path, alpha, betta)\n                knn_pdr_cm_wpgrid_path = snap_to_grid(knn_pdr_cm_path, floor_wp_grid, thresshold)\n\n                sel_path['fixed_x'] = knn_pdr_cm_wpgrid_path['x']\n                sel_path['fixed_y'] = knn_pdr_cm_wpgrid_path['y']\n                wp_fixed_df = wp_fixed_df.append(sel_path, ignore_index=True)\n    \n    return wp_fixed_df           ","metadata":{"execution":{"iopub.status.busy":"2021-06-13T18:35:26.355438Z","iopub.execute_input":"2021-06-13T18:35:26.355811Z","iopub.status.idle":"2021-06-13T18:35:26.376504Z","shell.execute_reply.started":"2021-06-13T18:35:26.355779Z","shell.execute_reply":"2021-06-13T18:35:26.375404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stop = stop","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:51:35.263480Z","iopub.execute_input":"2021-06-13T13:51:35.264195Z","iopub.status.idle":"2021-06-13T13:51:35.403657Z","shell.execute_reply.started":"2021-06-13T13:51:35.264157Z","shell.execute_reply":"2021-06-13T13:51:35.401782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# RUNTIME","metadata":{}},{"cell_type":"markdown","source":"### KNN Analysis","metadata":{}},{"cell_type":"code","source":"eval_df = pd.read_csv('../input/indoor-location-knn-kfold-predictions/knn_fold_eval.csv')\neval_df = eval_df.drop(columns=['y_acc','wp_mpe','x_acc'])\neval_df = eval_df.rename({'f_acc': 'floor_knn_acc'}, axis='columns')\nsites_path  = sorted(glob.glob('../input/indoor-location-knn-kfold-predictions/5*.csv'))\nwp_knn_mpe = []\nwp_knn_acc = []\nfloor_knn_acc = []\n\nwp_knn_rmse = []\nwp_knn_xrmse = []\nwp_knn_yrmse = []\n\n\nfor site_path in sites_path:\n    site = pd.read_csv(site_path)\n    wp_knn_mpe.append(np.mean(np.sqrt(np.power(site.pred_x - site.x, 2) + np.power(site.pred_y - site.y, 2))))\n    \n    wp_knn_acc.append(sum(site.x == site.pred_x)/site.shape[0])\n    floor_knn_acc.append(sum(site.f == site.pred_f)/site.shape[0])\n    \n    wp_knn_xrmse.append(mean_squared_error(site.x, site.pred_x, squared=False))\n    wp_knn_yrmse.append(mean_squared_error(site.y, site.pred_y, squared=False))\n    \nwp_knn_rmse = np.sqrt((np.power(wp_knn_xrmse, 2) + np.power(wp_knn_yrmse, 2)))    \n\n\neval_df['wp_knn_rmse'] = wp_knn_rmse\neval_df['wp_knn_mpe'] = wp_knn_mpe\neval_df['wp_knn_acc'] = wp_knn_acc\neval_df['floor_knn_acc'] = floor_knn_acc\nprint(f\"Floor KNN Acc | Mean/Median {np.mean(eval_df.floor_knn_acc):,.2%} / {np.median(eval_df.floor_knn_acc):,.2%}  | Min/Max: {np.min(eval_df.floor_knn_acc):,.2%} / {np.max(eval_df.floor_knn_acc):,.2%}\")\nprint(f\"XY KNN MPE | Mean/Median {np.mean(eval_df.wp_knn_mpe):0.2f} / {np.median(eval_df.wp_knn_mpe):0.2f} | Min/Max: {np.min(eval_df.wp_knn_mpe):0.2f} / {np.max(eval_df.wp_knn_mpe):0.2f}\")\nprint(f\"XY KNN Acc | Mean/Median {np.mean(eval_df.wp_knn_acc):,.2%} / {np.median(eval_df.wp_knn_acc):,.2%} | Min/Max: {np.min(eval_df.wp_knn_acc):,.2%} / {np.max(eval_df.wp_knn_acc):,.2%}\")\ndisplay(eval_df)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T19:15:57.772391Z","iopub.execute_input":"2021-06-13T19:15:57.772957Z","iopub.status.idle":"2021-06-13T19:15:58.059917Z","shell.execute_reply.started":"2021-06-13T19:15:57.772920Z","shell.execute_reply":"2021-06-13T19:15:58.059217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#WP RMSE\nfig_wp_rmse = go.Figure(data=[\n    go.Bar(name='Predicted', x=eval_df.index.astype(str), y=eval_df.wp_knn_rmse,text=eval_df.wp_knn_rmse,textposition='auto'),\n])\nfig_wp_rmse.update_traces(texttemplate='%{text:.2f}', textposition='outside')\nfig_wp_rmse.update_layout(legend=dict(yanchor=\"top\",y=0.95,xanchor=\"left\",x=0.01), barmode='group', xaxis_tickangle=0)\n\n#WP MPE\nfig_wp_mpe = go.Figure(data=[\n    go.Bar(name='Predicted', x=eval_df.index.astype(str), y=eval_df.wp_knn_mpe,text=eval_df.wp_knn_mpe,textposition='auto'),\n])\nfig_wp_mpe.update_traces(texttemplate='%{text:.2f}', textposition='outside')\nfig_wp_mpe.update_layout(legend=dict(yanchor=\"top\",y=0.95,xanchor=\"left\",x=0.01), barmode='group', xaxis_tickangle=0)\n\n#WP ACC\nfig_wp_acc = go.Figure(data=[\n    go.Bar(name='Predicted', x=eval_df.index.astype(str), y=eval_df.wp_knn_acc,text=eval_df.wp_knn_acc,textposition='auto')\n])\nfig_wp_acc.update_traces(texttemplate='%{text:,.0%}', textposition='outside')\nfig_wp_acc.update_layout(yaxis=dict(tickformat=\".0%\"))\nfig_wp_acc.update_layout(legend=dict(yanchor=\"top\",y=0.25,xanchor=\"left\",x=0.01))\n#Floor ACC\nfig_f_acc = go.Figure(data=[\n    go.Bar(name='Predicted', x=eval_df.index.astype(str), y=eval_df.floor_knn_acc,text=eval_df.floor_knn_acc,textposition='auto'),\n])\nfig_f_acc.update_traces(texttemplate='%{text:,.0%}', textposition='outside')\nfig_f_acc.update_layout(yaxis=dict(tickformat=\".0%\"))\nfig_f_acc.update_layout(legend=dict(yanchor=\"top\",y=0.25,xanchor=\"left\",x=0.01))\n\nfig_wp_rmse.show()\nfig_wp_mpe.show()\nfig_wp_acc.show()\nfig_f_acc.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:51:35.406328Z","iopub.status.idle":"2021-06-13T13:51:35.406810Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_df = pd.read_csv('../input/mcs-submissions/final_fixe_eval.csv')\nprint(f\"Floor KNN Acc | Mean/Median {np.mean(eval_df.floor_knn_acc):,.2%} / {np.median(eval_df.floor_knn_acc):,.2%}  | Min/Max: {np.min(eval_df.floor_knn_acc):,.2%} / {np.max(eval_df.floor_knn_acc):,.2%}\")\nprint(f\"Floor FIX Acc | Mean/Median {np.mean(eval_df.floor_fix_acc):,.2%} / {np.median(eval_df.floor_fix_acc):,.2%}  | Min/Max: {np.min(eval_df.floor_fix_acc):,.2%} / {np.max(eval_df.floor_fix_acc):,.2%}\\n\")\n\nprint(f\"XY KNN MPE | Mean/Median {np.mean(eval_df.wp_knn_mpe):0.2f} / {np.median(eval_df.wp_knn_mpe):0.2f} | Min/Max: {np.min(eval_df.wp_knn_mpe):0.2f} / {np.max(eval_df.wp_knn_mpe):0.2f}\")\nprint(f\"XY FIX MPE | Mean/Median {np.mean(eval_df.wp_fix_mpe):0.2f} / {np.median(eval_df.wp_fix_mpe):0.2f} | Min/Max: {np.min(eval_df.wp_fix_mpe):0.2f} / {np.max(eval_df.wp_fix_mpe):0.2f}\")\nprint(f\"XY GRID MPE | Mean/Median {np.mean(eval_df.wp_grid_mpe):0.2f} / {np.median(eval_df.wp_grid_mpe):0.2f} | Min/Max: {np.min(eval_df.wp_grid_mpe):0.2f} / {np.max(eval_df.wp_grid_mpe):0.2f}\\n\")\n\nprint(f\"XY KNN Acc | Mean/Median {np.mean(eval_df.wp_knn_acc):,.2%} / {np.median(eval_df.wp_knn_acc):,.2%} | Min/Max: {np.min(eval_df.wp_knn_acc):,.2%} / {np.max(eval_df.wp_knn_acc):,.2%}\")\nprint(f\"XY FIX Acc | Mean/Median {np.mean(eval_df.wp_fix_acc):,.2%} / {np.median(eval_df.wp_fix_acc):,.2%} | Min/Max: {np.min(eval_df.wp_fix_acc):,.2%} / {np.max(eval_df.wp_fix_acc):,.2%}\")\ndisplay(eval_df.style.apply(highlight_eval, axis=None))","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:51:35.407873Z","iopub.status.idle":"2021-06-13T13:51:35.408310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = go.Figure(data=[go.Bar(x=eval_df.index.astype(str), y=eval_df.wp_knn_mpe,text=eval_df.wp_knn_mpe,textposition='auto',)])\n#fig.update_traces(texttemplate='%{text:,.0%}', textposition='outside')\n#fig.update_layout(yaxis=dict(tickformat=\".0%\"))\n\nfig.update_traces(texttemplate='%{text:0.2f}', textposition='outside')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:51:35.409214Z","iopub.status.idle":"2021-06-13T13:51:35.409708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# POST-PROCESSING","metadata":{}},{"cell_type":"code","source":"#fixed_df = pd.DataFrame(columns=['path','timestamp','x','y','pred_x','pred_y','f','pred_f','fixed_f']) \npd.options.mode.chained_assignment = None\nwp_fix_mpe = []\nwp_fix_acc = []\nfloor_fix_acc = []\nwp_fix_rmse = []","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sites_path = sites_path[22:24]","metadata":{"execution":{"iopub.status.busy":"2021-06-13T18:30:35.465548Z","iopub.execute_input":"2021-06-13T18:30:35.466203Z","iopub.status.idle":"2021-06-13T18:30:35.472809Z","shell.execute_reply.started":"2021-06-13T18:30:35.466144Z","shell.execute_reply":"2021-06-13T18:30:35.471207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfor s, site_path in enumerate(sites_path):\n    \n    #load\n    site_id = site_path.split('/')[-1].split('.')[0]\n    site_df = pd.read_csv(site_path)\n    paths_list = site_df.groupby('path')['path'].count().reset_index(name=\"count\").path.to_numpy()\n    floors_list = site_df.groupby('pred_f')['pred_f'].count().reset_index(name=\"count\").pred_f.to_numpy()\n    \n    #fix\n    wp_fixed_df = wp_fix(site_id, site_df, floors_list, paths_list, s, len(sites_path))\n    floor_fixed_df = fix_floor(wp_fixed_df, paths_list, s+1, len(sites_path))\n    \n    outdir = './fixed_predictions'\n    if not os.path.exists(outdir):\n        os.mkdir(outdir)\n    wp_fixed_df.to_csv('./fixed_predictions/{}.csv'.format(site_id),index=False)\n    \n    #evaluate\n    fixed_df = floor_fixed_df.copy()\n    wp_fix_mpe.append(np.mean(np.sqrt(np.power(fixed_df.fixed_x - fixed_df.x, 2) + np.power(fixed_df.fixed_y - fixed_df.y, 2))))\n    wp_fix_acc.append(sum(fixed_df.x == fixed_df.fixed_x)/fixed_df.shape[0])\n    \n    wp_fix_xrmse = mean_squared_error(fixed_df.x, fixed_df.fixed_x, squared=False)\n    wp_fix_yrmse = mean_squared_error(fixed_df.y, fixed_df.fixed_y, squared=False)\n    wp_fix_rmse.append(np.sqrt((np.power(wp_fix_xrmse, 2) + np.power(wp_fix_yrmse, 2))))\n    \n    #add score\n    floor_fix_acc.append(sum(fixed_df.f == fixed_df.fixed_f)/fixed_df.shape[0])\n\neval_df['floor_fix_acc'] = floor_fix_acc\neval_df['wp_fix_rmse'] = wp_fix_rmse\neval_df['wp_fix_mpe'] = wp_fix_mpe\neval_df['wp_fix_acc'] = wp_fix_acc","metadata":{"execution":{"iopub.status.busy":"2021-06-13T18:35:34.144677Z","iopub.execute_input":"2021-06-13T18:35:34.145251Z","iopub.status.idle":"2021-06-13T19:08:19.915221Z","shell.execute_reply.started":"2021-06-13T18:35:34.145217Z","shell.execute_reply":"2021-06-13T19:08:19.913293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_df['floor_fix_acc'] = floor_fix_acc\neval_df['wp_fix_rmse'] = wp_fix_rmse\neval_df['wp_fix_mpe'] = wp_fix_mpe\neval_df['wp_fix_acc'] = wp_fix_acc","metadata":{"execution":{"iopub.status.busy":"2021-06-13T19:16:01.014865Z","iopub.execute_input":"2021-06-13T19:16:01.015221Z","iopub.status.idle":"2021-06-13T19:16:01.023274Z","shell.execute_reply.started":"2021-06-13T19:16:01.015190Z","shell.execute_reply":"2021-06-13T19:16:01.022235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EVALUATIONS","metadata":{}},{"cell_type":"code","source":"#eval_df = pd.read_csv('../input/mcs-submissions/final_fixe_eval.csv')\nprint(f\"Floor KNN Acc | Mean/Median {np.mean(eval_df.floor_knn_acc):,.2%} / {np.median(eval_df.floor_knn_acc):,.2%}  | Min/Max: {np.min(eval_df.floor_knn_acc):,.2%} / {np.max(eval_df.floor_knn_acc):,.2%}\")\nprint(f\"Floor FIX Acc | Mean/Median {np.mean(eval_df.floor_fix_acc):,.2%} / {np.median(eval_df.floor_fix_acc):,.2%}  | Min/Max: {np.min(eval_df.floor_fix_acc):,.2%} / {np.max(eval_df.floor_fix_acc):,.2%}\\n\")\n\nprint(f\"XY KNN RSME | Mean/Median {np.mean(eval_df.wp_knn_rmse):0.2f} / {np.median(eval_df.wp_knn_rmse):0.2f} | Min/Max: {np.min(eval_df.wp_knn_rmse):0.2f} / {np.max(eval_df.wp_knn_rmse):0.2f}\")\nprint(f\"XY FIX RSME | Mean/Median {np.mean(eval_df.wp_fix_rmse):0.2f} / {np.median(eval_df.wp_fix_rmse):0.2f} | Min/Max: {np.min(eval_df.wp_fix_rmse):0.2f} / {np.max(eval_df.wp_fix_rmse):0.2f}\\n\")\n\nprint(f\"XY KNN MDE | Mean/Median {np.mean(eval_df.wp_knn_mpe):0.2f} / {np.median(eval_df.wp_knn_mpe):0.2f} | Min/Max: {np.min(eval_df.wp_knn_mpe):0.2f} / {np.max(eval_df.wp_knn_mpe):0.2f}\")\nprint(f\"XY FIX MDE | Mean/Median {np.mean(eval_df.wp_fix_mpe):0.2f} / {np.median(eval_df.wp_fix_mpe):0.2f} | Min/Max: {np.min(eval_df.wp_fix_mpe):0.2f} / {np.max(eval_df.wp_fix_mpe):0.2f}\\n\")\n\nprint(f\"XY KNN Acc | Mean/Median {np.mean(eval_df.wp_knn_acc):,.2%} / {np.median(eval_df.wp_knn_acc):,.2%} | Min/Max: {np.min(eval_df.wp_knn_acc):,.2%} / {np.max(eval_df.wp_knn_acc):,.2%}\")\nprint(f\"XY FIX Acc | Mean/Median {np.mean(eval_df.wp_fix_acc):,.2%} / {np.median(eval_df.wp_fix_acc):,.2%} | Min/Max: {np.min(eval_df.wp_fix_acc):,.2%} / {np.max(eval_df.wp_fix_acc):,.2%}\")\ndisplay(eval_df.style.apply(highlight_eval, axis=None))","metadata":{"execution":{"iopub.status.busy":"2021-06-13T19:23:32.203231Z","iopub.execute_input":"2021-06-13T19:23:32.203621Z","iopub.status.idle":"2021-06-13T19:23:32.251027Z","shell.execute_reply.started":"2021-06-13T19:23:32.203583Z","shell.execute_reply":"2021-06-13T19:23:32.250315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rmse_dec = 100- eval_df.wp_fix_rmse*100/eval_df.wp_knn_rmse\nrmse_dec[1]\nl1 = [ti for ti in eval_df.index.astype(str)]\nl2 = [f'(↘{ti:.0f}%)' for ti in rmse_dec]\nannotationsList = [l1,l2]\n#WP MPE\nfig_wp_rmse = go.Figure(data=[\n    go.Bar(name='Predicted', x=annotationsList, y=eval_df.wp_knn_rmse,text=eval_df.wp_knn_rmse,textposition='auto'),\n    go.Bar(name='Fixed', x=annotationsList, y=eval_df.wp_fix_rmse,text=eval_df.wp_fix_rmse,textposition='auto')\n])\nfig_wp_rmse.update_traces(texttemplate='%{text:.2f}', textposition='outside')\nfig_wp_rmse.update_layout(legend=dict(yanchor=\"top\",y=0.95,xanchor=\"left\",x=0.01), barmode='group', xaxis_tickangle=0)\n\nmpe_dec = 100- eval_df.wp_fix_mpe*100/eval_df.wp_knn_mpe\nmpe_dec[1]\nl1 = [ti for ti in eval_df.index.astype(str)]\nl2 = [f'(↘{ti:.0f}%)' for ti in mpe_dec]\nannotationsList = [l1,l2]\n#WP MPE\nfig_wp_mpe = go.Figure(data=[\n    go.Bar(name='Predicted', x=annotationsList, y=eval_df.wp_knn_mpe,text=eval_df.wp_knn_mpe,textposition='auto'),\n    go.Bar(name='Fixed', x=annotationsList, y=eval_df.wp_fix_mpe,text=eval_df.wp_fix_mpe,textposition='auto')\n])\nfig_wp_mpe.update_traces(texttemplate='%{text:.2f}', textposition='outside')\nfig_wp_mpe.update_layout(legend=dict(yanchor=\"top\",y=0.95,xanchor=\"left\",x=0.01), barmode='group', xaxis_tickangle=0)\n\n#WP ACC\nfig_wp_acc = go.Figure(data=[\n    go.Bar(name='Predicted', x=eval_df.index.astype(str), y=eval_df.wp_knn_acc,text=eval_df.wp_knn_acc,textposition='auto'),\n    go.Bar(name='Fixed', x=eval_df.index.astype(str), y=eval_df.wp_fix_acc,text=eval_df.wp_fix_acc,textposition='auto')\n    \n])\nfig_wp_acc.update_traces(texttemplate='%{text:,.0%}', textposition='outside')\nfig_wp_acc.update_layout(yaxis=dict(tickformat=\".0%\"))\nfig_wp_acc.update_layout(legend=dict(yanchor=\"top\",y=0.25,xanchor=\"left\",x=0.01))\n#Floor ACC\nfig_f_acc = go.Figure(data=[\n    go.Bar(name='Predicted', x=eval_df.index.astype(str), y=eval_df.floor_knn_acc,text=eval_df.floor_knn_acc,textposition='auto'),\n    go.Bar(name='Fixed', x=eval_df.index.astype(str), y=eval_df.floor_fix_acc,text=eval_df.floor_fix_acc,textposition='auto')\n])\nfig_f_acc.update_traces(texttemplate='%{text:,.0%}', textposition='outside')\nfig_f_acc.update_layout(yaxis=dict(tickformat=\".0%\"))\nfig_f_acc.update_layout(legend=dict(yanchor=\"top\",y=0.25,xanchor=\"left\",x=0.01))\n\nfig_wp_rmse.show()\nfig_wp_mpe.show()\nfig_wp_acc.show()\nfig_f_acc.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T19:20:17.606502Z","iopub.execute_input":"2021-06-13T19:20:17.606905Z","iopub.status.idle":"2021-06-13T19:20:17.676993Z","shell.execute_reply.started":"2021-06-13T19:20:17.606871Z","shell.execute_reply":"2021-06-13T19:20:17.676091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_df_1 = pd.read_csv('../input/mcs-submissions/fixed_eval_1.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:51:35.415164Z","iopub.status.idle":"2021-06-13T13:51:35.415634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Floor KNN Acc | Mean/Median {np.mean(eval_df_1.floor_knn_acc):,.2%} / {np.median(eval_df_1.floor_knn_acc):,.2%}  | Min/Max: {np.min(eval_df_1.floor_knn_acc):,.2%} / {np.max(eval_df_1.floor_knn_acc):,.2%}\")\nprint(f\"Floor FIX Acc | Mean/Median {np.mean(eval_df_1.floor_fix_acc):,.2%} / {np.median(eval_df_1.floor_fix_acc):,.2%}  | Min/Max: {np.min(eval_df_1.floor_fix_acc):,.2%} / {np.max(eval_df_1.floor_fix_acc):,.2%}\\n\")\n\nprint(f\"XY KNN MPE | Mean/Median {np.mean(eval_df_1.wp_knn_mpe):0.2f} / {np.median(eval_df_1.wp_knn_mpe):0.2f} | Min/Max: {np.min(eval_df_1.wp_knn_mpe):0.2f} / {np.max(eval_df_1.wp_knn_mpe):0.2f}\")\nprint(f\"XY FIX MPE | Mean/Median {np.mean(eval_df_1.wp_fix_mpe):0.2f} / {np.median(eval_df_1.wp_fix_mpe):0.2f} | Min/Max: {np.min(eval_df_1.wp_fix_mpe):0.2f} / {np.max(eval_df_1.wp_fix_mpe):0.2f}\\n\")\n\nprint(f\"XY KNN Acc | Mean/Median {np.mean(eval_df_1.wp_knn_acc):,.2%} / {np.median(eval_df_1.wp_knn_acc):,.2%} | Min/Max: {np.min(eval_df_1.wp_knn_acc):,.2%} / {np.max(eval_df_1.wp_knn_acc):,.2%}\")\nprint(f\"XY FIX Acc | Mean/Median {np.mean(eval_df_1.wp_fix_acc):,.2%} / {np.median(eval_df_1.wp_fix_acc):,.2%} | Min/Max: {np.min(eval_df_1.wp_fix_acc):,.2%} / {np.max(eval_df_1.wp_fix_acc):,.2%}\\n\")\n\ndisplay(eval_df_1.style.apply(highlight_eval, axis=None))","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:51:35.416719Z","iopub.status.idle":"2021-06-13T13:51:35.417187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#WP MPE\nfig_wp_mpe = go.Figure(data=[\n    go.Bar(name='Predicted', x=eval_df_1.index.astype(str), y=eval_df_1.wp_knn_mpe,text=eval_df_1.wp_knn_mpe,textposition='auto'),\n    go.Bar(name='Fixed', x=eval_df_1.index.astype(str), y=eval_df_1.wp_fix_mpe,text=eval_df_1.wp_fix_mpe,textposition='auto')\n])\nfig_wp_mpe.update_traces(texttemplate='%{text:.2f}', textposition='outside')\nfig_wp_mpe.update_layout(legend=dict(yanchor=\"top\",y=0.95,xanchor=\"left\",x=0.01))\n#WP ACC\nfig_wp_acc = go.Figure(data=[\n    go.Bar(name='Predicted', x=eval_df_1.index.astype(str), y=eval_df_1.wp_knn_acc,text=eval_df_1.wp_knn_acc,textposition='auto'),\n    go.Bar(name='Fixed', x=eval_df_1.index.astype(str), y=eval_df_1.wp_fix_acc,text=eval_df_1.wp_fix_acc,textposition='auto')\n])\nfig_wp_acc.update_traces(texttemplate='%{text:,.0%}', textposition='outside')\nfig_wp_acc.update_layout(yaxis=dict(tickformat=\".0%\"))\nfig_wp_acc.update_layout(legend=dict(yanchor=\"top\",y=0.25,xanchor=\"left\",x=0.01))\n#Floor ACC\nfig_f_acc = go.Figure(data=[\n    go.Bar(name='Predicted', x=eval_df_1.index.astype(str), y=eval_df_1.floor_knn_acc,text=eval_df_1.floor_knn_acc,textposition='auto'),\n    go.Bar(name='Fixed', x=eval_df_1.index.astype(str), y=eval_df_1.floor_fix_acc,text=eval_df_1.floor_fix_acc,textposition='auto')\n])\nfig_f_acc.update_traces(texttemplate='%{text:,.0%}', textposition='outside')\nfig_f_acc.update_layout(yaxis=dict(tickformat=\".0%\"))\nfig_f_acc.update_layout(legend=dict(yanchor=\"top\",y=0.25,xanchor=\"left\",x=0.01))\n\nfig_wp_mpe.show()\nfig_wp_acc.show()\nfig_f_acc.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:51:35.418226Z","iopub.status.idle":"2021-06-13T13:51:35.418693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Example Visualization","metadata":{}},{"cell_type":"code","source":"#../input/indoor-location-knn-kfold-predictions/5a0546857ecc773753327266.csv","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:51:35.419778Z","iopub.status.idle":"2021-06-13T13:51:35.420244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"site_sample_path = '../input/indoor-location-knn-kfold-predictions/5c3c44b80379370013e0fd2b.csv'\nsite_to_check = site_sample_path.split('/')[-1].split('.')[0]\nsite_df = pd.read_csv(site_sample_path)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:51:35.421199Z","iopub.status.idle":"2021-06-13T13:51:35.421653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df = site_df[['x','pred_x']].sort_values('pred_x')\ny_range = list(range(1, len(sample_df.x)))\nstdev = mean_squared_error(sample_df.x, sample_df.pred_x, squared=False)\nfig3 = go.Figure()\nfig3.add_trace(go.Scatter(y=sample_df.x, x=y_range, mode='markers', name='Original'))\nfig3.add_trace(go.Scatter(y=sample_df.pred_x, x=y_range, mode='lines', name='Predictions', line_shape='spline'))\nfig3.add_trace(go.Scatter(y=sample_df.pred_x+stdev, x=y_range, mode='lines', name='Residual Standard Deviation', line = dict(color = 'green', dash = 'dash')))\nfig3.add_trace(go.Scatter(y=sample_df.pred_x-stdev, x=y_range, mode='lines', name='RMSE', showlegend=False, line = dict(color = 'green', dash = 'dash')))\n\n\nfig3.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:51:35.422524Z","iopub.status.idle":"2021-06-13T13:51:35.422987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"floor_to_check = get_site_floor_from_id(site_to_check, rand_path.pred_f.sample(1).values[0])\nfloor_paths_df, floor_wp_grid, floor_map_grid = generate_grid_for_floor(site_to_check, floor_to_check, 3, True)\nprint('GRID GENERATING IS COMPLITE')","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:51:35.423840Z","iopub.status.idle":"2021-06-13T13:51:35.424261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rand_path_id = site_df.path[site_df.pred_f == 0].sample(1).values[0]\n#rand_path_id = '5e15bf941506f2000638fec5'\n#rand_path_id = '5dd2616e44333f00067a9979' #all wrong wp fixed, site 5d27097f03f801723c320d97\nrand_path = site_df[site_df.path == rand_path_id].sort_values('timestamp')\ndisplay(rand_path.style.apply(highlight_dif, axis=None))","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:51:35.425220Z","iopub.status.idle":"2021-06-13T13:51:35.425687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.options.mode.chained_assignment = None\nknn_path = rand_path.drop(columns=['x', 'y', 'f'])\nknn_path = knn_path.rename({'pred_x': 'x', 'pred_y': 'y', 'pred_f': 'floor'}, axis='columns').sort_values('timestamp')\nknn_path['site'] = site_to_check\n\ntemp = knn_path.copy()\nknn_pdr_cm_path = correct_path(site_to_check, temp, 2, 3)\n\ntemp = knn_path.copy()\ntemp = snap_to_grid(temp, floor_wp_grid, 2)\nknn_wpgrid_path = temp\n\ntemp = knn_path.copy()\ntemp = snap_to_grid(temp, floor_map_grid, 5)\nknn_mapgrid_path = temp\n\ntemp = snap_to_grid(knn_pdr_cm_path, floor_wp_grid, 5)\nknn_pdr_cm_wpgrid_path = temp\n\ntemp = snap_to_grid(knn_pdr_cm_path, floor_map_grid, 5)\nknn_pdr_cm_mapgrid_path = temp\n\noriginal_path = floor_paths_df[floor_paths_df.path == rand_path_id]\noriginal_path['path'] = '1.Original'\nknn_path['path'] = '2.KNN only'\nknn_pdr_cm_path['path'] = '3.KNN+PDR+CM'\nknn_pdr_cm_wpgrid_path['path'] = '4.KNN+PDR+CM+WP-Grid'\n\nknn_wpgrid_path['path'] = '6.1.KNN+WP-Grid'\nknn_mapgrid_path['path'] = '6.1.KNN+Gen-Grid'\nknn_pdr_cm_mapgrid_path['path'] = '6.3.KNN+PDR+CM+Gen-Grid'\n\ncompare_paths = original_path.append([knn_path, knn_pdr_cm_path, knn_wpgrid_path, knn_mapgrid_path, knn_pdr_cm_wpgrid_path, knn_pdr_cm_mapgrid_path], ignore_index=True)\ndel compare_paths['xy']\n#display(compare_paths)\nprint('POST-PROCESSING IS COMPLITE')","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:51:35.426722Z","iopub.status.idle":"2021-06-13T13:51:35.427163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_floor(site_to_check, floor_to_check, compare_paths, floor_wp_grid, floor_map_grid).show()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:51:35.428074Z","iopub.status.idle":"2021-06-13T13:51:35.428497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sites_path","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:51:35.429654Z","iopub.status.idle":"2021-06-13T13:51:35.430103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_map_path","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:51:35.431067Z","iopub.status.idle":"2021-06-13T13:51:35.431491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_map_path = sites_path[22:24]\nfor s, site_path in enumerate(all_map_path):\n    print(site_path)\n    site_id = site_path.split('/')[-1].split('.')[0]\n    floor_to_check = get_site_floor_from_id(site_id, 0)\n    if floor_to_check == 0:\n        floor_to_check = get_site_floor_from_id(site_id, 1)\n    print(s, site_id, floor_to_check)\n    floor_wp_grid, floor_map_grid = generate_grid_for_floor(site_id, floor_to_check, 4)\n    plot_floor(site_id, floor_to_check, None, floor_wp_grid, floor_map_grid).show()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:51:35.432802Z","iopub.status.idle":"2021-06-13T13:51:35.433233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}