{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install nudged","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport scipy.stats as stats\nfrom pathlib import Path\nimport glob\nimport json\nimport psutil\nimport random\nimport os\nimport time\nimport sys\nimport math\nimport scipy.interpolate\nimport scipy.sparse\nfrom tqdm import tqdm\nfrom contextlib import contextmanager\nimport matplotlib.pylab as plt\nimport shapely\nimport matplotlib\nfrom shapely.geometry import shape, GeometryCollection, Polygon, MultiPolygon\nfrom shapely.affinity import affine_transform\nfrom PIL import Image, ImageOps\nimport nudged\nfrom skimage.morphology import convex_hull_image\nimport matplotlib.pyplot as plt\nimport plotly.graph_objs as go\nimport pickle\nfrom shapely.geometry import Point\nfrom scipy.spatial.distance import cdist","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Provided GitHub functions\n!cp -r ../input/indoor-location-competition-20-git/* ./\nimport io_f\nimport visualize_f \nimport compute_f\nimport main  ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# FUNCTIONS","metadata":{}},{"cell_type":"markdown","source":"* ### Additional","metadata":{}},{"cell_type":"code","source":"floors_key_assign = {'1F':0, '2F':1, '3F':2, '4F':3, '5F':4, '6F':5, '7F':6, '8F':7, '9F':8, \n          'B':-1, 'B1':-1,'B2':-2, 'B3':-3, 'BF':-1, 'BM':-1, \n          'F1':0, 'F2':1, 'F3':2, 'F4':3, 'F5':4, 'F6':5,'F7':6, 'F8':7, 'F9':8, 'F10':9,\n          'G':0, 'LG1':-1, 'LG2':-2, \n          \"L1\":0,\"L2\":1,\"L3\":2,\"L4\":3,\"L5\":4,\"L6\":5,\"L7\":6,\"L8\":7,\"L9\":8,\"L10\":9,'L11':10, \n          'LM':0, 'M':0, 'P1':-1, 'P2':-2,\n          \"地下一层\":-1,\"一层\":0,\"二层\":1,\"三层\":2,\"四层\":3}\ndef get_site_floor_from_id(site, floor_id):\n    site_p = \"../input/indoor-location-navigation/train/\"+site\n    floor_p = \"/*\"\n    site_floor_names = []\n    floors_files = sorted(glob.glob(site_p + floor_p))\n    for f, floor in enumerate(floors_files):\n        floor_name = floor.split('/')[-1]\n        site_floor_names.append(floor_name)\n    for key_f in floors_key_assign:\n        if key_f in site_floor_names:\n            if floors_key_assign[key_f] == floor_id:\n                return key_f\n    return floor_id","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def highlight_dif(x):\n    r = 'red'\n    g = 'green'\n\n    m1 = x['f'] != x['pred_f']\n    m2 = x['f'] == x['pred_f']\n    \n    m3 = x['x'] != x['pred_x']\n    m4 = x['x'] == x['pred_x']\n    \n    m5 = x['y'] != x['pred_y']\n    m6 = x['y'] == x['pred_y']\n\n    df1 = pd.DataFrame('color: ', index=x.index, columns=x.columns)\n    #rewrite values by boolean masks\n    df1['pred_f'] = np.where(m1, 'color: {}'.format(r), df1['pred_f'])\n    df1['pred_f'] = np.where(m2, 'color: {}'.format(g), df1['pred_f'])\n    \n    df1['pred_x'] = np.where(m3, 'color: {}'.format(r), df1['pred_x'])\n    df1['pred_x'] = np.where(m4, 'color: {}'.format(g), df1['pred_x'])\n    \n    df1['pred_y'] = np.where(m5, 'color: {}'.format(r), df1['pred_y'])\n    df1['pred_y'] = np.where(m6, 'color: {}'.format(g), df1['pred_y'])\n    return df1","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* ### Plot on Map","metadata":{}},{"cell_type":"code","source":"def plot_floor(\n    site,\n    floorNo,\n    paths_wp_df=None,\n    display_corridors=True,\n    display_map=True,\n    base=\"../input/indoor-location-navigation\",\n    base_pol = \"../input/indoor-location-navigation-scaled-geojson/scaled_geojson\",\n):\n    map_floor = floorNo\n    # Prepare width_meter & height_meter (taken from the .json file)\n    floor_plan_filename = f\"{base}/metadata/{site}/{map_floor}/floor_image.png\"\n    json_plan_filename = f\"{base}/metadata/{site}/{map_floor}/floor_info.json\"\n    with open(json_plan_filename) as json_file:\n        json_data = json.load(json_file)\n    width_meter = json_data[\"map_info\"][\"width\"]\n    height_meter = json_data[\"map_info\"][\"height\"]\n    floor_img = f\"{base}/metadata/{site}/{map_floor}/floor_image.png\"\n    \n    file = f\"{base_pol}/{site}/{map_floor}/shapely_geometry.pkl\"\n    with open(file, 'rb') as f:\n        geometry = pickle.load(f)\n    \n    m_opacity=0.8\n    colorbar_title=\"RSSI\"\n    #Display\n    fig = go.Figure()\n    if display_corridors:\n        for coord in extract_coords_from_polygon(geometry):\n            x, y = coord\n            fig.add_trace(\n                go.Scattergl(\n                    x=x,\n                    y=y,\n                    opacity=0.4,\n                    text=None,\n                    marker_color='orange',\n                    hoverinfo='skip',\n                    showlegend=False\n                ))\n    if paths_wp_df is not None:\n        fig.add_trace(\n            go.Scatter(x=paths_wp_df.x,\n                       y=paths_wp_df.y,\n                       mode='markers',\n                       marker=dict(size=20,\n                                   color=paths_wp_df.rssi,\n                                   colorbar=dict(title=colorbar_title),\n                                   colorscale=\"Rainbow\"),\n                       text=paths_wp_df.rssi,\n                       opacity=0.7,\n                        showlegend=False))\n    \n    if display_map:\n        floor_plan = Image.open(floor_img)\n        fig.update_layout(images=[\n            go.layout.Image(\n                source=floor_plan,\n                xref=\"x\",\n                yref=\"y\",\n                x=0,\n                y=height_meter,\n                sizex=width_meter,\n                sizey=height_meter,\n                sizing=\"contain\",\n                opacity=0.5,\n                layer=\"below\",\n            )\n        ])\n    \n\n    \n    # configure\n    fig.update_xaxes(autorange=False, range=[0, width_meter])\n    fig.update_yaxes(autorange=False, range=[0, height_meter], scaleanchor=\"x\", scaleratio=1)\n    fig.update_layout(coloraxis_colorbar=dict(title=\"RSSI\"))\n    fig.update_layout(\n        title=go.layout.Title(\n            text=\"Site: %s Floor: %s\" % (site, map_floor),\n            xref=\"paper\",\n            x=0,\n        ),\n        autosize=True,\n        width=700,\n        height=200 + 700 * height_meter / width_meter,\n        template=\"plotly_white\",\n    )\n\n    return fig\n\n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* ### Generate Grid from Map","metadata":{}},{"cell_type":"code","source":"def generate_grid_from_map(stite_check, floor_check, step = 2):\n    print (\"\\033[A                                                          \\033[A\", end=\"\\r\")\n    print(\"Map Grid Generatig...\", end=\"\\r\")\n    base_pol = \"../input/indoor-location-navigation-scaled-geojson/scaled_geojson\"\n    file = f\"{base_pol}/{stite_check}/{floor_check}/shapely_geometry.pkl\"\n    with open(file, 'rb') as f:\n        corridor = pickle.load(f)\n    \n    x_min, y_min, x_max, y_max = get_bounding_box(corridor)\n\n    x_range = range(math.ceil(x_min), math.floor(x_max), step)\n    y_range = range(math.ceil(y_min), math.floor(y_max), step)\n\n    valid_coords = []\n\n    for x in x_range:\n        for y in y_range:\n            if Point(x, y).within(corridor):\n                valid_coords.append([x, y])\n\n    return np.array(valid_coords)\n\ndef get_bounding_box(shapes):\n    \"\"\"\n    To extract bounding box from Polygon\n    \"\"\"\n    x_min = 10000\n    y_min = 10000\n    x_max = 0\n    y_max = 0\n    \n    if type(shapes) == Polygon:\n            shapes = [shapes]\n    for shape in shapes:\n        x, y = shape.exterior.xy\n        x_min = min(min(x), x_min)\n        y_min = min(min(y), y_min)\n        x_max = max(max(x), x_max)\n        y_max = max(max(y), y_max)\n    return x_min, y_min, x_max, y_max\n\ndef extract_coords_from_polygon(polygon):\n    coords = []\n    if type(polygon) == MultiPolygon:\n        polygons = polygon.geoms\n    else:\n        polygons = [polygon]\n\n    for polygon in polygons:\n        x, y = polygon.exterior.xy\n        coords.append((np.array(x), np.array(y)))\n        for interior in polygon.interiors:\n            x, y = interior.xy\n            coords.append((np.array(x), np.array(y)))\n\n    return coords","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* ### Generate Grid from Paths","metadata":{}},{"cell_type":"code","source":"def generate_grid_from_wp(site:str=None,floor:str=None, start:int=None, stop:int=None):\n    site_p = \"../input/indoor-location-navigation/train/*\"\n    floor_p = \"/*\"\n    path_p = \"/*\"\n    \n    if (site is not None) & (floor is None):\n        site_p = \"../input/indoor-location-navigation/train/\" + site\n        print (\"\\033[A                                                          \\033[A\", end=\"\\r\")\n        print(\"Grid Generatig Based on Paths in {} from all Floors...\".format(site))\n\n    if (site is not None) & (floor is not None):\n        site_p = \"../input/indoor-location-navigation/train/\" + site\n        floor_p = \"/\" + floor\n        print (\"\\033[A                                                          \\033[A\", end=\"\\r\")\n        print(\"Grid Generatig Based on Paths in {} from {}...\".format(site,floor))\n    \n    #Go through sites\n    sites = sorted(glob.glob(site_p))\n    \n    if (start is not None) & (stop is not None):\n        sites = sites[start:stop]\n    \n    #Create DF of all waypoints in whole site\n    main_wp_df = pd.DataFrame(columns = ['site', 'floor', 'x', 'y'])\n    for s, site in enumerate(sites):\n        #print site info\n        site_id = site.split('/')[-1]\n        print(\"Site\", s+1, \"/\", len(sites), \"({})\".format(site_id), end=\"\\n\\r\")\n\n        #Create DF of all waypoints in whole site\n        site_wp_df = pd.DataFrame(columns = ['site', 'floor', 'x', 'y'])\n        total_path = 0\n\n        #Go through floors\n        floors = sorted(glob.glob(site + floor_p))  \n        for f, floor in enumerate(floors):\n            #print floor info\n            floor_id = floor.split('/')[-1]\n            print(\"~ Floor\", f+1, \"/\", len(floors), \"({})\".format(floor_id), end=\"\\n\\r\")\n\n            #Create DF of all waypoints on each floor\n            wp_floor_df = pd.DataFrame(columns = ['site', 'floor', 'x', 'y'])\n\n            #Go through paths\n            paths = sorted(glob.glob(floor + path_p))   \n            for p, path in enumerate(paths):                \n                total_path += len(paths)\n\n                #print path info\n                path_id = path.split('/')[-1].split('.')[0]\n                print (\"\\033[A                                                          \\033[A\", end=\"\\r\")\n                print(\"~~ Path\", p+1, \"/\", len(paths), \"({})\".format(path_id), end=\"\\r\")\n\n                #Exstract All Paths Waypoints\n                path_wp = io_f.read_data_file(path).waypoint           \n                for wp in path_wp:\n                    new_row = {'site':site_id, 'floor':floor_id, 'x':wp[1], 'y':wp[2]}\n                    wp_floor_df = wp_floor_df.append(new_row, ignore_index=True)\n\n                #Drop duplicate waypoints\n                wp_floor_df = wp_floor_df.drop_duplicates(subset=['x', 'y'], keep='last')\n\n            #load to site wp df\n            site_wp_df = site_wp_df.append(wp_floor_df, ignore_index=True)\n\n        main_wp_df = main_wp_df.append(site_wp_df, ignore_index=True)\n\n    site_floor_list = main_wp_df.groupby(['site', 'floor'],as_index=False).agg({'x': 'count'})    \n    if (site is not None) | (floor is not None):\n        return main_wp_df, site_floor_list\n    else:\n        return main_wp_df","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* ### Snap to Grid","metadata":{}},{"cell_type":"code","source":"def snap_to_grid(wp, grid, threshold):\n    print (\"\\033[A                                                          \\033[A\", end=\"\\r\")\n    print(\"Snapping to the Grid...\", end=\"\\r\")\n    wp = find_closest_grid(wp, grid)\n    wp['dist'] = np.sqrt((wp.x-wp.x_)**2 + (wp.y-wp.y_)**2)\n    \n    wp = snap_points_to_grid(wp, threshold=threshold)\n    wp = wp[['site', 'floor', 'path', 'timestamp','_x_','_y_']].rename(columns={'_x_':'x', '_y_':'y'})\n    return wp\n\ndef add_xy(df):\n    df['xy'] = [(x, y) for x,y in zip(df['x'], df['y'])]\n    return df\ndef closest_point(point, points):\n    \"\"\" Find closest point from a list of points. \"\"\"\n    return points[cdist([point], points).argmin()]\ndef find_closest_grid(wp, grid):\n    grid_df = pd.DataFrame(grid, columns = ['x','y'])\n    wp = add_xy(wp)\n    grid_df = add_xy(grid_df)\n    ds = []\n    for (site, myfloor), d in wp.groupby(['site','floor']):\n        true_floor_locs = grid_df.reset_index(drop=True)\n        if len(true_floor_locs) == 0:\n            print(f'Skipping {site} {myfloor}')\n            continue\n        d['matched_point'] = [closest_point(x, list(true_floor_locs['xy'])) for x in d['xy']]\n        d['x_'] = d['matched_point'].apply(lambda x: x[0])\n        d['y_'] = d['matched_point'].apply(lambda x: x[1])\n        ds.append(d)\n\n    wp = pd.concat(ds)\n    \n    return(wp)\ndef snap_points_to_grid(sub, threshold):\n    \"\"\"\n    Snap to grid if within a threshold.\n    \n    x, y are the predicted points.\n    x_, y_ are the closest grid points.\n    _x_, _y_ are the new predictions after post processing.\n    \"\"\"\n    sub['_x_'] = sub['x']\n    sub['_y_'] = sub['y']\n    sub.loc[sub['dist'] < threshold, '_x_'] = sub.loc[sub['dist'] < threshold]['x_']\n    sub.loc[sub['dist'] < threshold, '_y_'] = sub.loc[sub['dist'] < threshold]['y_']\n    return sub.copy()","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* ### Training & Validation","metadata":{}},{"cell_type":"code","source":"def train_way_knn(train_pdf: pd.DataFrame, n_neighbor, leaf):\n    from sklearn.neighbors import KNeighborsRegressor\n    from sklearn.neighbors import KNeighborsClassifier\n    \n    features = train_pdf.iloc[:,:-4]\n    x_label = train_pdf.iloc[:,-4]\n    y_label = train_pdf.iloc[:,-3]\n    f_label = train_pdf.iloc[:,-2]\n\n    x = KNeighborsRegressor(n_neighbors=n_neighbor,weights=\"uniform\",algorithm=\"ball_tree\",leaf_size=leaf)\n    x_model = x.fit(features, x_label)\n    y = KNeighborsRegressor(n_neighbors=n_neighbor,weights=\"uniform\",algorithm=\"ball_tree\",leaf_size=leaf)\n    y_model = y.fit(features, y_label)\n    f = KNeighborsClassifier(n_neighbors=n_neighbor,weights=\"uniform\",algorithm=\"ball_tree\",leaf_size=leaf)\n    f_model = f.fit(features, f_label)\n\n    return x_model, y_model, f_model\n\ndef model_validation(df, model_x, model_y, model_f):\n    v_features = df.iloc[:,:-4]\n    v_label_x = df.iloc[:,-4]\n    v_label_y = df.iloc[:,-3]\n    v_label_f = df.iloc[:,-2]\n    \n    x_pred = model_x.predict(v_features)\n    y_pred = model_y.predict(v_features)\n    f_pred = model_f.predict(v_features)\n        \n    df = df[[\"path\",'x','y','f']].copy()\n    df['pred_x'] = x_pred\n    df['pred_y'] = y_pred\n    df['pred_f'] = f_pred.astype(int)\n    df['wp_error'] = np.sqrt(np.power(df['pred_x'] - v_label_x, 2) + np.power(df['pred_y'] - v_label_y, 2))\n    \n    return df.reset_index(drop=True)","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* ### Load Timestamps","metadata":{}},{"cell_type":"code","source":"def load_timestamps(df, site_id):\n    site_p = \"../input/indoor-location-navigation/train/\"+site_id\n    path_p = \"/*/*\"\n    floor_paths_files = sorted(glob.glob(site_p + path_p))\n    final_df = pd.DataFrame(columns=['path','timestamp','x','y','pred_x','pred_y','f','pred_f'])\n    for p, path in enumerate(floor_paths_files):\n        path_id = path.split('/')[-1].split('.')[0]\n        print (\"\\033[A                                                                                                  \\033[A\", end=\"\\r\")\n        print(\"~Timestamp Assign: {:03d}/{} ({})\".format(p+1, len(floor_paths_files), path_id), end='\\r')\n        path_wp = pd.DataFrame(io_f.read_data_file(path).waypoint, columns=['timestamp', 'x', 'y']).reset_index(drop=True)\n        path_df = df[df.path == path_id].reset_index(drop=True)\n        #path_wp['timestamp'] = path_wp['timestamp'] - path_wp['timestamp'].min()\n        if path_df.shape[0] > 0:\n            #Assign most frequent floor\n            #temp = path_df.groupby(['pred_f'])[\"pred_f\"].count().reset_index(name=\"count\")\n            #pred_floor = temp.pred_f[temp.groupby('pred_f')['count'].idxmax()].values[0]\n            act_floor = path_df.f.values[0]\n\n            if path_wp.shape[0] == path_df.shape[0]:\n                df2 = pd.merge(path_wp,path_df, on=['x','y'])\n            else:\n                temp = pd.merge(path_wp,path_df, on=['x','y'])\n                temp = temp.groupby(['timestamp','pred_x','pred_y','pred_f'])[\"timestamp\"].count().reset_index(name=\"count\")\n                temp = temp.loc[temp.groupby('timestamp')['count'].idxmax()]\n                df2 = pd.merge(path_wp,temp.drop(columns=['count']), on=['timestamp'])\n\n            df2['path'] = path_id\n            df2['f'] = act_floor\n            #df2['pred_f'] = pred_floor\n            final_df = final_df.append(df2)\n    return final_df","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* ### Get All Path from Floor","metadata":{}},{"cell_type":"code","source":"def get_floor_paths(stite_check,floor_check):\n    print (\"\\033[A                                                          \\033[A\", end=\"\\r\")\n    print(\"Loading Floor Paths...\", end=\"\\r\")\n    base = '../input/indoor-location-navigation/train'\n    floor_paths_files = sorted(glob.glob(\"{}/{}/{}/*\".format(base,stite_check,floor_check)))\n    floor_paths_df = pd.DataFrame(columns = ['site', 'floor', 'path', 'timestamp', 'x', 'y'])\n    for floor_paths_file in floor_paths_files:\n        path_id = floor_paths_file.split('/')[-1].split('.')[0]\n        #Exstract All Paths Waypoints\n        path_wp = io_f.read_data_file(floor_paths_file).waypoint\n        path_wp_df = pd.DataFrame(columns = ['site', 'floor', 'path', 'timestamp', 'x', 'y'])\n        for wp in path_wp:\n                new_row = {'site':stite_check, 'floor':floor_check, 'path':path_id, 'timestamp':wp[0], 'x':wp[1], 'y':wp[2]}\n                path_wp_df = path_wp_df.append(new_row, ignore_index=True)\n\n        floor_paths_df = floor_paths_df.append(path_wp_df, ignore_index=True)\n    return (floor_paths_df)","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* ### Cost Minimisation","metadata":{}},{"cell_type":"code","source":"def compute_rel_positions(acce_datas, ahrs_datas, posi_datas):\n    step_timestamps, step_indexs, step_acce_max_mins = compute_f.compute_steps(acce_datas)\n    headings = compute_f.compute_headings(ahrs_datas)\n    stride_lengths = compute_f.compute_stride_length(step_acce_max_mins)\n    step_headings = compute_f.compute_step_heading(step_timestamps, headings)\n    rel_positions = compute_f.compute_rel_positions(stride_lengths, step_headings)\n    step_positions = compute_f.correct_positions(rel_positions, posi_datas)\n    return rel_positions, step_positions\n\ndef correct_path(site, path_df, alpha_mod):\n    site_p = \"../input/indoor-location-navigation/train/\"\n    path = path_df['path'].values[0]\n    floor = path_df['floor'].values[0]\n    T_ref  = path_df['timestamp'].values\n    xy_hat = path_df[['x', 'y']].values\n    \n    site_floor = get_site_floor_from_id(site,floor)\n    \n    example = io_f.read_data_file(f'{site_p}/{site}/{site_floor}/{path}.txt')\n    \n    rel_positions, step_positions = compute_rel_positions(example.acce, example.ahrs, path_df[['timestamp','x','y']].to_numpy())\n    posi_datas = pd.DataFrame(columns=['timestamp','x', 'y'])\n    posi_datas['timestamp'] = step_positions[:, 0]\n    posi_datas['x'] = step_positions[:, 1]\n    posi_datas['y'] = step_positions[:, 2]\n    posi_datas['floor'] = floor\n    posi_datas['path'] = path\n    \n    if T_ref[-1] > rel_positions[-1, 0]:\n        rel_positions = [np.array([[0, 0, 0]]), rel_positions, np.array([[T_ref[-1], 0, 0]])]\n    else:\n        rel_positions = [np.array([[0, 0, 0]]), rel_positions]\n    rel_positions = np.concatenate(rel_positions)\n    \n    T_rel = rel_positions[:, 0]\n    delta_xy_hat = np.diff(scipy.interpolate.interp1d(T_rel, np.cumsum(rel_positions[:, 1:3], axis=0), axis=0)(T_ref), axis=0)\n\n    N = xy_hat.shape[0]\n    delta_t = np.diff(T_ref)\n    alpha = (8.1)**(-alpha_mod) * np.ones(N)\n    beta  = (0.3 + 0.3 * 1e-3 * delta_t)**(-2)\n    A = scipy.sparse.spdiags(alpha, [0], N, N)\n    B = scipy.sparse.spdiags( beta, [0], N-1, N-1)\n    D = scipy.sparse.spdiags(np.stack([-np.ones(N), np.ones(N)]), [0, 1], N-1, N)\n\n    Q = A + (D.T @ B @ D)\n    c = (A @ xy_hat) + (D.T @ (B @ delta_xy_hat))\n    xy_star = scipy.sparse.linalg.spsolve(Q, c)\n    \n    path_df['x'] = xy_star[:, 0]\n    path_df['y'] = xy_star[:, 1]\n    \n    return posi_datas, path_df","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# SCRIPTS","metadata":{}},{"cell_type":"markdown","source":"* ### Training Script","metadata":{}},{"cell_type":"code","source":"def knn_site_train(site_to_check=None, n_split=10, ts_ass=True):\n    from sklearn.model_selection import KFold\n    from sklearn.model_selection import train_test_split\n    N_SPLITS = n_split\n    SEED = 42\n    def set_seed(seed=42):\n        random.seed(seed)\n        os.environ[\"PYTHONHASHSEED\"] = str(seed)\n        np.random.seed(seed)\n\n    set_seed(SEED)\n    score_df = pd.DataFrame()\n\n    feature_dir = \"../input/indoor-navigation-and-location-wifi-features/wifi_features/train\"\n    train_files = sorted(glob.glob(os.path.join(f'{feature_dir}/*_1000_train.csv')))\n    if site_to_check is not None:\n        train_files = sorted(glob.glob(os.path.join(f'{feature_dir}/{site_to_check}_1000_train.csv')))\n    sites_pred_eval = pd.DataFrame(columns = ['site','wp_mpe','f_acc'])\n    \n    for e, file in enumerate(train_files):\n        total_mpe = []\n        total_x_acc = []\n        total_y_acc = []\n        total_f_acc = []\n        \n        final_predictions = pd.DataFrame(columns = ['path','x','y','f','pred_x','pred_y','pred_f'])\n        site_id = file.split('/')[-1].split(\"_\")[0]\n        print(\"Site {}/{} ({})\".format(e+1, len(train_files), site_id))\n        print (\"\\033[A                                                                                                  \\033[A\", end=\"\\r\")\n        print(\"~Getting CSV...\", end='\\r')\n        data = pd.read_csv(file, index_col=0)\n        \n        kf = KFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED)\n        print(\"~Folding the Dataset...\", end='\\r')\n        for fold, (trn_idx, val_idx) in enumerate(kf.split(data.iloc[:, :-4])):\n            print (\"\\033[A                                                                                                  \\033[A\", end=\"\\r\")\n            print(\"~Folder {}/{} | Splitting Dataset...\".format(fold+1, N_SPLITS), end='\\r')\n            train = data.iloc[trn_idx]\n            valid = data.iloc[val_idx]\n\n            print (\"\\033[A                                                                                                  \\033[A\", end=\"\\r\")\n            print(\"~Folder {:02d}/{} | KNN Trtaining...\".format(fold+1, N_SPLITS), end='\\r')\n            model_knn_x, model_knn_y, model_knn_f = train_way_knn(train, 1, 17)\n\n            print (\"\\033[A                                                                                                  \\033[A\", end=\"\\r\")\n            print(\"~Folder {:02d}/{} | Validating...\".format(fold+1, N_SPLITS), end='\\r')\n            pred_knn_df = model_validation(valid, model_knn_x, model_knn_y, model_knn_f)\n\n            print (\"\\033[A                                                                                                  \\033[A\", end=\"\\r\")\n            print(\"~Folder {:02d}/{} | Finishing Up...\".format(fold+1, N_SPLITS), end='\\r')  \n            knn_mpe = pred_knn_df.wp_error.mean()\n            knn_x_acc = sum(pred_knn_df.x == pred_knn_df.pred_x)/pred_knn_df.shape[0]\n            knn_y_acc = sum(pred_knn_df.y == pred_knn_df.pred_y)/pred_knn_df.shape[0]\n            knn_f_acc = sum(pred_knn_df.f == pred_knn_df.pred_f)/pred_knn_df.shape[0]\n            total_mpe.append(knn_mpe)\n            total_x_acc.append(knn_x_acc)\n            total_y_acc.append(knn_y_acc)\n            total_f_acc.append(knn_f_acc)\n\n            if ts_ass:\n                final_predictions = final_predictions.append(pred_knn_df[['path','x','y','pred_x','pred_y','f','pred_f']], ignore_index=True)\n\n        \n        \n        print (\"\\033[A                                                                                                  \\033[A\", end=\"\\r\")\n        print(\"~DONE | WP MPE: {:.5f} | X ACC: {:.5f} | Y ACC: {:.5f} | F ACC: {:.5f}\".format(\n            np.mean(total_mpe), np.mean(total_x_acc), np.mean(total_y_acc), np.mean(total_f_acc)))\n\n        new_row = {'site':site_id, 'wp_mpe':np.mean(total_mpe),'x_acc':np.mean(total_x_acc),'y_acc':np.mean(total_y_acc), 'f_acc':np.mean(total_f_acc)}\n        sites_pred_eval = sites_pred_eval.append(new_row, ignore_index=True)\n        \n        if ts_ass:\n            final_predictions = load_timestamps(final_predictions, site_id)\n            final_predictions['timestamp'] = final_predictions['timestamp'].astype(int)\n            final_predictions = final_predictions.sort_values(['path','timestamp']).reset_index(drop=True)\n            final_predictions.to_csv('./knn_fold_predictions/{}.csv'.format(site_id),index=False)\n\n    return sites_pred_eval\n","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* ### Grid Generation","metadata":{}},{"cell_type":"code","source":"def generate_grid_for_floor(stite_check, floor_check, step):\n    #collect all waypoints on floor\n    floor_paths_df = get_floor_paths(stite_check,floor_check)\n    #grid from waypoints\n    floor_wp_grid_df, site_floor_list = generate_grid_from_wp(stite_check, floor_check)\n    floor_wp_grid = floor_wp_grid_df[['x','y']].to_numpy() #for ploting\n    #generate grid based on map\n    floor_map_grid = generate_grid_from_map(stite_check, floor_check, step)\n    #snap to grid\n    #fixed_floor_paths_df = snap_to_grid(floor_paths_df, floor_map_grid, 2)\n    print (\"\\033[A                                                          \\033[A\", end=\"\\r\")\n    print(\"Grid Generated | Site: {}, Floor: {}\".format(stite_check, floor_check))\n    return floor_paths_df, floor_wp_grid, floor_map_grid","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# POST PROCESSING","metadata":{}},{"cell_type":"code","source":"stop = stop","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# RUNTIME","metadata":{}},{"cell_type":"code","source":"outdir = './knn_fold_predictions'\nif not os.path.exists(outdir):\n    os.mkdir(outdir)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_eval = knn_site_train(site_to_check=None, n_split=10, ts_ass=True)\ndisplay(pred_eval)\nprint(pred_eval.wp_mpe.mean())\npred_eval.to_csv('knn_fold_predictions/knn_fold_eval.csv',index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"site_sample_path = './knn_fold_predictions/5d27096c03f801723c31e5e0.csv'\nsite_to_check = site_sample_path.split('/')[-1].split('.')[0]\nsite_df = pd.read_csv(site_sample_path)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# RSSI ANALYSIS","metadata":{}},{"cell_type":"code","source":"rm_path = '../input/indoor-navigation-and-location-wifi-features/wifi_features/train/5da138274db8ce0c98bbd3d2_1000_train.csv'\nsite_id = rm_path.split('/')[-1].split(\"_\")[0]\nfloor = 0\nfloor_id = get_site_floor_from_id(site_id, floor)\nrm_df = pd.read_csv(rm_path)\nprint(f\"Site: {site_id}, Floor: {floor_id}({floor})\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rm_df.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rm_df.loc[rm_df.f==floor]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_rm_df.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_rm_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ap = '28ec3d3d264c01931d61ce5ec6882334c75e0496'\npath_rm_df = rm_df[[ap, 'x','y','path']].loc[rm_df.f==floor]\npath_rm_df = path_rm_df.rename({ap: 'rssi'}, axis='columns')\npath_rm_df = path_rm_df[path_rm_df.rssi != -999]\nplot_floor(site_id, floor_id,path_rm_df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.express as px\nfig = px.scatter(path_rm_df, x=\"x\", y=\"y\", color=\"rssi\", color_continuous_scale = [\"blue\",\"aqua\",\"lime\",'yellow', \"red\"])\nfig.update_layout(coloraxis_colorbar=dict(title=\"RSSI\"))\nfig.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}