{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom myutil import load_pickle\nfrom scipy.signal import savgol_filter\nimport json\nfrom PIL import Image \nimport plotly.graph_objs as go\nfrom pathlib import Path\n\nimport multiprocessing\nimport scipy.interpolate\nimport scipy.sparse\nfrom tqdm import tqdm\n\nfrom indoor_location_github_script import read_data_file\nimport indoor_location_github_script as compute_f","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INPUT_PATH = '../input/indoor-location-navigation'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# functions","metadata":{}},{"cell_type":"code","source":"def floor_reverse(floor):\n    floor = int(floor)\n    if floor < 0:\n        return f'B{abs(floor)}'\n    else:\n         return f'F{floor + 1}'\n        \n\n\ndef visualize_trajectory(trajectory, floor_plan_filename, width_meter, height_meter, title=None, mode='lines + markers + text', show=False):\n    fig = go.Figure()\n\n    # add trajectory\n    size_list = [6] * trajectory.shape[0]\n    size_list[0] = 10\n    size_list[-1] = 10\n\n    color_list = ['rgba(4, 174, 4, 0.5)'] * trajectory.shape[0]\n    color_list[0] = 'rgba(12, 5, 235, 1)'\n    color_list[-1] = 'rgba(235, 5, 5, 1)'\n\n    position_count = {}\n    text_list = []\n    for i in range(trajectory.shape[0]):\n        if str(trajectory[i]) in position_count:\n            position_count[str(trajectory[i])] += 1\n        else:\n            position_count[str(trajectory[i])] = 0\n        text_list.append('        ' * position_count[str(trajectory[i])] + f'{i}')\n    text_list[0] = 'Start Point: 0'\n    text_list[-1] = f'End Point: {trajectory.shape[0] - 1}'\n\n    fig.add_trace(\n        go.Scattergl(\n            x=trajectory[:, 0],\n            y=trajectory[:, 1],\n            mode=mode,\n            marker=dict(size=size_list, color=color_list),\n            line=dict(shape='linear', color='rgb(100, 10, 100)', width=2, dash='dot'),\n#             text=text_list,\n            textposition=\"top center\",\n            name='trajectory',\n        ))\n\n    # add floor plan\n    floor_plan = Image.open(floor_plan_filename)\n    fig.update_layout(images=[\n        go.layout.Image(\n            source=floor_plan,\n            xref=\"x\",\n            yref=\"y\",\n            x=0,\n            y=height_meter,\n            sizex=width_meter,\n            sizey=height_meter,\n            sizing=\"contain\",\n            opacity=1,\n            layer=\"below\",\n        )\n    ])\n\n    # configure\n    fig.update_xaxes(autorange=False, range=[0, width_meter])\n    fig.update_yaxes(autorange=False, range=[0, height_meter], scaleanchor=\"x\", scaleratio=1)\n    fig.update_layout(\n        title=go.layout.Title(\n            text=title or \"No title.\",\n            xref=\"paper\",\n            x=0,\n        ),\n        autosize=True,\n        width=900,\n        height=200 + 900 * height_meter / width_meter,\n        template=\"plotly_white\",\n    )\n\n    if show:\n        fig.show()\n\n    return fig\n\n\n\ndef compute_rel_positions(acce_datas, ahrs_datas):\n    step_timestamps, step_indexs, step_acce_max_mins = compute_f.compute_steps(acce_datas)\n    headings = compute_f.compute_headings(ahrs_datas)\n    stride_lengths = compute_f.compute_stride_length(step_acce_max_mins)\n    step_headings = compute_f.compute_step_heading(step_timestamps, headings)\n    rel_positions = compute_f.compute_rel_positions(stride_lengths, step_headings)\n    return rel_positions\n\n\n\ndef correct_path(args):\n    win = 31\n    pol = 1\n    path, path_df = args\n    \n    T_ref  = path_df['timestamp'].values\n    \n    trajectory = path_df.iloc[:, 1:3].values\n    x = savgol_filter(trajectory[:, 0], win, pol)\n    y = savgol_filter(trajectory[:, 1], win, pol)\n    xy_hat = np.hstack((x.reshape(-1, 1), y.reshape(-1, 1)))\n#     xy_hat = path_df[['x', 'y']].values\n    \n    example = read_data_file(f'{INPUT_PATH}/test/{path}.txt')\n    rel_positions = compute_rel_positions(example.acce, example.ahrs)\n    if T_ref[-1] > rel_positions[-1, 0]:\n        rel_positions = [np.array([[0, 0, 0]]), rel_positions, np.array([[T_ref[-1], 0, 0]])]\n    else:\n        rel_positions = [np.array([[0, 0, 0]]), rel_positions]\n    rel_positions = np.concatenate(rel_positions)\n    \n    T_rel = rel_positions[:, 0]\n    delta_xy_hat = np.diff(scipy.interpolate.interp1d(T_rel, np.cumsum(rel_positions[:, 1:3], axis=0), axis=0)(T_ref), axis=0)\n\n    N = xy_hat.shape[0]\n    delta_t = np.diff(T_ref)\n    alpha = (8.1)**(-2) * np.ones(N)\n    beta  = (0.2 + 0.2 * 1e-3 * delta_t)**(-2)\n    A = scipy.sparse.spdiags(alpha, [0], N, N)\n    B = scipy.sparse.spdiags( beta, [0], N-1, N-1)\n    D = scipy.sparse.spdiags(np.stack([-np.ones(N), np.ones(N)]), [0, 1], N-1, N)\n\n    Q = A + (D.T @ B @ D)\n    c = (A @ xy_hat) + (D.T @ (B @ delta_xy_hat))\n    xy_star = scipy.sparse.linalg.spsolve(Q, c)\n\n    return pd.DataFrame({\n        'floor' : path_df['floor'],\n        'x' : xy_star[:, 0],\n        'y' : xy_star[:, 1],\n        'path' : path_df['path'],\n        'timestamp' : path_df['timestamp']\n    })","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# data","metadata":{}},{"cell_type":"code","source":"ssub = pd.read_csv('../input/indoor-location-navigation/sample_submission.csv')\n# preds = pd.read_csv('../input/indoor-location-rnn-v2/predictions1617688222_embed128_cycle16_batch885.csv')\nmeta_data = load_pickle('../input/indoor-location-rnn-test-data-v2/test-meta-data.pickle')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"files = list(Path('../input/indoor-location-rnn-v2').glob('*.csv'))\npreds = 0\nfor file in files:\n    pred = pd.read_csv(file)\n    preds += pred\npreds /= len(files)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds['path'] = ''\npreds['timestamp'] = 0\n\nfor trace in meta_data['pos_range']:\n    start_idx, end_idx = meta_data['pos_range'][trace]\n    preds.iloc[start_idx: end_idx, -2] = trace\n    \ntss = [int(meta_data['timestamps'][idx]) for idx in meta_data['timestamps']]\npreds['timestamp'] = tss\npreds.columns = ['floor', 'x', 'y', 'path', 'timestamp']\npreds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"site_path_timestamp = ssub['site_path_timestamp'].apply(lambda x: pd.Series(x.split('_')))\nssub['site'] = site_path_timestamp[0]\nssub['path'] = site_path_timestamp[1]\nssub['timestamp'] = site_path_timestamp[2]\nssub['timestamp'] = ssub['timestamp'].astype(int)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"original_floors = preds.iloc[:, 0].copy()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# use mode for floors","metadata":{}},{"cell_type":"code","source":"print('fix')\nfor path in preds['path'].unique():\n    rel_preds = preds[preds['path'] == path]\n    preds.iloc[rel_preds.index, 0] = np.round(preds.iloc[rel_preds.index, 0].mean())\n    \nfor path in preds['path'].unique():\n    rel_preds = preds[preds['path'] == path]\n    if rel_preds.iloc[:, 0].unique().shape[0] != 1:\n        print('oof')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# get confidene of predictions","metadata":{}},{"cell_type":"code","source":"confidence = (preds.iloc[:, 0] - original_floors).abs() # lower is better","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# visualize predictions","metadata":{}},{"cell_type":"code","source":"count = 0\nfor path in preds['path'].unique():\n    cut = 28\n    if path == 'b406c5c925f3b64d8972b2c0':\n        rel_preds = preds[preds['path'] == path]\n        path_file = f'../input/indoor-location-navigation/test/{path}.txt'\n        with open(path_file, 'r') as f:\n            for line in f:\n                if 'SiteID' in line:\n                    building = line[9:33]\n                    break\n        floor = floor_reverse(rel_preds.iloc[0, 0])\n        try:\n            with open(f'../input/indoor-location-navigation/metadata/{building}/{floor}/floor_info.json', 'rb') as f:\n                floor_info = json.load(f)\n        except:\n            floor = floor[::-1]\n            with open(f'../input/indoor-location-navigation/metadata/{building}/{floor}/floor_info.json', 'rb') as f:\n                floor_info = json.load(f)\n        height = floor_info['map_info']['height']\n        width = floor_info['map_info']['width']\n        floor_img = f'../input/indoor-location-navigation/metadata/{building}/{floor}/floor_image.png'\n        trajectory = rel_preds.iloc[:, 1:3].values\n        visualize_trajectory(trajectory, floor_img, width, height, show=True)\n#         win = 5\n#         pol = 1\n#         x = savgol_filter(trajectory[:, 0], win, pol)\n#         y = savgol_filter(trajectory[:, 1], win, pol)\n#         filtered = np.hstack((x.reshape(-1, 1), y.reshape(-1, 1)))\n#         visualize_trajectory(filtered, floor_img, width, height, show=True)\n        \n        processes = multiprocessing.cpu_count()\n        with multiprocessing.Pool(processes=processes) as pool:\n            dfs = pool.imap_unordered(correct_path, rel_preds.groupby('path'))\n            dfs = tqdm(dfs)\n            dfs = list(dfs)\n        rel_preds = pd.concat(dfs).sort_index()\n        trajectory = rel_preds.iloc[:, 1:3].values\n        visualize_trajectory(trajectory, floor_img, width, height, show=True)\n        \n        rel_ssub = ssub[ssub['path'] == path]\n        trajectory = rel_preds.iloc[:, 1:3].values\n        ssub_points = []\n        for row in rel_ssub.itertuples():\n            try:\n                pred = rel_preds[rel_preds['timestamp'] == row.timestamp].iloc[0, 1:3]\n            except:\n                print(f'missing ts at ssub index {row.Index}')\n                pred = rel_preds.iloc[(rel_preds['timestamp'] - row.timestamp).abs().argmin(), 1:3]\n                print(pred)\n            ssub_points.append(pred)\n        visualize_trajectory(np.array(ssub_points), floor_img, width, height, show=True)\n    count += 1","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# smooth predictions","metadata":{}},{"cell_type":"code","source":"processes = multiprocessing.cpu_count()\nwith multiprocessing.Pool(processes=processes) as pool:\n    dfs = pool.imap_unordered(correct_path, preds.groupby('path'))\n    dfs = tqdm(dfs)\n    dfs = list(dfs)\nprocessed_preds = pd.concat(dfs).sort_index()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# win = 35\n# pol = 1\nfor path in ssub['path'].unique():\n    rel_ssub = ssub[ssub['path'] == path]\n    rel_preds = processed_preds[processed_preds['path'] == path]\n    trajectory = rel_preds.iloc[:, :3].values\n#     x = savgol_filter(trajectory[:, 0], win, pol)\n#     y = savgol_filter(trajectory[:, 1], win, pol)\n#     filtered = np.hstack((rel_preds.iloc[:, 0:1], x.reshape(-1, 1), y.reshape(-1, 1)))\n    for row in rel_ssub.itertuples():\n        pred = trajectory[rel_preds['timestamp'] == row.timestamp]\n        if len(pred) == 0:\n            print(f'missing ts at ssub index {row.Index}')\n            pred = trajectory[(rel_preds['timestamp'] - row.timestamp).abs().argmin()]\n            ssub.loc[row.Index, 'floor':'y'] = pred\n        else:\n            ssub.loc[row.Index, 'floor':'y'] = pred[0]\n        \nssub.iloc[:, :4].to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ssub[ssub['y'] == ssub['x']]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# closest location","metadata":{}},{"cell_type":"code","source":"%%time\nfor path in ssub['path'].unique():\n    rel_ssub = ssub[ssub['path'] == path]\n    rel_preds = preds[preds['path'] == path]\n    for row in rel_ssub.itertuples():\n        try:\n            pred = rel_preds[rel_preds['timestamp'] == row.timestamp].iloc[0, :3]\n        except IndexError:\n            print(f'missing ts at ssub index {row.Index}')\n            pred = rel_preds.iloc[(rel_preds['timestamp'] - row.timestamp).abs().argmin()][:3]\n        ssub.loc[row.Index, 'floor':'y'] = pred.tolist()\n        \nssub.iloc[:, :4].to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ------------------------------------------------","metadata":{}}]}