{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys\nimport os\nimport pickle\nimport multiprocessing\nimport numpy as np\nimport pandas as pd\nimport scipy.sparse\nimport scipy.sparse.linalg\nfrom tqdm import tqdm","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SUPPLEMENTALS_DIR = '/kaggle/input/indoor-supplementals-for-postprocessing'\n\nwith open(f'{SUPPLEMENTALS_DIR}/delta_using_device_id_v2.pkl', 'rb') as f:\n    DELTA = pickle.load(f)\n\nCONFIDENCE = pd.read_csv(f'{SUPPLEMENTALS_DIR}/wifi_confidence.csv').sort_values('site_path_timestamp').reset_index(drop=True)\nWAYPOINTS = pd.read_csv(f'{SUPPLEMENTALS_DIR}/waypoint.csv')\nEXTRA_GRID_POINTS = pd.read_csv('/kaggle/input/indoor-extra-grid-points/extra_grid_points_v2.csv')\nEXTRA_HALLWAY_POINTS = pd.read_csv('/kaggle/input/indoor-extra-hallway-points/extra_hallway_points.csv')\nXY = ['x', 'y']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from dataclasses import dataclass\n\n@dataclass\nclass Waypoint:\n    arr: np.ndarray\n    snapped: bool = False\n\ndef find_nearest_waypoints(xy_hat, waypoints):\n    N = xy_hat.shape[0]\n    xy_nn = np.zeros((N, 2))\n    for i in range(N):\n        r = np.sum((waypoints - xy_hat[i, :])**2, axis=1)\n        j = np.argmin(r)\n        xy_nn[i, :] = waypoints[j, :]\n    return xy_nn\n\ndef find_k_nearest_waypoints(xy_hat, waypoints, k):\n    N = xy_hat.shape[0]\n    xy_nn = np.zeros((N, k, 2))\n    for i in range(N):\n        r = np.sum((waypoints - xy_hat[i, :])**2, axis=1)\n        j = np.argpartition(r, k)[0:k]\n        xy_nn[i] = waypoints[j]\n    return xy_nn\n\n# def find_k_nearest_waypoints(xy_hat, waypoints, waypoints2, k):\n#     N = xy_hat.shape[0]\n#     ret = []\n#     for i in range(N):\n#         r = np.sum((waypoints - xy_hat[i, :])**2, axis=1)\n#         indices = np.argpartition(r, k)[0:k]\n#         found = []\n#         for idx in indices:\n#             found.append(waypoints2[idx])\n#         ret.append(found)\n#     return ret","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def correct_path(args, retry=False):\n    path, path_df = args\n    site  = path_df['site'].iloc[0]\n    floor = path_df['floor'].iloc[0]\n    \n    #========================================\n    # waypoint\n    #========================================\n    waypoints = WAYPOINTS[(WAYPOINTS['site']  == site) &\n                          (WAYPOINTS['floor'] == floor)]\n    waypoints = waypoints[['x', 'y']].values\n    waypoints2 = [Waypoint(arr=wp) for wp in waypoints]\n    \n    extra_grid_points = EXTRA_GRID_POINTS[(EXTRA_GRID_POINTS['site']  == site) &\n                                          (EXTRA_GRID_POINTS['floor'] == floor)]\n    extra_grid_points = extra_grid_points[['x', 'y']].values\n\n    waypoints_all = np.concatenate([waypoints, extra_grid_points], axis=0)\n    \n    #========================================\n    # 機械学習の予測位置\n    #========================================\n    ref_positions = path_df[['timestamp', 'x', 'y']].values\n    T_ref   = ref_positions[:, 0]\n    xy_hat  = ref_positions[:, 1:3]\n    delta_t = 1e-3 * np.diff(T_ref)\n\n    #========================================\n    # 加速度・方向センサによる相対移動距離\n    #========================================\n    delta_xy_hat, delta_xy_std_hat, _ = DELTA[path]\n\n    #========================================\n    # 機械学習とセンサデータを統合\n    #========================================\n    N = xy_hat.shape[0]\n    if retry == True:\n        xy_std_hat = 2.0 * path_df['std'].values\n    else:\n        xy_std_hat = 0.6 * path_df['std'].values\n    alpha = (xy_std_hat)**(-2)\n    # beta  = (0.8722777 + (0.0128752 * delta_t) + (1.0412663 * delta_xy_std_hat))**(-2)\n    beta  = (0.8676949 + (0.00874235 * delta_t) + (1.0868441 * delta_xy_std_hat))**(-2)\n    A = scipy.sparse.spdiags(alpha, [0], N, N)\n    B = scipy.sparse.spdiags( beta, [0], N-1, N-1)\n    D = scipy.sparse.spdiags(np.stack([-np.ones(N), np.ones(N)]), [0, 1], N-1, N)\n\n    Q = A + D.T @ B @ D\n    c = (A @ xy_hat) + (D.T @ (B @ delta_xy_hat))\n    xy_star = scipy.sparse.linalg.spsolve(Q, c)\n\n    #========================================\n    # 地図情報を統合\n    #========================================\n    gamma = (2.5**(-2) * np.ones(N))\n    C = scipy.sparse.spdiags(gamma, [0], N, N)\n\n    for r in np.arange(0.1, 0.8, 0.05):\n        xy_nn = find_nearest_waypoints(xy_star, waypoints_all)\n        Q = ((1 - r) * A) + (r * C) + D.T @ B @ D\n        c = ((1 - r) * (A @ xy_hat)) + (r * (C @ xy_nn)) + (D.T @ (B @ delta_xy_hat))\n        xy_star = scipy.sparse.linalg.spsolve(Q, c)\n    r = 0.8\n    for i in range(10):\n        xy_nn = find_nearest_waypoints(xy_star, waypoints_all)\n        Q = ((1 - r) * A) + (r * C) + D.T @ B @ D\n        c = ((1 - r) * (A @ xy_hat)) + (r * (C @ xy_nn)) + (D.T @ (B @ delta_xy_hat))\n        xy_star = scipy.sparse.linalg.spsolve(Q, c)\n    for i in range(10):\n        xy_nn = find_nearest_waypoints(xy_star, waypoints_all)\n        Q = C + D.T @ B @ D\n        c = (C @ xy_nn) + (D.T @ (B @ delta_xy_hat))\n        xy_star = scipy.sparse.linalg.spsolve(Q, c)\n\n    xy_nn = find_k_nearest_waypoints(xy_star, waypoints, 2)\n    calc_r = lambda x1, x2 : np.sqrt(np.sum((x1 - x2)**2, axis=-1))\n    for i in range(N):\n        r01 = calc_r(xy_star[i] , xy_nn[i][0])\n        r02 = calc_r(xy_star[i] , xy_nn[i][1])\n        r12 = calc_r(xy_nn[i][0], xy_nn[i][1])\n        cond0 = r01 < 1.5\n        cond1 = r12 < r02\n        cond2 = r01 < 5.0\n        if cond0 or (cond1 and cond2):\n            xy_star[i] =xy_nn[i][0]\n\n    out_df = pd.DataFrame({\n        'site_path_timestamp' : path_df['site_path_timestamp'],\n        'floor' : path_df['floor'],\n        'x' : xy_star[:, 0],\n        'y' : xy_star[:, 1],\n    })\n    return out_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def detect_error_by_hallway(path, out_df):\n    site  = out_df['site_path_timestamp'].iloc[0].split('_')[0]\n    floor = out_df['floor'].iloc[0]\n\n    waypoints = WAYPOINTS[(WAYPOINTS['site']  == site) &\n                          (WAYPOINTS['floor'] == floor)]\n    waypoints = waypoints[['x', 'y']].values\n    \n    extra_hallway_points = EXTRA_HALLWAY_POINTS[(EXTRA_HALLWAY_POINTS['site']  == site) &\n                                                (EXTRA_HALLWAY_POINTS['floor'] == floor)]\n    extra_hallway_points = extra_hallway_points[['x', 'y']].values\n\n    hallway_points_all = np.concatenate([waypoints, extra_hallway_points], axis=0)\n\n    def calc_r(xy_in):\n        xy_nn = find_nearest_waypoints(xy_in, hallway_points_all)\n        r = np.sqrt(np.sum((xy_in - xy_nn)**2, axis=1))\n        return r\n    \n    xy_hat   = out_df[['x', 'y']].values\n    xy_start = xy_hat[0:-1]\n    xy_end   = xy_hat[1:]\n    xy_mid   = 0.5 * (xy_start + xy_end)\n\n    r_start = calc_r(xy_start)\n    r_end   = calc_r(xy_end)\n    r_mid   = calc_r(xy_mid)\n    retval  = (((r_start > 3.0) | (r_start > 3.0)) & (r_mid > 5.0)).any()\n    return retval","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def correct_path_with_retry(args):\n    path, _ = args\n    out_df = correct_path(args, retry=False)\n    if detect_error_by_hallway(path, out_df):\n        print('\\ndetect error @', path)\n        out_df = correct_path(args, retry=True)\n    return out_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def do_blending(sub1, sub2, rate):\n    out = sub1.copy()\n    out[XY] = rate * sub1[XY] + (1 - rate) * sub2[XY]\n    return out\n\ndef read_sub(filename):\n    sub = pd.read_csv(filename).sort_values('site_path_timestamp').reset_index(drop=True)\n    tmp = sub['site_path_timestamp'].apply(lambda x : pd.Series(x.split('_')))\n    sub['site'] = tmp[0]\n    sub['path'] = tmp[1]\n    sub['timestamp'] = tmp[2].astype(float)\n    sub['std'] = CONFIDENCE['std']\n    return sub","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def main():\n    sub_0 = read_sub('/kaggle/input/indoor-ensemble/no_hand_labeling_pseudo_labeling_from_lb_2.693.csv')\n    sub_1 = read_sub('/kaggle/input/indoor-submissions/submission_raw_wifi_pseudo_labeling_nohand_from2586.csv')\n    floor_df  = read_sub('/kaggle/input/indoor-submissions/submission_raw_wifi_pseudo_labeling_nohand_from3112.csv')\n    sub_0['floor'] = floor_df['floor']\n    assert( (sub_0['floor'] == sub_1['floor']).all() )\n    \n    sub_01 = do_blending(sub_0, sub_1, 0.5)\n\n    processes = multiprocessing.cpu_count()\n    with multiprocessing.Pool(processes=processes) as pool:\n        dfs = pool.imap_unordered(correct_path_with_retry, sub_01.groupby('path'))\n        dfs = tqdm(dfs)\n        dfs = list(dfs)\n    sub = pd.concat(dfs)\n    sub = sub.sort_values('site_path_timestamp')\n    sub.to_csv('11.no_hand_labeling_pseudo_labeling_from_lb_2.586_with_retry_submission.csv', index=False)\n    return","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"main()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}