{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"I would like to share with you a simple post-processing that I used.\n\nIt's not powerful, but you can use it not only once, but after other post-processings to correct any misalignment caused by other post-processings.\n\nIt's very poorly coded, but I'd be very happy to get feedback.","metadata":{}},{"cell_type":"code","source":"import os\nfrom glob import glob\nfrom pathlib import Path\nimport warnings\n\n\nimport numpy as np\nimport pandas as pd\n\nwarnings.simplefilter('ignore')","metadata":{"execution":{"iopub.status.busy":"2021-08-05T21:14:20.645533Z","iopub.execute_input":"2021-08-05T21:14:20.646065Z","iopub.status.idle":"2021-08-05T21:14:20.650644Z","shell.execute_reply.started":"2021-08-05T21:14:20.646031Z","shell.execute_reply":"2021-08-05T21:14:20.649886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dir = Path(\"../input/google-smartphone-decimeter-challenge\")\n\ntrain_base = pd.read_csv(dir / \"baseline_locations_train.csv\")\ntest_base = pd.read_csv(dir / \"baseline_locations_test.csv\")\nsub = pd.read_csv(dir / \"sample_submission.csv\")\n\n\ndef get_groundtruth(path: Path) -> pd.DataFrame:\n        output_df = pd.DataFrame()\n        \n        for path in glob(str(dir / 'train/*/*/ground_truth.csv')):\n            _df = pd.read_csv(path)\n            output_df = pd.concat([output_df, _df])\n        output_df = output_df.reset_index(drop=True)\n        \n        _columns = ['latDeg', 'lngDeg', 'heightAboveWgs84EllipsoidM']\n        output_df[['t_'+col for col in _columns]] = output_df[_columns]\n        output_df = output_df.drop(columns=_columns, axis=1)\n        return output_df\n\ntrain_base = train_base.merge(\n    get_groundtruth(dir),\n    on=['collectionName', 'phoneName', 'millisSinceGpsEpoch']\n)","metadata":{"execution":{"iopub.status.busy":"2021-08-05T21:11:02.920985Z","iopub.execute_input":"2021-08-05T21:11:02.921384Z","iopub.status.idle":"2021-08-05T21:11:05.413564Z","shell.execute_reply.started":"2021-08-05T21:11:02.921349Z","shell.execute_reply":"2021-08-05T21:11:05.412704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calc_haversine(lat1, lon1, lat2, lon2):\n    RADIUS = 6_367_000\n    lat1, lon1, lat2, lon2 = map(np.radians, [lat1, lon1, lat2, lon2])\n    dlat = lat2 - lat1\n    dlon = lon2 - lon1\n    d = np.sin(dlat/2)**2 + np.cos(lat1) * np.cos(lat2) * np.sin(dlon/2)**2\n    dist = 2 * RADIUS * np.arcsin(d**0.5)\n    return dist\n\ndef check_score(input_df: pd.DataFrame) -> pd.DataFrame:\n    output_df = input_df.copy()\n    \n    output_df['meter'] = input_df.apply(\n        lambda r: calc_haversine(\n            r.latDeg, r.lngDeg, r.t_latDeg, r.t_lngDeg\n        ),\n        axis=1\n    )\n\n    meter_score = output_df['meter'].mean()\n    print(f'error meter: {meter_score}')\n\n    scores = []\n    for phone in output_df['phone'].unique():\n        _index = output_df['phone']==phone\n        p_50 = np.percentile(output_df.loc[_index, 'meter'], 50)\n        p_95 = np.percentile(output_df.loc[_index, 'meter'], 95)\n        scores.append(p_50)\n        scores.append(p_95)\n\n    score = sum(scores) / len(scores)\n    print(f'score: {score}')\n    \n    return output_df","metadata":{"execution":{"iopub.status.busy":"2021-08-05T21:11:27.065807Z","iopub.execute_input":"2021-08-05T21:11:27.066287Z","iopub.status.idle":"2021-08-05T21:11:27.076194Z","shell.execute_reply.started":"2021-08-05T21:11:27.066253Z","shell.execute_reply":"2021-08-05T21:11:27.075374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Check Degrees\n\nThe process is very simple.\nIf the degree of ∠ABC is less than n°, correct the coordinates of B.\nIn this notebook, n = 155.","metadata":{}},{"cell_type":"code","source":"def add_check_degrees_features(input_df):\n    output_df = input_df.copy()\n    \n    output_df[\"latDeg_pre\"] = output_df[\"latDeg\"].shift(1)\n    output_df[\"latDeg_pro\"] = output_df[\"latDeg\"].shift(-1)\n    output_df[\"lngDeg_pre\"] = output_df[\"lngDeg\"].shift(1)\n    output_df[\"lngDeg_pro\"] = output_df[\"lngDeg\"].shift(-1)\n    output_df[\"millisSinceGpsEpoch_pre\"] = output_df[\"millisSinceGpsEpoch\"].shift(1)\n    output_df[\"millisSinceGpsEpoch_pro\"] = output_df[\"millisSinceGpsEpoch\"].shift(-1)\n    output_df[\"latDeg_mean_point\"] \\\n        = (output_df[\"latDeg_pre\"] + ((output_df[\"latDeg_pro\"] - output_df[\"latDeg_pre\"]) * \n           ((output_df[\"millisSinceGpsEpoch\"] - output_df[\"millisSinceGpsEpoch_pre\"]) /\n           (output_df[\"millisSinceGpsEpoch_pro\"] - output_df[\"millisSinceGpsEpoch_pre\"]))))\n        \n    output_df[\"lngDeg_mean_point\"] \\\n        = (output_df[\"lngDeg_pre\"] + ((output_df[\"lngDeg_pro\"] - output_df[\"lngDeg_pre\"]) * \n           ((output_df[\"millisSinceGpsEpoch\"] - output_df[\"millisSinceGpsEpoch_pre\"]) /\n           (output_df[\"millisSinceGpsEpoch_pro\"] - output_df[\"millisSinceGpsEpoch_pre\"]))))\n                 \n\n    degree_list = []\n    for lat_pre, lng_pre, lat, lng, lat_pro, lng_pro in zip(\n        output_df[\"latDeg_pre\"].to_numpy(),\n        output_df[\"lngDeg_pre\"].to_numpy(),\n        output_df[\"latDeg\"].to_numpy(),\n        output_df[\"lngDeg\"].to_numpy(),\n        output_df[\"latDeg_pro\"].to_numpy(),\n        output_df[\"lngDeg_pro\"].to_numpy()\n    ):\n        p0 = np.array([lat_pre, lng_pre])\n        p1 = np.array([lat, lng])\n        p2 = np.array([lat_pro, lng_pro])\n            \n        vec_p01 = p0 - p1\n        vec_p12 = p2 - p1\n        length_vec_p01 = np.linalg.norm(vec_p01)\n        length_vec_p12 = np.linalg.norm(vec_p12)\n        inner = np.inner(vec_p01, vec_p12)\n        degree = np.rad2deg(np.arccos(inner / (length_vec_p01 * length_vec_p12)))\n        degree_list.append(degree)\n    \n    output_df[\"degree\"] = degree_list\n    return output_df\n\n\ndef check_degrees(input_df):\n    output_df = input_df.copy()\n\n    lat_list = []\n    lng_list = []\n\n    for collection in output_df[\"collectionName\"].unique():\n        collection_df = output_df[output_df[\"collectionName\"] == collection]\n        for phone in collection_df[\"phoneName\"].unique():\n            phone_df = collection_df[collection_df[\"phoneName\"] == phone]\n            degree_df = add_check_degrees_features(phone_df)\n            for lat, lng, lat_mp, lng_mp, degee in zip(\n                degree_df[\"latDeg\"].to_numpy(),\n                degree_df[\"lngDeg\"].to_numpy(),\n                degree_df[\"latDeg_mean_point\"].to_numpy(),\n                degree_df[\"lngDeg_mean_point\"].to_numpy(),\n                degree_df[\"degree\"].to_numpy()\n            ):\n                if degee < 155:\n                    lat_ = (lat + lat_mp)/2\n                    lng_ = (lng + lng_mp)/2\n                    lat_list.append(lat_)\n                    lng_list.append(lng_)\n                else:\n                    lat_list.append(lat)\n                    lng_list.append(lng)\n        \n    output_df[\"latDeg\"] = lat_list\n    output_df[\"lngDeg\"] = lng_list\n    return output_df[input_df.columns]","metadata":{"execution":{"iopub.status.busy":"2021-08-05T21:36:09.593368Z","iopub.execute_input":"2021-08-05T21:36:09.593848Z","iopub.status.idle":"2021-08-05T21:36:09.611524Z","shell.execute_reply.started":"2021-08-05T21:36:09.593819Z","shell.execute_reply":"2021-08-05T21:36:09.610719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This post-processing can be applied multiple times to increase the accuracy.","metadata":{}},{"cell_type":"code","source":"print(\"<train_base score>\")\ntrain_base = check_score(train_base)\nprint(\"******************************\")\nprint(\"******************************\")\n\n\nprint(\"<chec_degrees score>\")\nprint(\"- once -\")\nonce  = check_degrees(train_base)\nonce = check_score(once)\nprint(\"------------------------------\")\n\nprint(\"- 10 times -\")\n_10_times  = check_degrees(train_base)\nfor i in range(9):\n    _10_times = check_degrees(_10_times)\n_10_times = check_score(_10_times)\nprint(\"------------------------------\")\n\nprint(\"- 50 times- \")\n_50_times  = check_degrees(train_base)\nfor i in range(49):\n    _50_times = check_degrees(_50_times)\n_50_times = check_score(_50_times)\nprint(\"------------------------------\")\n\nprint(\"- 100 times -\")\n_100_times  = check_degrees(train_base)\nfor i in range(99):\n    _100_times = check_degrees(_100_times)\n_100_times = check_score(_100_times)\nprint(\"------------------------------\")\n\nprint(\"- 300 times -\")\n_300_times  = check_degrees(train_base)\nfor i in range(299):\n    _300_times = check_degrees(_300_times)\n_300_times = check_score(_300_times)\nprint(\"------------------------------\")","metadata":{"execution":{"iopub.status.busy":"2021-08-05T21:54:02.102726Z","iopub.execute_input":"2021-08-05T21:54:02.103086Z","iopub.status.idle":"2021-08-05T22:43:39.184663Z","shell.execute_reply.started":"2021-08-05T21:54:02.103036Z","shell.execute_reply":"2021-08-05T22:43:39.18345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The accuracy is improving as the number of times increases.\n\nIn the case of once, the process takes a short time, but in the case of tens to hundreds of times as described above, it takes long time.","metadata":{}},{"cell_type":"markdown","source":"I used this post-processing in the following way.\n\nbaseline  ->  check_degrees  ->  other post-processing  ->  check_degrees  ->  other post-processing  ->  check_degrees  ->  ... ","metadata":{}},{"cell_type":"markdown","source":"In many cases, it is possible to correct misalignments caused by other post-processing.\n\nHowever, the accuracy may deteriorate for post-processing using train_grand_truth.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}