{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Thank you everyone!\n\nThis is a notbook of 25th solution. My solution is very simple.\n1. I use the results of [public best submission](https://www.kaggle.com/ahmedewida/indoorlocation-ensembling).\n2. Correct floor prediction based on the leakages of [shared wifi records](https://www.kaggle.com/tomooinubushi/retrieving-user-id-from-leaked-wifi-feature) and [device IDs](https://www.kaggle.com/c/indoor-location-navigation/discussion/234543)\n3. Postprocess the waypoints based on [Dijkstra's algorithm](https://en.wikipedia.org/wiki/Dijkstra%27s_algorithm) to search the path with minimal [cost](https://www.kaggle.com/saitodevel01/indoor-post-processing-by-cost-minimization)\n\nI use public sub, because my NN model is never better than 7.7 in CV.\n\nI use some codes and ideas from following notebooks. Thank you very much.\n\n* https://www.kaggle.com/kenmatsu4/feature-store-for-indoor-location-navigation\n* https://www.kaggle.com/jiweiliu/fix-the-timestamps-of-test-data-using-dask\n* https://www.kaggle.com/ahmedewida/indoorlocation-ensembling\n* https://www.kaggle.com/c/indoor-location-navigation/discussion/234543\n* https://www.kaggle.com/saitodevel01/indoor-post-processing-by-cost-minimization\n* https://www.kaggle.com/museas/with-magn-cost-minimization\n* https://www.kaggle.com/higepon/visualize-submissions-with-post-processing","metadata":{}},{"cell_type":"code","source":"import optuna\nfrom scipy.sparse.csgraph import shortest_path\nimport matplotlib.pyplot\nimport numpy\nimport cv2\nimport argparse\nfrom scipy.spatial import distance\nfrom collections import OrderedDict\nimport warnings\nimport torch\nfrom sklearn.utils.validation import _deprecate_positional_args\nimport os\nimport random\nimport time\nimport math\nfrom collections import defaultdict\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.utils.validation import check_is_fitted, column_or_1d\nimport seaborn as sns\nfrom IPython.display import display  # Allows the use of display() for DataFrames\nfrom pylab import imread\nfrom IPython.display import Image\nfrom matplotlib import animation as ani\nfrom matplotlib_venn import venn2, venn2_circles\nfrom matplotlib import rc\nimport matplotlib.cm as cm\nimport matplotlib.pyplot as plt\nfrom matplotlib import font_manager\nimport matplotlib\nimport json\nimport re\nimport gc\nimport pickle\nimport itertools\nimport pandas as pd\nimport numpy as np\nfrom glob import glob\nfrom datetime import datetime as dt\nfrom pathlib import Path\nfrom tqdm import tqdm\nimport datetime\nimport dill\nfrom pandas.api.types import is_categorical_dtype\nfrom pandas.api.types import is_datetime64_any_dtype as is_datetime\nfrom typing import Dict, List, Tuple, NamedTuple\n\nimport torch\nimport torch.optim as optim\nfrom torch import nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import datasets, models, transforms\nimport torch.nn.functional as F\nfrom torch.nn.modules.loss import _WeightedLoss\nfrom torch.nn import CrossEntropyLoss, MSELoss\nfrom torch.autograd import Variable\nfrom scipy.ndimage import maximum_filter\nfrom sklearn.model_selection import KFold, StratifiedKFold, GroupKFold\nfrom shapely.geometry import Point\nfrom shapely.geometry.polygon import Polygon\nfrom shapely.ops import nearest_points\nimport shapely.ops as so\nfrom descartes import PolygonPatch\nimport scipy.signal as signal\nfrom scipy.signal import butter, lfilter\nfrom scipy.signal import find_peaks\nts_conv = np.vectorize(datetime.datetime.fromtimestamp)  # ut(10 digit) -> date\n\n# pandas settings -----------------------------------------\npd.set_option(\"display.max_colwidth\", 100)\npd.set_option(\"display.max_rows\", None)\npd.set_option(\"display.max_columns\", None)\npd.options.display.float_format = '{:,.5f}'.format\n\n# Graph drawing -------------------------------------------\n\nplt.rcParams[\"patch.force_edgecolor\"] = True\nsns.set(style=\"whitegrid\", palette=\"muted\", color_codes=True)\nsns.set_style(\"whitegrid\", {'grid.linestyle': '--'})\nred = sns.xkcd_rgb[\"light red\"]\ngreen = sns.xkcd_rgb[\"medium green\"]\nblue = sns.xkcd_rgb[\"denim blue\"]\n\n\n# ML -------------------------------------------\n\nwarnings.filterwarnings('ignore')\n\n# import cudf\n# import cupy as cp\n\n# cfg = {\n#     'experiment_name': 'jsmp',\n#     'kaggle': False,\n#     'seed': 42,\n#     'now_str' : datetime.datetime.now().strftime('%Y%m%d_%H%M%S')\n# }\nfrom scipy.spatial import Delaunay, delaunay_plot_2d, Voronoi, voronoi_plot_2d","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pickle_dump(obj, path):\n    with open(path, mode='wb') as f:\n        pickle.dump(obj, f)\n\n\ndef pickle_load(path):\n    with open(path, mode='rb') as f:\n        data = pickle.load(f)\n        return data\n\n\ndef pickle_dump_dill(obj, path):\n    with open(path, mode='wb') as f:\n        dill.dump(obj, f)\n\n\ndef pickle_load_dill(path):\n    with open(path, mode='rb') as f:\n        data = dill.load(f)\n        return data\n\n# Original code from https://www.kaggle.com/gemartin/load-data-reduce-memory-usage by @gemartin\n# Modified to support timestamp type, categorical type\n# Modified to add option to use float16 or not. feather format does not support float16.\n\n\ndef reduce_mem_usage(df, use_float16=False):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n\n    for col in df.columns:\n        if is_datetime(df[col]) or is_categorical_dtype(df[col]):\n            # skip datetime type or categorical type\n            continue\n        col_type = df[col].dtype\n\n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                if use_float16 and c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(\n        100 * (start_mem - end_mem) / start_mem))\n\n    return df\n\n\ndef seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\n\nseed_everything()\n\ndata_dir = '../input/indoor-location-navigation/'\nmydata_dir = '../input/indoordataset/'\nout_dir = '../input/indoor-public/features/'","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FeatureStore():\n\n    # necessayr to re-check\n    floor_convert = {'1F':  0, '2F': 1, '3F': 2, '4F': 3, '5F': 4,\n                     '6F': 5, '7F': 6, '8F': 7, '9F': 8,\n                     'B': -1, 'B1': -1, 'B2': -2, 'B3': -3,\n                     'BF': -1, 'BM': -1,\n                     'F1': 0, 'F2': 1, 'F3': 2, 'F4': 3, 'F5': 4,\n                     'F6': 5, 'F7': 6, 'F8': 7, 'F9': 8, 'F10': 9,\n                     'L1': 0, 'L2': 1, 'L3': 2, 'L4': 3, 'L5': 4,\n                     'L6': 5, 'L7': 6, 'L8': 7, 'L9': 8, 'L10': 9,\n                     'L11': 10,\n                     'G': 0, 'LG1': 0, 'LG2': 1, 'LM': 0, 'M': 0,\n                     'P1': 0, 'P2': 1, }\n\n    df_types = ['accelerometer',\n                'accelerometer_uncalibrated',\n                'beacon',\n                'gyroscope',\n                'gyroscope_uncalibrated',\n                'magnetic_field',\n                'magnetic_field_uncalibrated',\n                'rotation_vector',\n                'waypoint',\n                'wifi']\n\n    # https://github.com/location-competition/indoor-location-competition-20\n    df_type_cols = {'accelerometer': [\"timestamp\", \"x\", \"y\", \"z\", \"accuracy\"],\n                    'accelerometer_uncalibrated': [\"timestamp\", \"x\", \"y\", \"z\",\n                                                   \"x2\", \"y2\", \"z2\", \"accuracy\"],\n                    'beacon': [\"timestamp\", \"uuid\", \"major_id\", \"minor_id\", \"tx_power\",\n                               \"rssi\", \"distance\", \"mac_addr\", \"timestamp2\"],\n                    'gyroscope': [\"timestamp\", \"x\", \"y\", \"z\", \"accuracy\"],\n                    'gyroscope_uncalibrated': [\"timestamp\", \"x\", \"y\", \"z\",\n                                               \"x2\", \"y2\", \"z2\", \"accuracy\"],\n                    'magnetic_field': [\"timestamp\", \"x\", \"y\", \"z\", \"accuracy\"],\n                    'magnetic_field_uncalibrated': [\"timestamp\", \"x\", \"y\", \"z\",\n                                                    \"x2\", \"y2\", \"z2\", \"accuracy\"],\n                    'rotation_vector': [\"timestamp\", \"x\", \"y\", \"z\", \"accuracy\"],\n                    'waypoint': [\"timestamp\", \"x\", \"y\"],\n                    'wifi': [\"timestamp\", \"ssid\", \"bssid\", \"rssi\", \"frequency\",\n                             \"last_seen_timestamp\", ]}\n\n    dtype_dict = {}\n    dtype_dict[\"accelerometer\"] = {\"timestamp\": int, \"x\": float, \"y\": float, \"z\": float,\n                                   \"accuracy\": int}\n    dtype_dict[\"accelerometer_uncalibrated\"] = {\"timestamp\": int, \"x\": float, \"y\": float,\n                                                \"z\": float, \"x2\": float, \"y2\": float,\n                                                \"z2\": float, \"accuracy\": int}\n    dtype_dict[\"beacon\"] = {\"timestamp\": int, \"uuid\": str, \"major_id\": str,\n                            \"minor_id\": str, \"tx_power\": int,  \"rssi\": int,\n                            \"distance\": float, \"mac_addr\": str, \"timestamp2\": int}\n    dtype_dict[\"gyroscope\"] = {\"timestamp\": int, \"x\": float, \"y\": float, \"z\": float,\n                               \"accuracy\": int}\n    dtype_dict[\"gyroscope_uncalibrated\"] = {\"timestamp\": int, \"x\": float, \"y\": float,\n                                            \"z\": float, \"x2\": float, \"y2\": float,\n                                            \"z2\": float, \"accuracy\": int}\n    dtype_dict[\"magnetic_field\"] = {\"timestamp\": int, \"x\": float, \"y\": float,\n                                    \"z\": float, \"accuracy\": int}\n    dtype_dict[\"magnetic_field_uncalibrated\"] = {\"timestamp\": int, \"x\": float,\n                                                 \"y\": float, \"z\": float, \"x2\": float,\n                                                 \"y2\": float, \"z2\": float, \"accuracy\": int}\n    dtype_dict[\"rotation_vector\"] = {\"timestamp\": int, \"x\": float, \"y\": float,\n                                     \"z\": float, \"accuracy\": int}\n    dtype_dict[\"waypoint\"] = {\"timestamp\": int,\n                              \"x\": float, \"y\": float, \"z\": float}\n    dtype_dict[\"wifi\"] = {\"timestamp\": int, \"ssid\": str, \"bssid\": str,\n                          \"rssi\": int, \"frequency\": int, \"last_seen_timestamp\": int}\n\n    def __init__(self, site_id, floor, path_id,\n                 input_path=\"../input/indoor-location-navigation/\",\n                 save_path=\"../mid\"):\n        self.site_id = site_id.strip()\n        self.floor = floor.strip()\n        self.n_floor = self.floor_convert[self.floor]\n        self.path_id = path_id.strip()\n\n        self.input_path = input_path\n        assert Path(input_path).exists(\n        ), f\"input_path do not exist: {input_path}\"\n\n        self.save_path = save_path\n        Path(save_path).mkdir(parents=True, exist_ok=True)\n\n        self.site_info = SiteInfo(\n            site_id=self.site_id, floor=self.floor, input_path=self.input_path)\n\n    def _flatten(self, l):\n        return list(itertools.chain.from_iterable(l))\n\n    def multi_line_spliter(self, s):\n        matches = re.finditer(\"TYPE_\", s)\n        matches_positions = [match.start() for match in matches]\n        split_idx = [0] + [matches_positions[i] -\n                           14 for i in range(1, len(matches_positions))] + [len(s)]\n        return [s[split_idx[i]:split_idx[i+1]] for i in range(len(split_idx)-1)]\n\n    def load_df(self, ):\n        path = str(Path(self.input_path) /\n                   f\"train/{self.site_id}/{self.floor}/{self.path_id}.txt\")\n        with open(path) as f:\n            data = f.readlines()\n\n        modified_data = []\n        for s in data:\n            if s.count(\"TYPE_\") > 1:\n                lines = self.multi_line_spliter(s)\n                modified_data.extend(lines)\n            else:\n                modified_data.append(s)\n        del data\n        self.meta_info_len = len([d for d in modified_data if d[0] == \"#\"])\n        self.meta_info_df = pd.DataFrame([m.replace(\"\\n\", \"\").split(\":\")\n                                          for m in self._flatten([d.split(\"\\t\")\n                                                                  for d in modified_data if d[0] == \"#\"]) if m != \"#\"])\n\n        data_df = pd.DataFrame([d.replace(\"\\n\", \"\").split(\"\\t\")\n                                for d in modified_data if d[0] != \"#\"])\n        for dt in self.df_types:\n            # select data type\n            df_s = data_df[data_df[1] == f\"TYPE_{dt.upper()}\"]\n            if len(df_s) == 0:\n                setattr(self, dt, pd.DataFrame(columns=self.df_type_cols[dt]))\n            else:\n                # remove empty cols\n                na_info = df_s.isna().sum(axis=0) == len(df_s)\n                df_s = df_s[[i for i in na_info[na_info ==\n                                                False].index if i != 1]].reset_index(drop=True)\n\n                if len(df_s.columns) != len(self.df_type_cols[dt]):\n                    df_s.columns = self.df_type_cols[dt][:len(df_s.columns)]\n                else:\n                    df_s.columns = self.df_type_cols[dt]\n\n                # set dtype\n                for c in df_s.columns:\n                    df_s[c] = df_s[c].astype(self.dtype_dict[dt][c])\n\n                # set DataFrame to attr\n                setattr(self, dt, df_s)\n\n    def get_site_info(self, keep_raw=False):\n        self.site_info.get_site_info(keep_raw=keep_raw)\n\n    def load_all_data(self, keep_raw=False):\n        self.load_df()\n        self.get_site_info(keep_raw=keep_raw)\n\n    def __getitem__(self, item):\n        if item in self.df_types:\n            return getattr(self, item)\n        else:\n            return None\n\n    def save(self, ):\n        # to be implemented\n        pass\n\n\nclass SiteInfo():\n    def __init__(self, site_id, floor, input_path=\"../input/indoor-location-navigation/\"):\n        self.site_id = site_id\n        self.floor = floor\n        self.input_path = input_path\n        assert Path(input_path).exists(\n        ), f\"input_path do not exist: {input_path}\"\n\n    def get_site_info(self, keep_raw=False):\n        floor_info_path = f\"{self.input_path}/metadata/{self.site_id}/{self.floor}/floor_info.json\"\n        with open(floor_info_path, \"r\") as f:\n            self.floor_info = json.loads(f.read())\n            self.site_height = self.floor_info[\"map_info\"][\"height\"]\n            self.site_width = self.floor_info[\"map_info\"][\"width\"]\n            if not keep_raw:\n                del self.floor_info\n\n        geojson_map_path = f\"{self.input_path}/metadata/{self.site_id}/{self.floor}/geojson_map.json\"\n        with open(geojson_map_path, \"r\") as f:\n            self.geojson_map = json.loads(f.read())\n            self.map_type = self.geojson_map[\"type\"]\n            self.features = self.geojson_map[\"features\"]\n\n            self.floor_coordinates = self.features[0][\"geometry\"][\"coordinates\"]\n            self.store_coordinates = [self.features[i][\"geometry\"][\"coordinates\"]\n                                      for i in range(1, len(self.features))]\n\n            if not keep_raw:\n                del self.geojson_map\n\n    def show_site_image(self):\n        path = f\"{self.input_path}/metadata/{self.site_id}/{self.floor}/floor_image.png\"\n        plt.imshow(imread(path), extent=[\n                   0, self.site_width, 0, self.site_height])\n\n    def draw_polygon(self, size=8, only_floor=False):\n\n        fig = plt.figure()\n        ax = plt.subplot(111)\n\n        xmax, xmin, ymax, ymin = self._draw(\n            self.floor_coordinates, ax, calc_minmax=True)\n        if not only_floor:\n            self._draw(self.store_coordinates, ax, fill=True)\n        plt.legend([])\n\n        xrange = xmax - xmin\n        yrange = ymax - ymin\n        ratio = yrange / xrange\n\n        self.x_size = size\n        self.y_size = size*ratio\n\n        fig.set_figwidth(size)\n        fig.set_figheight(size*ratio)\n        # plt.show()\n        return ax\n\n    def _draw(self, coordinates, ax, fill=False, calc_minmax=False):\n        xmax, ymax = -np.inf, -np.inf\n        xmin, ymin = np.inf, np.inf\n        for i in range(len(coordinates)):\n            ndim = np.ndim(coordinates[i])\n            if ndim == 2:\n                corrd_df = pd.DataFrame(coordinates[i])\n                if fill:\n                    ax.fill(corrd_df[0], corrd_df[1], alpha=0.7)\n                else:\n                    corrd_df.plot.line(x=0, y=1, style=\"-\", ax=ax)\n\n                if calc_minmax:\n                    xmax = max(xmax, corrd_df[0].max())\n                    xmin = min(xmin, corrd_df[0].min())\n\n                    ymax = max(ymax, corrd_df[1].max())\n                    ymin = min(ymin, corrd_df[1].min())\n            elif ndim == 3:\n                for j in range(len(coordinates[i])):\n                    corrd_df = pd.DataFrame(coordinates[i][j])\n                    if fill:\n                        ax.fill(corrd_df[0], corrd_df[1], alpha=0.6)\n                    else:\n                        corrd_df.plot.line(x=0, y=1, style=\"-\", ax=ax)\n\n                    if calc_minmax:\n                        xmax = max(xmax, corrd_df[0].max())\n                        xmin = min(xmin, corrd_df[0].min())\n\n                        ymax = max(ymax, corrd_df[1].max())\n                        ymin = min(ymin, corrd_df[1].min())\n            else:\n                assert False, f\"ndim of coordinates should be 2 or 3: {ndim}\"\n        if calc_minmax:\n            return xmax, xmin, ymax, ymin\n        else:\n            return None","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_meta_data\ntrain_meta = glob(data_dir+\"train/*/*/*\")\ntrain_meta_org = pd.DataFrame(train_meta)\ntrain_meta = train_meta_org[0].str.split(\"/\", expand=True)[[4, 5, 6]]\ntrain_meta.columns = [\"site_id\", \"floor\", \"path_id\"]\ntrain_meta[\"path_id\"] = train_meta[\"path_id\"].str.replace(\".txt\", \"\")\ntrain_meta[\"path\"] = train_meta_org[0]\ntrain_meta.head()\n\n\n# test_meta_data\nsample_sub = pd.read_csv(data_dir+'sample_submission.csv')\ntest_meta = sample_sub[\"site_path_timestamp\"].apply(\n    lambda x: pd.Series(x.split(\"_\")))\ntest_meta.columns = [\"site_id\", \"path_id\", \"timestamp\"]\ntest_meta = test_meta.drop('timestamp', axis=1)\ntest_meta = test_meta.drop_duplicates(\n    subset=[\"site_id\", \"path_id\"]).reset_index(drop=True)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_txt(file):\n    with open(file) as f:\n        txt = f.readlines()\n\n    modified_data = []\n    for s in txt:\n        if s.count(\"TYPE_\") > 1:\n            lines = multi_line_spliter(s)\n            modified_data.extend(lines)\n        else:\n            modified_data.append(s)\n    return modified_data\n\n\ndef _flatten(l):\n    return list(itertools.chain.from_iterable(l))\n\n\ndef get_feature_test(site_id, path_id, input_path, sample_sub):\n    file = f\"{input_path}/test/{path_id}.txt\"\n    content = read_txt(file)\n    data_df = pd.DataFrame([d.replace(\"\\n\", \"\").split(\"\\t\")\n                            for d in content if d[0] != \"#\"])\n    data_dict = OrderedDict()\n    for dt in FeatureStore.df_types:\n        # select data type\n        df_s = data_df[data_df[1] == f\"TYPE_{dt.upper()}\"]\n        if len(df_s) == 0:\n            setattr(data_dict, dt, pd.DataFrame(\n                columns=FeatureStore.df_type_cols[dt]))\n        else:\n            # remove empty cols\n            na_info = df_s.isna().sum(axis=0) == len(df_s)\n            df_s = df_s[[i for i in na_info[na_info ==\n                                            False].index if i != 1]].reset_index(drop=True)\n\n            if len(df_s.columns) != len(FeatureStore.df_type_cols[dt]):\n                df_s.columns = FeatureStore.df_type_cols[dt][:len(\n                    df_s.columns)]\n            else:\n                df_s.columns = FeatureStore.df_type_cols[dt]\n\n            # set dtype\n            for c in df_s.columns:\n                df_s[c] = df_s[c].astype(FeatureStore.dtype_dict[dt][c])\n            setattr(data_dict, dt, df_s)\n    data_dict.meta_info_df = pd.DataFrame([m.replace(\"\\n\", \"\").split(\":\")\n                                           for m in _flatten([d.split(\"\\t\")\n                                                              for d in content if d[0] == \"#\"]) if m != \"#\"])\n    data_dict.meta_info_df[data_dict.meta_info_df[0] == 'startTime'][1] = int(\n        data_dict.meta_info_df[0][int(np.where(data_dict.meta_info_df[0] == 'startTime')[0]+1)])\n    data_dict.meta_info_df[data_dict.meta_info_df[0] == 'endTime'][1] = int(\n        data_dict.meta_info_df[0][int(np.where(data_dict.meta_info_df[0] == 'endTime')[0]+1)])\n\n    data_dict.waypoint['timestamp'] = sample_sub[sample_sub.path_id ==\n                                                 path_id].timestamp.values.astype(int)\n    data_dict.waypoint['x'] = 0\n    data_dict.waypoint['y'] = 0\n    data_dict.n_floor = 0\n    data_dict.site_id = site_id\n    return data_dict\n\ncreate_feature_pickle=False\nif create_feature_pickle:\n    print('create_feature_pickle')\n    for i in tqdm(range(len(train_meta))):\n        t = train_meta.iloc[i]\n        #print(f\"site_id: {t.site_id}, floor: {t.floor}, path_id: {t.path_id}\")\n        feature = FeatureStore(site_id=t.site_id, floor=t.floor,\n                               path_id=t.path_id, input_path=data_dir)\n        feature.load_all_data()\n        pickle_dump_dill(feature, out_dir+'feature/'+t.path_id+'.pickle')\n\n    for i in tqdm(range(len(test_meta))):\n        t = test_meta.iloc[i]\n        #print(f\"site_id: {t.site_id}, floor: {t.floor}, path_id: {t.path_id}\")\n        feature = get_feature_test(\n            site_id=t.site_id, path_id=t.path_id, input_path=data_dir, sample_sub=sample_sub)\n        print(feature.meta_info_df[0][1])\n        pickle_dump_dill(feature, out_dir+'feature/'+t.path_id+'.pickle')\n\n\ndef generate_count_rank(df, column):\n    count_rank = df.groupby(column)[column].count().rank(\n        ascending=False, method='first')\n    df[column+'_count_rank'] = df[column].map(count_rank)\n    #df[column+'_count_rank'] = df[column+'_count_rank'].astype(np.int32)\n    return df, count_rank\n\n\ndef add_count_rank(df, count_rank_dict, column, mode='convert'):\n    if mode == 'convert':\n        df[column] = df[column].map(count_rank_dict.get)\n    elif mode == 'add':\n        df[column+'_count_rank'] = df[column].map(count_rank_dict.get)\n    #df[column+'_count_rank'] = df[column+'_count_rank'].astype(np.int32)\n    return df","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fast_left_merge(df1, df2, key='timestamp'):\n    df2 = df2.set_index(key)\n    df = pd.concat([df1.reset_index(drop=True), df2.reindex(\n        df1[key].values).reset_index(drop=True)], axis=1)\n    return df\n\n\ndef drop_duplicate_wifi(wifi_df, last_seen_timestamp_margin=10):\n    old_bssid = ''\n    old_rssi = -9999\n    old_last_seen_timestamp = -9999\n    if len(wifi_df) > 0:\n        new_rows = []\n        wifi_df = wifi_df.sort_values(['bssid', 'timestamp'])\n        for cnt, row in enumerate(wifi_df[['timestamp', 'ssid', 'bssid', 'rssi', 'frequency', 'last_seen_timestamp']].values):\n            bssid = str(row[2])\n            rssi = int(row[3])\n            last_seen_timestamp = int(row[4])\n            if (bssid != old_bssid) or (rssi != old_rssi) or (abs(last_seen_timestamp-old_last_seen_timestamp) > last_seen_timestamp_margin):\n                new_rows.append(row)\n            old_bssid = bssid\n            old_rssi = rssi\n            old_last_seen_timestamp = last_seen_timestamp\n        new_rows = np.vstack(new_rows)\n        new_wifi_df = pd.DataFrame(new_rows, columns=[\n                                   'timestamp', 'ssid', 'bssid', 'rssi', 'frequency', 'last_seen_timestamp'])\n        new_wifi_df = new_wifi_df.sort_values('timestamp')\n    else:\n        new_wifi_df = wifi_df\n    return new_wifi_df\n\n# https://github.com/location-competition/indoor-location-competition-20/blob/master/compute_f.py\n\n\ndef split_ts_seq(ts_seq, sep_ts):\n    \"\"\"\n    :param ts_seq:\n    :param sep_ts:\n    :return:\n    \"\"\"\n    tss = ts_seq[:, 0].astype(float)\n    unique_sep_ts = np.unique(sep_ts)\n    ts_seqs = []\n    start_index = 0\n    for i in range(0, unique_sep_ts.shape[0]):\n        end_index = np.searchsorted(tss, unique_sep_ts[i], side='right')\n        if start_index == end_index:\n            continue\n        ts_seqs.append(ts_seq[start_index:end_index, :].copy())\n        start_index = end_index\n\n    # tail data\n    if start_index < ts_seq.shape[0]:\n        ts_seqs.append(ts_seq[start_index:, :].copy())\n\n    return ts_seqs\n\n\ndef correct_trajectory(original_xys, end_xy):\n    \"\"\"\n    :param original_xys: numpy ndarray, shape(N, 2)\n    :param end_xy: numpy ndarray, shape(1, 2)\n    :return:\n    \"\"\"\n    corrected_xys = np.zeros((0, 2))\n\n    A = original_xys[0, :]\n    B = end_xy\n    Bp = original_xys[-1, :]\n\n    angle_BAX = np.arctan2(B[1] - A[1], B[0] - A[0])\n    angle_BpAX = np.arctan2(Bp[1] - A[1], Bp[0] - A[0])\n    angle_BpAB = angle_BpAX - angle_BAX\n    AB = np.sqrt(np.sum((B - A) ** 2))\n    ABp = np.sqrt(np.sum((Bp - A) ** 2))\n\n    corrected_xys = np.append(corrected_xys, [A], 0)\n    for i in np.arange(1, np.size(original_xys, 0)):\n        angle_CpAX = np.arctan2(\n            original_xys[i, 1] - A[1], original_xys[i, 0] - A[0])\n\n        angle_CAX = angle_CpAX - angle_BpAB\n\n        ACp = np.sqrt(np.sum((original_xys[i, :] - A) ** 2))\n\n        AC = ACp * AB / ABp\n\n        delta_C = np.array([AC * np.cos(angle_CAX), AC * np.sin(angle_CAX)])\n\n        C = delta_C + A\n\n        corrected_xys = np.append(corrected_xys, [C], 0)\n\n    return corrected_xys\n\n\ndef correct_positions(rel_positions, reference_positions):\n    \"\"\"\n    :param rel_positions:\n    :param reference_positions:\n    :return:\n    \"\"\"\n    rel_positions_list = split_ts_seq(rel_positions, reference_positions[:, 0])\n    if len(rel_positions_list) != reference_positions.shape[0] - 1:\n        # print(f'Rel positions list size: {len(rel_positions_list)}, ref positions size: {reference_positions.shape[0]}')\n        del rel_positions_list[-1]\n    assert len(rel_positions_list) == reference_positions.shape[0] - 1\n\n    corrected_positions = np.zeros((0, 3))\n    for i, rel_ps in enumerate(rel_positions_list):\n        start_position = reference_positions[i]\n        end_position = reference_positions[i + 1]\n        abs_ps = np.zeros(rel_ps.shape)\n        abs_ps[:, 0] = rel_ps[:, 0]\n        # abs_ps[:, 1:3] = rel_ps[:, 1:3] + start_position[1:3]\n        abs_ps[0, 1:3] = rel_ps[0, 1:3] + start_position[1:3]\n        for j in range(1, rel_ps.shape[0]):\n            abs_ps[j, 1:3] = abs_ps[j-1, 1:3] + rel_ps[j, 1:3]\n        abs_ps = np.insert(abs_ps, 0, start_position, axis=0)\n        corrected_xys = correct_trajectory(abs_ps[:, 1:3], end_position[1:3])\n        corrected_ps = np.column_stack((abs_ps[:, 0], corrected_xys))\n        if i == 0:\n            corrected_positions = np.append(\n                corrected_positions, corrected_ps, axis=0)\n        else:\n            corrected_positions = np.append(\n                corrected_positions, corrected_ps[1:], axis=0)\n\n    corrected_positions = np.array(corrected_positions)\n\n    return corrected_positions\n\n\ndef init_parameters_filter(sample_freq, warmup_data, cut_off_freq=2):\n    order = 4\n    filter_b, filter_a = signal.butter(\n        order, cut_off_freq / (sample_freq / 2), 'low', False)\n    zf = signal.lfilter_zi(filter_b, filter_a)\n    _, zf = signal.lfilter(filter_b, filter_a, warmup_data, zi=zf)\n    _, filter_zf = signal.lfilter(filter_b, filter_a, warmup_data, zi=zf)\n\n    return filter_b, filter_a, filter_zf\n\n\ndef get_rotation_matrix_from_vector(rotation_vector):\n    q1 = rotation_vector[0]\n    q2 = rotation_vector[1]\n    q3 = rotation_vector[2]\n\n    if rotation_vector.size >= 4:\n        q0 = rotation_vector[3]\n    else:\n        q0 = 1 - q1*q1 - q2*q2 - q3*q3\n        if q0 > 0:\n            q0 = np.sqrt(q0)\n        else:\n            q0 = 0\n\n    sq_q1 = 2 * q1 * q1\n    sq_q2 = 2 * q2 * q2\n    sq_q3 = 2 * q3 * q3\n    q1_q2 = 2 * q1 * q2\n    q3_q0 = 2 * q3 * q0\n    q1_q3 = 2 * q1 * q3\n    q2_q0 = 2 * q2 * q0\n    q2_q3 = 2 * q2 * q3\n    q1_q0 = 2 * q1 * q0\n\n    R = np.zeros((9,))\n    if R.size == 9:\n        R[0] = 1 - sq_q2 - sq_q3\n        R[1] = q1_q2 - q3_q0\n        R[2] = q1_q3 + q2_q0\n\n        R[3] = q1_q2 + q3_q0\n        R[4] = 1 - sq_q1 - sq_q3\n        R[5] = q2_q3 - q1_q0\n\n        R[6] = q1_q3 - q2_q0\n        R[7] = q2_q3 + q1_q0\n        R[8] = 1 - sq_q1 - sq_q2\n\n        R = np.reshape(R, (3, 3))\n    elif R.size == 16:\n        R[0] = 1 - sq_q2 - sq_q3\n        R[1] = q1_q2 - q3_q0\n        R[2] = q1_q3 + q2_q0\n        R[3] = 0.0\n\n        R[4] = q1_q2 + q3_q0\n        R[5] = 1 - sq_q1 - sq_q3\n        R[6] = q2_q3 - q1_q0\n        R[7] = 0.0\n\n        R[8] = q1_q3 - q2_q0\n        R[9] = q2_q3 + q1_q0\n        R[10] = 1 - sq_q1 - sq_q2\n        R[11] = 0.0\n\n        R[12] = R[13] = R[14] = 0.0\n        R[15] = 1.0\n\n        R = np.reshape(R, (4, 4))\n\n    return R\n\n\ndef get_orientation(R):\n    flat_R = R.flatten()\n    values = np.zeros((3,))\n    if np.size(flat_R) == 9:\n        values[0] = np.arctan2(flat_R[1], flat_R[4])\n        values[1] = np.arcsin(-flat_R[7])\n        values[2] = np.arctan2(-flat_R[6], flat_R[8])\n    else:\n        values[0] = np.arctan2(flat_R[1], flat_R[5])\n        values[1] = np.arcsin(-flat_R[9])\n        values[2] = np.arctan2(-flat_R[8], flat_R[10])\n\n    return values\n\n\ndef compute_steps(acce_datas):\n    step_timestamps = np.array([])\n    step_indexs = np.array([], dtype=int)\n    step_acce_max_mins = np.zeros((0, 4))\n    sample_freq = 50\n    window_size = 22\n    low_acce_mag = 0.6\n    step_criterion = 1\n    interval_threshold = 250\n\n    acce_max = np.zeros((2,))\n    acce_min = np.zeros((2,))\n    acce_binarys = np.zeros((window_size,), dtype=int)\n    acce_mag_pre = 0\n    state_flag = 0\n\n    warmup_data = np.ones((window_size,)) * 9.81\n    filter_b, filter_a, filter_zf = init_parameters_filter(\n        sample_freq, warmup_data)\n    acce_mag_window = np.zeros((window_size, 1))\n\n    # detect steps according to acceleration magnitudes\n    for i in np.arange(0, np.size(acce_datas, 0)):\n        acce_data = acce_datas[i, :]\n        acce_mag = np.sqrt(np.sum(acce_data[1:] ** 2))\n\n        acce_mag_filt, filter_zf = signal.lfilter(\n            filter_b, filter_a, [acce_mag], zi=filter_zf)\n        acce_mag_filt = acce_mag_filt[0]\n\n        acce_mag_window = np.append(acce_mag_window, [acce_mag_filt])\n        acce_mag_window = np.delete(acce_mag_window, 0)\n        mean_gravity = np.mean(acce_mag_window)\n        acce_std = np.std(acce_mag_window)\n        mag_threshold = np.max([low_acce_mag, 0.4 * acce_std])\n\n        # detect valid peak or valley of acceleration magnitudes\n        acce_mag_filt_detrend = acce_mag_filt - mean_gravity\n        if acce_mag_filt_detrend > np.max([acce_mag_pre, mag_threshold]):\n            # peak\n            acce_binarys = np.append(acce_binarys, [1])\n            acce_binarys = np.delete(acce_binarys, 0)\n        elif acce_mag_filt_detrend < np.min([acce_mag_pre, -mag_threshold]):\n            # valley\n            acce_binarys = np.append(acce_binarys, [-1])\n            acce_binarys = np.delete(acce_binarys, 0)\n        else:\n            # between peak and valley\n            acce_binarys = np.append(acce_binarys, [0])\n            acce_binarys = np.delete(acce_binarys, 0)\n\n        if (acce_binarys[-1] == 0) and (acce_binarys[-2] == 1):\n            if state_flag == 0:\n                acce_max[:] = acce_data[0], acce_mag_filt\n                state_flag = 1\n            elif (state_flag == 1) and ((acce_data[0] - acce_max[0]) <= interval_threshold) and (\n                    acce_mag_filt > acce_max[1]):\n                acce_max[:] = acce_data[0], acce_mag_filt\n            elif (state_flag == 2) and ((acce_data[0] - acce_max[0]) > interval_threshold):\n                acce_max[:] = acce_data[0], acce_mag_filt\n                state_flag = 1\n\n        # choose reasonable step criterion and check if there is a valid step\n        # save step acceleration data: step_acce_max_mins = [timestamp, max, min, variance]\n        step_flag = False\n        if step_criterion == 2:\n            if (acce_binarys[-1] == -1) and ((acce_binarys[-2] == 1) or (acce_binarys[-2] == 0)):\n                step_flag = True\n        elif step_criterion == 3:\n            if (acce_binarys[-1] == -1) and (acce_binarys[-2] == 0) and (np.sum(acce_binarys[:-2]) > 1):\n                step_flag = True\n        else:\n            if (acce_binarys[-1] == 0) and acce_binarys[-2] == -1:\n                if (state_flag == 1) and ((acce_data[0] - acce_min[0]) > interval_threshold):\n                    acce_min[:] = acce_data[0], acce_mag_filt\n                    state_flag = 2\n                    step_flag = True\n                elif (state_flag == 2) and ((acce_data[0] - acce_min[0]) <= interval_threshold) and (\n                        acce_mag_filt < acce_min[1]):\n                    acce_min[:] = acce_data[0], acce_mag_filt\n        if step_flag:\n            step_timestamps = np.append(step_timestamps, acce_data[0])\n            step_indexs = np.append(step_indexs, [i])\n            step_acce_max_mins = np.append(step_acce_max_mins,\n                                           [[acce_data[0], acce_max[1], acce_min[1], acce_std ** 2]], axis=0)\n        acce_mag_pre = acce_mag_filt_detrend\n\n    return step_timestamps, step_indexs, step_acce_max_mins\n\n\ndef compute_stride_length(step_acce_max_mins):\n    K = 0.4\n    K_max = 0.8\n    K_min = 0.4\n    para_a0 = 0.21468084\n    para_a1 = 0.09154517\n    para_a2 = 0.02301998\n\n    stride_lengths = np.zeros((step_acce_max_mins.shape[0], 2))\n    k_real = np.zeros((step_acce_max_mins.shape[0], 2))\n    step_timeperiod = np.zeros((step_acce_max_mins.shape[0] - 1, ))\n    stride_lengths[:, 0] = step_acce_max_mins[:, 0]\n    window_size = 2\n    step_timeperiod_temp = np.zeros((0, ))\n\n    # calculate every step period - step_timeperiod unit: second\n    for i in range(0, step_timeperiod.shape[0]):\n        step_timeperiod_data = (\n            step_acce_max_mins[i + 1, 0] - step_acce_max_mins[i, 0]) / 1000\n        step_timeperiod_temp = np.append(\n            step_timeperiod_temp, [step_timeperiod_data])\n        if step_timeperiod_temp.shape[0] > window_size:\n            step_timeperiod_temp = np.delete(step_timeperiod_temp, [0])\n        step_timeperiod[i] = np.sum(\n            step_timeperiod_temp) / step_timeperiod_temp.shape[0]\n\n    # calculate parameters by step period and acceleration magnitude variance\n    k_real[:, 0] = step_acce_max_mins[:, 0]\n    k_real[0, 1] = K\n    for i in range(0, step_timeperiod.shape[0]):\n        k_real[i + 1, 1] = np.max([(para_a0 + para_a1 / step_timeperiod[i] +\n                                    para_a2 * step_acce_max_mins[i, 3]), K_min])\n        k_real[i + 1, 1] = np.min([k_real[i + 1, 1], K_max]) * (K / K_min)\n\n    # calculate every stride length by parameters and max and min data of acceleration magnitude\n    stride_lengths[:, 1] = np.max([(step_acce_max_mins[:, 1] - step_acce_max_mins[:, 2]),\n                                   np.ones((step_acce_max_mins.shape[0], ))], axis=0)**(1 / 4) * k_real[:, 1]\n\n    return stride_lengths\n\n\ndef compute_headings(ahrs_datas):\n    headings = np.zeros((np.size(ahrs_datas, 0), 2))\n    for i in np.arange(0, np.size(ahrs_datas, 0)):\n        ahrs_data = ahrs_datas[i, :]\n        rot_mat = get_rotation_matrix_from_vector(ahrs_data[1:])\n        azimuth, pitch, roll = get_orientation(rot_mat)\n        around_z = (-azimuth) % (2 * np.pi)\n        headings[i, :] = ahrs_data[0], around_z\n    return headings\n\n\ndef compute_step_heading(step_timestamps, headings):\n    step_headings = np.zeros((len(step_timestamps), 2))\n    step_timestamps_index = 0\n    for i in range(0, len(headings)):\n        if step_timestamps_index < len(step_timestamps):\n            if headings[i, 0] == step_timestamps[step_timestamps_index]:\n                step_headings[step_timestamps_index, :] = headings[i, :]\n                step_timestamps_index += 1\n        else:\n            break\n    assert step_timestamps_index == len(step_timestamps)\n\n    return step_headings\n\n\ndef compute_rel_positions(stride_lengths, step_headings):\n    rel_positions = np.zeros((stride_lengths.shape[0], 3))\n    for i in range(0, stride_lengths.shape[0]):\n        rel_positions[i, 0] = stride_lengths[i, 0]\n        rel_positions[i, 1] = -stride_lengths[i, 1] * \\\n            np.sin(step_headings[i, 1])\n        rel_positions[i, 2] = stride_lengths[i, 1] * \\\n            np.cos(step_headings[i, 1])\n\n    return rel_positions\n\n\ndef compute_step_positions(acce_datas, ahrs_datas, posi_datas):\n    step_timestamps, step_indexs, step_acce_max_mins = compute_steps(\n        acce_datas)\n    headings = compute_headings(ahrs_datas)\n    stride_lengths = compute_stride_length(step_acce_max_mins)\n    step_headings = compute_step_heading(step_timestamps, headings)\n    rel_positions = compute_rel_positions(stride_lengths, step_headings)\n    step_positions = correct_positions(rel_positions, posi_datas)\n    return step_positions\n\n\norder = 3\nfs = 50.0  # sample rate, Hz\n# fs = 100\n# cutoff = 3.667  # desired cutoff frequency of the filter, Hz\ncutoff = 3\n\nstep_distance = 0.8\nw_height = 1.7\nm_trans = -5\n\n\ndef butter_lowpass(cutoff, fs, order=5):\n    nyq = 0.5 * fs\n    normal_cutoff = cutoff / nyq\n    b, a = butter(order, normal_cutoff, btype='low', analog=False)\n    return b, a\n\n\ndef butter_lowpass_filter(data, cutoff, fs, order=5):\n    b, a = butter_lowpass(cutoff, fs, order=order)\n    y = lfilter(b, a, data)\n    return y\n\n\ndef peak_accel_threshold(data, timestamps, threshold):\n    d_acc = []\n    last_state = 'below'\n    crest_troughs = 0\n    crossings = []\n\n    for i, datum in enumerate(data):\n\n        current_state = last_state\n        if datum < threshold:\n            current_state = 'below'\n        elif datum > threshold:\n            current_state = 'above'\n\n        if current_state is not last_state:\n            if current_state is 'above':\n                crossing = [timestamps[i], threshold]\n                crossings.append(crossing)\n            else:\n                crossing = [timestamps[i], threshold]\n                crossings.append(crossing)\n\n            crest_troughs += 1\n        last_state = current_state\n    return np.array(crossings)\n\n\ndef steps_compute_rel_positions(acce, magn):\n\n    mix_acce = np.sqrt(acce[:, 1:2]**2 + acce[:, 2:3]**2 + acce[:, 3:4]**2)\n    mix_acce = np.concatenate([acce[:, 0:1], mix_acce], 1)\n    mix_df = pd.DataFrame(mix_acce)\n    mix_df.columns = [\"timestamp\", \"acce\"]\n\n    filtered = butter_lowpass_filter(mix_df[\"acce\"], cutoff, fs, order)\n\n    threshold = filtered.mean() * 1.1\n    crossings = peak_accel_threshold(filtered, mix_df[\"timestamp\"], threshold)\n\n    step_sum = len(crossings)/2\n    distance = w_height * 0.4 * step_sum\n\n    mag_df = pd.DataFrame(magn)\n    mag_df.columns = [\"timestamp\", \"x\", \"y\", \"z\"]\n\n    acce_df = pd.DataFrame(acce)\n    acce_df.columns = [\"timestamp\", \"ax\", \"ay\", \"az\"]\n\n    mag_df = pd.merge(mag_df, acce_df, on=\"timestamp\")\n    mag_df.dropna()\n\n    time_di_list = []\n\n    for i in mag_df.iterrows():\n\n        gx, gy, gz = i[1][1], i[1][2], i[1][3]\n        ax, ay, az = i[1][4], i[1][5], i[1][6]\n\n        roll = math.atan2(ay, az)\n        pitch = math.atan2(-1*ax, (ay * math.sin(roll) + az * math.cos(roll)))\n\n        q = m_trans - math.degrees(math.atan2(\n            (gz*math.sin(roll)-gy*math.cos(roll)), (gx*math.cos(pitch) + gy *\n                                                    math.sin(roll)*math.sin(pitch) + gz*math.sin(pitch)*math.cos(roll))\n        )) - 90\n        if q <= 0:\n            q += 360\n        time_di_list.append((i[1][0], q))\n\n    d_list = [x[1] for x in time_di_list]\n\n    steps = []\n    step_time = []\n    di_dict = dict(time_di_list)\n\n    for n, i in enumerate(crossings[:, :1]):\n        if n % 2 == 1:\n            continue\n        direct_now = di_dict[i[0]]\n        dx = math.sin(math.radians(direct_now))\n        dy = math.cos(math.radians(direct_now))\n#         print(int(n/2+1),\"歩目/x:\",dx,\"/y:\",dy,\"/角度：\",direct_now)\n        steps.append((i[0], dx, dy))\n        step_time.append(i[0])\n\n        step_dtime = np.diff(step_time)/1000\n        step_dtime = step_dtime.tolist()\n        step_dtime.insert(0, 5)\n\n        rel_position = []\n\n        wp_idx = 0\n#         print(\"WP:\",round(sample_file.waypoint[0,1],3),round(sample_file.waypoint[0,2],3),sample_file.waypoint[0,0])\n#         print(\"------------------\")\n        for p, i in enumerate(steps):\n            step_distance = 0\n            if step_dtime[p] >= 1:\n                step_distance = w_height*0.25\n            elif step_dtime[p] >= 0.75:\n                step_distance = w_height*0.3\n            elif step_dtime[p] >= 0.5:\n                step_distance = w_height*0.4\n            elif step_dtime[p] >= 0.35:\n                step_distance = w_height*0.45\n            elif step_dtime[p] >= 0.2:\n                step_distance = w_height*0.5\n            else:\n                step_distance = w_height*0.4\n\n#             step_x += i[1]*step_distance\n#             step_y += i[2]*step_distance\n\n            rel_position.append([i[0], i[1]*step_distance, i[2]*step_distance])\n#     print(rel_position)\n\n    return np.array(rel_position)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Helper Functions\nimport pandas as pd\nimport numpy as np\n\nimport json\nimport matplotlib.pylab as plt\nimport random\ndef sub_process(sub, train_waypoints):\n    sub = split_col(sub[['site_path_timestamp','floor','x','y']]).copy()\n    sub = sub.merge(train_waypoints[['site','floorNo','floor']].drop_duplicates(), how='left')\n    return sub\ndef split_col(df):\n    df = pd.concat([\n        df['site_path_timestamp'].str.split('_', expand=True) \\\n        .rename(columns={0:'site',\n                         1:'path',\n                         2:'timestamp'}),\n        df\n    ], axis=1).copy()\n    return df\n\nfloor_map = {\"B2\":-2, \"B1\":-1, \"F1\":0, \"F2\": 1, \"F3\":2,\n             \"F4\":3, \"F5\":4, \"F6\":5, \"F7\":6,\"F8\":7,\"F9\":8,\n             \"1F\":0, \"2F\":1, \"3F\":2, \"4F\":3, \"5F\":4, \"6F\":5,\n             \"7F\":6, \"8F\": 7, \"9F\":8}\n\n\ndef plot_preds(\n    ax,\n    context_text,\n    site,\n    floorNo,\n    sub=None,\n    true_locs=None,\n    base=\"../input/indoor-location-navigation\",\n    show_train=True,\n    show_preds=True,\n    fix_labels=True,\n    map_floor=None\n):\n    \"\"\"\n    Plots predictions on floorplan map.\n    \n    map_floor : use a different floor's map\n    \"\"\"\n    if map_floor is None:\n        map_floor = floorNo\n    # Prepare width_meter & height_meter (taken from the .json file)\n    floor_plan_filename = f\"{base}/metadata/{site}/{map_floor}/floor_image.png\"\n    json_plan_filename = f\"{base}/metadata/{site}/{map_floor}/floor_info.json\"\n    with open(json_plan_filename) as json_file:\n        json_data = json.load(json_file)\n\n    width_meter = json_data[\"map_info\"][\"width\"]\n    height_meter = json_data[\"map_info\"][\"height\"]\n\n    floor_img = plt.imread(f\"{base}/metadata/{site}/{map_floor}/floor_image.png\")\n\n\n    ax.imshow(floor_img)\n\n    if show_train:\n        true_locs = true_locs.query('site == @site and floorNo == @map_floor').copy()\n        true_locs[\"x_\"] = true_locs[\"x\"] * floor_img.shape[0] / height_meter\n        true_locs[\"y_\"] = (\n            true_locs[\"y\"] * -1 * floor_img.shape[1] / width_meter\n        ) + floor_img.shape[0]\n        true_locs.query(\"site == @site and floorNo == @map_floor\").groupby(\"path\").plot(\n            x=\"x_\",\n            y=\"y_\",\n            style=\"+\",\n            ax=ax,\n            label=\"train waypoint location\",\n            color=\"grey\",\n            alpha=0.5,\n        )\n\n    if show_preds:\n        sub = sub.query('site == @site and floorNo == @floorNo').copy()\n        sub[\"x_\"] = sub[\"x\"] * floor_img.shape[0] / height_meter\n        sub[\"y_\"] = (\n            sub[\"y\"] * -1 * floor_img.shape[1] / width_meter\n        ) + floor_img.shape[0]\n        for path, path_data in sub.query(\n            \"site == @site and floorNo == @floorNo\"\n        ).groupby(\"path\"):\n            path_data.plot(\n                x=\"x_\",\n                y=\"y_\",\n                style=\".-\",\n                ax=ax,\n                title=f\"{context_text} - {site} - floor - {floorNo}\",\n                alpha=1,\n                label=path,\n            )\n    if fix_labels:\n        handles, labels = ax.get_legend_handles_labels()\n        by_label = dict(zip(labels, handles))\n        ax.legend(\n            by_label.values(), by_label.keys(), loc=\"center left\", bbox_to_anchor=(1, 0.5)\n        )\n    return\n\nfrom pathlib import Path\ndef generate_target_sites(sub_df):\n    return sorted(sub_df['site'].unique())\n\ndef generate_site_floors_dict(sub_df):\n    sites = generate_target_sites(sub_df)\n    site_floors_dict = {}\n    for site in sites:\n        site_path = Path('/kaggle/input/indoor-location-navigation/train') / site\n        site_floors_dict[site] = [path.name for path in site_path.glob('*')]\n    return site_floors_dict\n\n\ndef plot_sites(sites, sub1_df, sub2_df, sub3_df):\n    num_floors = 0\n    for site in sites:\n        num_floors += len(site_floors_dict[site])\n\n    fig, ax = plt.subplots(num_floors, 3, figsize=(42, 12 * num_floors))\n\n    idx = 0\n    for site in sites:\n        floors = site_floors_dict[site]\n\n        for floor in floors:\n            plot_preds(ax[idx][0], \"raw submission\", site, floor, sub1_df, train_waypoints, show_preds=True)\n            plot_preds(ax[idx][1], \"Postprocess without augmentation\", site, floor, sub2_df, train_waypoints, show_preds=True)\n            plot_preds(ax[idx][2], \"Postprocess with augmentation\", site, floor, sub3_df, train_waypoints, show_preds=True)\n            idx += 1\n    plt.show()\n\n","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generate augmented waypoints\n* Augmented waypoints have similar X and Y values of train ones.\n* Augmented waypoints are in hallways.\n* Augmented waypoints are sufficiently distant from train ones and each other.\n","metadata":{}},{"cell_type":"code","source":"from shapely.geometry import Point\nfrom shapely.geometry.polygon import Polygon\nfrom shapely.ops import nearest_points\nimport shapely.ops as so\nfrom descartes import PolygonPatch\n\n    \ndef normalize_array(array, floor_max, floor_min, site_max):\n    array=(array-floor_min)/(floor_max-floor_min)* site_max\n    return array\n\ncreate_polygon_dict=False\nif create_polygon_dict:\n\n    polygon_dict=defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: Polygon([]))))\n    for i in tqdm(range(len(train_meta))):\n        t = train_meta.iloc[i]\n        n_floor = FeatureStore.floor_convert[t.floor]\n        if polygon_dict[t.site_id][n_floor]['floor_polygons'] == Polygon([]):\n            feature = pickle_load_dill(out_dir+'features/'+path_id+'.pickle')\n            floor_arrays=np.zeros([0,2])\n            for j in range(len(feature.site_info.floor_coordinates)):\n                floor_array=np.array(feature.site_info.floor_coordinates[j]).squeeze()\n                floor_arrays=np.vstack([floor_arrays,floor_array])\n            floor_min=floor_arrays.min(axis=0)\n            floor_max=floor_arrays.max(axis=0)\n            site_max=np.array([feature.site_info.site_width,feature.site_info.site_height])\n\n            floor_polygons=[]\n            for j in range(len(feature.site_info.floor_coordinates)):\n                floor_array=np.array(feature.site_info.floor_coordinates[j]).squeeze()\n                floor_array = normalize_array(floor_array, floor_max, floor_min, site_max)\n                floor_polygon = Polygon(floor_array)\n                floor_polygons.append(floor_polygon)\n            floor_polygons= so.unary_union(floor_polygons)\n\n            store_polygons=[]\n            for j in range(len(feature.site_info.store_coordinates)):\n                for k in range(len(feature.site_info.store_coordinates[j])):\n                    store_array=np.array(feature.site_info.store_coordinates[j][k]).squeeze()\n                    store_array = normalize_array(store_array, floor_max, floor_min, site_max)\n                    store_polygon  = Polygon(store_array)\n                    store_polygons.append(store_polygon)\n            store_polygons= so.unary_union(store_polygons)\n\n            polygon_dict[t.site_id][feature.n_floor]['floor_polygons']=floor_polygons\n            polygon_dict[t.site_id][feature.n_floor]['store_polygons']=store_polygons\nelse:\n    polygon_dict=pickle_load_dill('../input/indoordataset/polygon_dict.pickle')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def subsample_arr(arr, threshold=1):\n    new_arr=[]\n    old_x=-9999\n    temp_list=[]\n    for x in np.sort(arr):\n        if len(temp_list)>0:\n            temp_mean=np.concatenate([temp_list]).mean()\n            if x-temp_mean > threshold:\n                new_arr.append(temp_mean)\n                temp_list=[]\n        temp_list.append(x)\n    if len(temp_list)>0:\n        temp_mean=np.concatenate([temp_list]).mean()\n        new_arr.append(temp_mean)\n        \n    new_arr=np.concatenate([new_arr])\n    return new_arr\n\ndef rot_arr(arr, deg):\n    rad=np.deg2rad(deg)\n    rot_mat = np.array([[np.cos(rad), -np.sin(rad)],\n                  [np.sin(rad),  np.cos(rad)]])\n    return  np.dot(arr,rot_mat)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_aug_wps(wps,site_id,n_floor,polygon_dict,floor_margin=0,store_margin=0,dist_threshold=4,dist_to_wps_threshold=6,grid_threshold=1,num_arr=300, rot_grid=True, aug_generator='grid'):\n    if aug_generator=='random':\n        aug_wps=np.vstack([np.random.random_sample((num_arr))*(wps.max(axis=0)[0]-wps.min(axis=0)[0])+wps.min(axis=0)[0],np.random.random_sample((num_arr))*(wps.max(axis=0)[1]-wps.min(axis=0)[1])+wps.min(axis=0)[1]]).transpose()\n    if aug_generator=='grid':\n        if rot_grid:\n            min_num_grid=np.inf\n            for angle in range(90):\n                rot_wps=rot_arr(wps, angle)\n                wps_x=subsample_arr(rot_wps[:,0], threshold=grid_threshold)\n                wps_y=subsample_arr(rot_wps[:,1], threshold=grid_threshold)\n                len_wps_x=len(wps_x)\n                len_wps_y=len(wps_y)\n                num_grid=len_wps_x*len_wps_y\n                if num_grid<min_num_grid:\n                    min_num_grid=num_grid\n                    min_angle=angle\n            rot_wps=rot_arr(wps, min_angle)\n            wps_x=subsample_arr(rot_wps[:,0], threshold=grid_threshold)\n            wps_y=subsample_arr(rot_wps[:,1], threshold=grid_threshold)\n            len_wps_x=len(wps_x)\n            len_wps_y=len(wps_y)\n            aug_wps=np.vstack([np.tile(wps_x, (len_wps_y)),np.repeat(wps_y, len_wps_x, axis=0)]).transpose()\n            aug_wps=rot_arr(aug_wps, -min_angle)\n        else:\n            wps_x=subsample_arr(wps[:,0], threshold=grid_threshold)\n            wps_y=subsample_arr(wps[:,1], threshold=grid_threshold)\n            len_wps_x=len(wps_x)\n            len_wps_y=len(wps_y)\n            aug_wps=np.vstack([np.tile(wps_x, (len_wps_y)),np.repeat(wps_y, len_wps_x, axis=0)]).transpose()\n    if aug_generator=='triangle':\n        aug_wps=wps_sub.copy()\n        tri = Delaunay(aug_wps)\n        aug_wps=(np.vstack(aug_wps[tri.simplices])+np.vstack(np.roll(aug_wps[tri.simplices],1, axis=1)))/2\n        idx = np.random.randint(len(aug_wps), size=100)\n        aug_wps=aug_wps[idx,:]\n        getunique\n        sorted_idx = np.lexsort(aug_wps.T)\n        sorted_data =  aug_wps[sorted_idx,:]\n        # Get unique row mask\n        row_mask = np.append([True],np.any(np.diff(sorted_data,axis=0),1))\n        # Get unique rows\n        aug_wps = sorted_data[row_mask]   \n    floor_polygons=polygon_dict[site_id][n_floor]['floor_polygons']\n    store_polygons=polygon_dict[site_id][n_floor]['store_polygons']\n    floor_polygons_eroded = floor_polygons.buffer(-floor_margin)\n    store_polygons_dilated = store_polygons.buffer(store_margin)\n    safe_area_polygons=floor_polygons_eroded.difference(store_polygons_dilated)\n    mask_arr=np.array([True]*len(aug_wps))\n    for cnt, row in enumerate(aug_wps):\n        x = float(row[0])\n        y = float(row[1])\n        point = Point([x,y])\n        if not safe_area_polygons.contains(point):\n            mask_arr[cnt]=False\n    #print(aug_wps.shape)\n    aug_wps=aug_wps[mask_arr]\n    #print(aug_wps.shape)\n    \n    #remove aug wps based on distance to wps\n    if len(aug_wps)>0:        \n        dist_aug = distance.cdist(aug_wps, wps, metric='euclidean')\n        dist_aug=dist_aug.min(axis=1)\n        #aug_wps=aug_wps[(dist_aug<7)&(dist_aug>2)]\n        aug_wps=aug_wps[dist_aug>dist_to_wps_threshold]\n        \n    #remove aug wps based on distance among aug wps\n    if len(aug_wps)>0:\n        dist_aug = distance.cdist(aug_wps, aug_wps, metric='euclidean')\n        dist_aug = np.tril(dist_aug, k=-1)\n        dist_aug = np.where(dist_aug == 0, np.inf, dist_aug) \n        for i in range(len(dist_aug)):\n            if dist_aug[i,:].min() < dist_threshold:\n                dist_aug[:,i]=np.inf    \n        dist_aug=dist_aug.min(axis=1)\n        #aug_wps=aug_wps[(dist_aug<7)&(dist_aug>2)]\n        aug_wps=aug_wps[dist_aug>dist_threshold]\n        \n    #print(aug_wps.shape)\n    return aug_wps","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wps_dict = pickle_load_dill(mydata_dir+'wps_dict.pickle')\ntrain_meta_sub = pickle_load_dill(mydata_dir+'train_meta_sub.pickle')\ntest_meta = pickle_load_dill(mydata_dir+'test_meta.pickle')\n#hlwps_df = pd.read_csv('../input/indoor-navigation-hand-labeled-waypoints/waypoint_by_hand.csv')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_aug_wps_dict=True\nif create_aug_wps_dict:\n    aug_wps_dict=defaultdict(lambda: defaultdict(lambda: np.empty([0, 2])))\n    for site_id in tqdm(wps_dict.keys()):\n        for n_floor in wps_dict[site_id].keys():\n            wps=wps_dict[site_id][n_floor]\n            aug_wps_dict[site_id][n_floor] = generate_aug_wps(wps,site_id,n_floor,polygon_dict,floor_margin=0,store_margin=0,dist_threshold=4,dist_to_wps_threshold=6,grid_threshold=1,rot_grid=False,num_arr=300, aug_generator='grid')\n    pickle_dump_dill(aug_wps_dict,'aug_wps_dict.pickle')\nelse:\n    aug_wps_dict = pickle_load_dill(mydata_dir+'aug_wps_dict.pickle')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Correct floor prediction based on the leakages of [shared wifi records](https://www.kaggle.com/tomooinubushi/retrieving-user-id-from-leaked-wifi-feature) and [device IDs](https://www.kaggle.com/c/indoor-location-navigation/discussion/234543)","metadata":{}},{"cell_type":"code","source":"create_df_user = False\nif create_df_user:\n    print('create_df_user')\n    train_meta['start_time'] = 0\n    train_meta['end_time'] = 0\n    train_meta['start_wp_time'] = 0\n    train_meta['start_wp_x'] = 0\n    train_meta['start_wp_y'] = 0\n    train_meta['end_wp_time'] = 0\n    train_meta['end_wp_x'] = 0\n    train_meta['end_wp_y'] = 0\n    train_meta['n_floor'] = 0\n    wifi_dict = defaultdict(lambda: pd.DataFrame())\n    mag_unc_dict = defaultdict(lambda: pd.DataFrame())\n    gyr_unc_dict = defaultdict(lambda: pd.DataFrame())\n    acc_unc_dict = defaultdict(lambda: pd.DataFrame())\n    for i in tqdm(range(len(train_meta))):\n        t = train_meta.iloc[i]\n        feature = pickle_load_dill(out_dir+'feature/'+t.path_id+'.pickle')\n        if feature.meta_info_df[feature.meta_info_df[0] == 'startTime'][1].values == None:\n            start_time = int(np.nanmin([feature.accelerometer.timestamp.min(\n            ), feature.wifi.timestamp.min(), feature.beacon.timestamp.min()]))\n        else:\n            start_time = int(\n                feature.meta_info_df[feature.meta_info_df[0] == 'startTime'][1])\n        if (len(feature.meta_info_df[feature.meta_info_df[0] == 'endTime']) == 0) or (feature.meta_info_df[feature.meta_info_df[0] == 'endTime'][1].values == None):\n            end_time = int(np.nanmax([feature.accelerometer.timestamp.max(\n            ), feature.wifi.timestamp.max(), feature.beacon.timestamp.max()]))\n        else:\n            end_time = int(\n                feature.meta_info_df[feature.meta_info_df[0] == 'endTime'][1])\n        waypoint_df = feature.waypoint.copy()\n        train_meta.loc[i, 'start_time'] = start_time\n        train_meta.loc[i,\n                       'start_wp_time'] = waypoint_df.iloc[0]['timestamp']\n        train_meta.loc[i, 'start_wp_x'] = waypoint_df.iloc[0]['x']\n        train_meta.loc[i, 'start_wp_y'] = waypoint_df.iloc[0]['y']\n        train_meta.loc[i, 'end_time'] = end_time\n        train_meta.loc[i,\n                       'end_wp_time'] = waypoint_df.iloc[-1]['timestamp']\n        train_meta.loc[i, 'end_wp_x'] = waypoint_df.iloc[-1]['x']\n        train_meta.loc[i, 'end_wp_y'] = waypoint_df.iloc[-1]['y']\n        train_meta.loc[i, 'n_floor'] = feature.n_floor\n\n        wifi_dict[t.path_id] = feature.wifi[[\n            'bssid', 'last_seen_timestamp']].drop_duplicates()\n        if 'x2' in feature.accelerometer_uncalibrated.columns:\n            acc_unc_dict[t.path_id] = feature.accelerometer_uncalibrated[[\n                'x2', 'y2', 'z2']].drop_duplicates()\n        if 'x2' in feature.gyroscope_uncalibrated.columns:\n            gyr_unc_dict[t.path_id] = feature.gyroscope_uncalibrated[[\n                'x2', 'y2', 'z2']].drop_duplicates()\n        if 'x2' in feature.magnetic_field_uncalibrated.columns:\n            mag_unc_dict[t.path_id] = feature.magnetic_field_uncalibrated[[\n                'x2', 'y2', 'z2']].drop_duplicates()\n    train_meta = train_meta.sort_values(\n        ['site_id', 'start_time']).reset_index(drop=True)\n\n    test_meta['start_time'] = 0\n    test_meta['end_time'] = 0\n    test_meta['start_wp_time'] = 0\n    test_meta['start_wp_x'] = 0\n    test_meta['start_wp_y'] = 0\n    test_meta['end_wp_time'] = 0\n    test_meta['end_wp_x'] = 0\n    test_meta['end_wp_y'] = 0\n    test_meta['n_floor'] = 0\n    for i in tqdm(range(len(test_meta))):\n        t = test_meta.iloc[i]\n        #print(f\"site_id: {t.site_id}, floor: {t.floor}, path_id: {t.path_id}\")\n        feature = pickle_load_dill(out_dir+'feature/'+t.path_id+'.pickle')\n        if feature.meta_info_df[feature.meta_info_df[0] == 'startTime'][1].values == None:\n            start_time = int(np.nanmin([feature.accelerometer.timestamp.min(\n            ), feature.wifi.timestamp.min(), feature.beacon.timestamp.min()]))\n        else:\n            start_time = int(\n                feature.meta_info_df[feature.meta_info_df[0] == 'startTime'][1])\n        if (len(feature.meta_info_df[feature.meta_info_df[0] == 'endTime']) == 0) or (feature.meta_info_df[feature.meta_info_df[0] == 'endTime'][1].values == None):\n            end_time = int(np.nanmax([feature.accelerometer.timestamp.max(\n            ), feature.wifi.timestamp.max(), feature.beacon.timestamp.max()]))\n        else:\n            end_time = int(\n                feature.meta_info_df[feature.meta_info_df[0] == 'endTime'][1])\n        if len(feature.beacon) > 0:\n            gap = feature.beacon.loc[0, 'timestamp2'] - \\\n                feature.beacon.loc[0, 'timestamp']\n        else:\n            gap = (feature.wifi.last_seen_timestamp.values -\n                   feature.wifi.timestamp.values).max()+324.5229450792481  # from mean gap\n        waypoint_df = feature.waypoint.copy()\n        test_meta.loc[i, 'start_time'] = start_time+gap\n        test_meta.loc[i,\n                      'start_wp_time'] = waypoint_df.iloc[0]['timestamp']\n        test_meta.loc[i, 'start_wp_x'] = waypoint_df.iloc[0]['x']\n        test_meta.loc[i, 'start_wp_y'] = waypoint_df.iloc[0]['y']\n        test_meta.loc[i, 'end_time'] = end_time+gap\n        test_meta.loc[i,\n                      'end_wp_time'] = waypoint_df.iloc[-1]['timestamp']\n        test_meta.loc[i, 'end_wp_x'] = waypoint_df.iloc[-1]['x']\n        test_meta.loc[i, 'end_wp_y'] = waypoint_df.iloc[-1]['y']\n        test_meta.loc[i, 'n_floor'] = feature.n_floor\n\n        wifi_dict[t.path_id] = feature.wifi[[\n            'bssid', 'last_seen_timestamp']].drop_duplicates()\n        if 'x2' in feature.accelerometer_uncalibrated.columns:\n            acc_unc_dict[t.path_id] = feature.accelerometer_uncalibrated[[\n                'x2', 'y2', 'z2']].drop_duplicates()\n        if 'x2' in feature.gyroscope_uncalibrated.columns:\n            gyr_unc_dict[t.path_id] = feature.gyroscope_uncalibrated[[\n                'x2', 'y2', 'z2']].drop_duplicates()\n        if 'x2' in feature.magnetic_field_uncalibrated.columns:\n            mag_unc_dict[t.path_id] = feature.magnetic_field_uncalibrated[[\n                'x2', 'y2', 'z2']].drop_duplicates()\n\n    df_user = pd.merge(train_meta, test_meta, how='outer')\n    df_user = df_user.sort_values(\n        ['site_id', 'start_time']).reset_index(drop=True)\n    print('create_user_id')\n    threshold = 0.00001\n    df_user['user_id_wifi'] = 0\n    df_user['counter_wifi'] = 0\n    df_user['user_id_acc'] = 0\n    df_user['counter_acc'] = 0\n    df_user['min_dist_acc'] = 0\n    df_user['user_id_gyr'] = 0\n    df_user['counter_gyr'] = 0\n    df_user['min_dist_gyr'] = 0\n    df_user['user_id_mag'] = 0\n    df_user['counter_mag'] = 0\n    df_user['min_dist_mag'] = 0\n    n_wifi = 0\n    n_acc = 0\n    n_gyr = 0\n    n_mag = 0\n    for i in tqdm(range(len(df_user))):\n        t = df_user.iloc[i]\n        current_wifi = wifi_dict[t.path_id]\n        min_last_seen_timestamp = current_wifi.last_seen_timestamp.min()\n        df_site = df_user[df_user.site_id == t.site_id]\n        df_site = df_site[df_site.end_time < t.start_time]\n        df_site = df_site[min_last_seen_timestamp < df_site.end_time]\n        counter_wifi = 0\n        counter_acc = 0\n        counter_gyr = 0\n        counter_mag = 0\n        min_dist_acc = np.inf\n        min_dist_gyr = np.inf\n        min_dist_mag = np.inf\n        if len(df_site) > 0:\n            acc = acc_unc_dict[t.path_id]\n            mag = mag_unc_dict[t.path_id]\n            gyr = gyr_unc_dict[t.path_id]\n            for j in range(len(df_site)):\n                t = df_site.iloc[j]\n                old_wifi = wifi_dict[t.path_id]\n                common_wifi = pd.merge(\n                    current_wifi, old_wifi, how='inner', on=['bssid'])\n                common_wifi['diff_time'] = abs(\n                    common_wifi.last_seen_timestamp_x-common_wifi.last_seen_timestamp_y)\n                if (common_wifi.diff_time < 5).sum() > 0:\n                    # If there is a leak\n                    df_user.loc[i, 'user_id_wifi'] = t.user_id_wifi\n                    counter_wifi += 1\n            if len(acc) > 0:\n                if acc.values.sum() > 0:\n                    for j in range(len(df_site)):\n                        t = df_site.iloc[j]\n                        if len(acc_unc_dict[t.path_id]) > 0:\n                            dist_M = distance.cdist(\n                                acc.values, acc_unc_dict[t.path_id].values, metric='euclidean')\n                            min_dist_acc = np.min([min_dist_acc, dist_M.min()])\n                            if dist_M.min() < threshold:\n                                df_user.loc[i, 'user_id_acc'] = t.user_id_acc\n                                counter_acc += 1\n            if len(mag) > 0:\n                for j in range(len(df_site)):\n                    t = df_site.iloc[j]\n                    if len(mag_unc_dict[t.path_id]) > 0:\n                        dist_M = distance.cdist(\n                            mag.values, mag_unc_dict[t.path_id].values, metric='euclidean')\n                        min_dist_mag = np.min([min_dist_mag, dist_M.min()])\n                        if dist_M.min() < threshold:\n                            df_user.loc[i, 'user_id_mag'] = t.user_id_mag\n                            counter_mag += 1\n            if len(gyr) > 0:\n                for j in range(len(df_site)):\n                    t = df_site.iloc[j]\n                    if len(gyr_unc_dict[t.path_id]) > 0:\n                        dist_M = distance.cdist(\n                            gyr.values, gyr_unc_dict[t.path_id].values, metric='euclidean')\n                        min_dist_gyr = np.min([min_dist_gyr, dist_M.min()])\n                        if dist_M.min() < threshold:\n                            df_user.loc[i, 'user_id_gyr'] = t.user_id_gyr\n                            counter_gyr += 1\n        if counter_wifi == 0:\n            df_user.loc[i, 'user_id_wifi'] = n_wifi\n            n_wifi += 1\n        df_user.loc[i, 'counter_wifi'] = counter_wifi\n\n        if counter_acc == 0:\n            df_user.loc[i, 'user_id_acc'] = n_acc\n            n_acc += 1\n        df_user.loc[i, 'counter_acc'] = counter_acc\n        df_user.loc[i, 'min_dist_acc'] = min_dist_acc\n\n        if counter_mag == 0:\n            df_user.loc[i, 'user_id_mag'] = n_mag\n            n_mag += 1\n        df_user.loc[i, 'counter_mag'] = counter_mag\n        df_user.loc[i, 'min_dist_mag'] = min_dist_mag\n\n        if counter_gyr == 0:\n            df_user.loc[i, 'user_id_gyr'] = n_gyr\n            n_gyr += 1\n        df_user.loc[i, 'counter_gyr'] = counter_gyr\n        df_user.loc[i, 'min_dist_gyr'] = min_dist_gyr\n\n    df_user['user_id'] = df_user.user_id_wifi.copy()\n    for user_id in df_user.user_id_acc.unique():\n        df_user.loc[df_user.user_id_acc == user_id,\n                    'user_id'] = df_user.loc[df_user.user_id_acc == user_id, 'user_id'].min()\n    for user_id in df_user.user_id_mag.unique():\n        df_user.loc[df_user.user_id_mag == user_id,\n                    'user_id'] = df_user.loc[df_user.user_id_mag == user_id, 'user_id'].min()\n    for user_id in df_user.user_id_gyr.unique():\n        df_user.loc[df_user.user_id_gyr == user_id,\n                    'user_id'] = df_user.loc[df_user.user_id_gyr == user_id, 'user_id'].min()\n    for user_id in df_user.user_id_wifi.unique():\n        df_user.loc[df_user.user_id_wifi == user_id,\n                    'user_id'] = df_user.loc[df_user.user_id_wifi == user_id, 'user_id'].min()\n    le = LabelEncoder()\n    labels = df_user.user_id\n    le.fit(labels)\n    df_user.user_id = le.transform(labels)\n\n    count_user_id_dict = defaultdict(int)\n    for cnt, row in enumerate(df_user[['start_wp_x', 'user_id']].values):\n        start_wp_x = int(row[0])\n        user_id = int(row[1])\n        if start_wp_x > 0:\n            count_user_id_dict[user_id] += 1\n        else:\n            count_user_id_dict[user_id] = count_user_id_dict[user_id]\n    count_rank_user_id_dict = {key: rank for rank, key in enumerate(\n        sorted(count_user_id_dict, key=count_user_id_dict.get, reverse=True), 1)}\n    if not cfg.debug:\n        df_user.to_csv(out_dir + 'df_user_id.csv')\n        pickle_dump_dill(count_user_id_dict, out_dir +\n                         'count_user_id_dict.pickle')\n        pickle_dump_dill(count_rank_user_id_dict, out_dir +\n                         'count_rank_user_id_dict.pickle')\nelse:\n    df_user = pd.read_csv(mydata_dir + 'df_user_id.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def correct_pred_leak(submission_df,df_user,nigel_pred_df,correct_start_end=True,correct_floor=True, if_all_count_one='nigel',time_threshold_site=10000,time_threshold_user=10000):\n    out_df=submission_df.copy()\n    df_user_train=df_user[df_user.start_wp_x>0]\n    counter_floor = 0\n    counter_start = 0\n    counter_end = 0\n    tmp = out_df['site_path_timestamp'].apply(\n        lambda s: pd.Series(s.split('_')))\n    out_df['site'] = tmp[0]\n    out_df['path'] = tmp[1]\n    out_df['timestamp'] = tmp[2].astype(float)\n    \n    tmp = nigel_pred_df['site_path_timestamp'].apply(\n        lambda s: pd.Series(s.split('_')))\n    nigel_pred_df['site'] = tmp[0]\n    nigel_pred_df['path'] = tmp[1]\n    nigel_pred_df['timestamp'] = tmp[2].astype(float)\n    \n    for path_id in tqdm(out_df.path.unique()):\n        site_id=out_df[out_df.path==path_id].site.values[0]\n        user_id=int(df_user[df_user.path_id==path_id].user_id)\n        pred_floor=out_df[out_df.path==path_id].floor.values[0]\n        start_time=int(df_user[df_user.path_id==path_id].start_time)\n        end_time=int(df_user[df_user.path_id==path_id].end_time)\n        \n        if correct_floor:\n            nigel_pred=int(nigel_pred_df[nigel_pred_df.path==path_id].floor.iloc[0])\n            corrected_floors=[nigel_pred]\n            gap_time_end=0\n            gap_time_start=0\n\n            user_df=df_user_train[df_user_train.user_id==user_id]\n            site_df=df_user_train[df_user_train.site_id==site_id]\n            \n            if len(user_df)>0:        \n                uniqs, counts = np.unique(user_df.n_floor.values, return_counts=True)\n                user_df_floor_mode=int(uniqs[counts == np.amax(counts)][0])\n                corrected_floors.append(user_df_floor_mode)\n                \n            user_df_end = user_df[user_df.end_time < start_time]\n            if len(user_df_end) > 0:\n                nearest_endpoint = user_df_end.loc[user_df_end.end_time.idxmax()]\n                corrected_floors.append(nearest_endpoint.n_floor)\n            else:   \n                site_df_end = site_df[(site_df.end_time < start_time)&(start_time-site_df.end_time<time_threshold_site)]\n                if len(site_df_end) > 0:\n                    nearest_endpoint = site_df_end.loc[site_df_end.end_time.idxmax()]\n                    gap_time_end=start_time-nearest_endpoint.end_time\n                    corrected_floors.append(nearest_endpoint.n_floor)\n                    \n            user_df_start = user_df[end_time < user_df.start_time]\n            if len(user_df_start) > 0:\n                nearest_startpoint = user_df_start.loc[user_df_start.start_time.idxmin()]\n                corrected_floors.append(nearest_startpoint.n_floor)\n            else:\n                site_df_start = site_df[(end_time < site_df.start_time)&(site_df.start_time-end_time<time_threshold_site)]        \n                if len(site_df_start) > 0:\n                    nearest_startpoint = site_df_start.loc[site_df_start.start_time.idxmin()]\n                    gap_time_start=nearest_startpoint.start_time-end_time\n                    corrected_floors.append(nearest_startpoint.n_floor)\n\n            corrected_floors=np.array(corrected_floors)\n            uniqs, counts = np.unique(corrected_floors, return_counts=True)\n            corrected_floor=int(uniqs[counts == np.amax(counts)][0])\n\n            if (counts==1).all():\n                if if_all_count_one == 'nigel':\n                    corrected_floor = nigel_pred\n                if if_all_count_one == 'leak':\n                    if len(user_df)>0:\n                        corrected_floor = user_df_floor_mode\n\n                    if (len(site_df_end) > 0)&(len(site_df_start) > 0):\n                        if gap_time_start < gap_time_end:\n                            corrected_floor = nearest_startpoint.n_floor\n                        else:\n                            corrected_floor = nearest_endpoint.n_floor\n                        \n            if pred_floor != corrected_floor:\n                tqdm.write(path_id+', '+site_id+': '+str(pred_floor)+' -> '+str(corrected_floor)+': '+str(corrected_floors)+': '+str(gap_time_end)+': '+str(gap_time_start))\n                out_df.loc[(out_df.path == path_id),\n                                  'floor'] = corrected_floor\n                pred_floor = corrected_floor\n                counter_floor += 1\n        \n        if correct_start_end:\n            site_df=df_user_train[(df_user_train.site_id==site_id)&(df_user_train.n_floor==pred_floor)]\n            user_df=df_user_train[(df_user_train.user_id==user_id)&(df_user_train.n_floor==pred_floor)]\n            \n            user_df_end = user_df[(user_df.end_time < start_time)]\n            if len(user_df_end) > 0:                \n                user_df_end = user_df_end[(start_time-user_df_end.end_time<time_threshold_user)]\n                if len(user_df_end) > 0:\n                    nearest_endpoint = user_df_end.loc[user_df_end.end_time.idxmax()]\n                    out_df.loc[(out_df.path == path_id) & (out_df.timestamp == \n                        out_df[out_df.path == path_id].timestamp.min()), 'x'] = nearest_endpoint.end_wp_x\n                    out_df.loc[(out_df.path == path_id) & (out_df.timestamp == \n                        out_df[out_df.path == path_id].timestamp.min()), 'y'] = nearest_endpoint.end_wp_y\n                    counter_start += 1\n            else:\n                site_df_end = site_df[(site_df.end_time < start_time)&(start_time-site_df.end_time<time_threshold_site)]\n                if len(site_df_end) > 0:\n                    nearest_endpoint = site_df_end.loc[site_df_end.end_time.idxmax()]\n                    out_df.loc[(out_df.path == path_id) & (out_df.timestamp == \n                        out_df[out_df.path == path_id].timestamp.min()), 'x'] = nearest_endpoint.end_wp_x\n                    out_df.loc[(out_df.path == path_id) & (out_df.timestamp == \n                        out_df[out_df.path == path_id].timestamp.min()), 'y'] = nearest_endpoint.end_wp_y\n                    counter_start += 1\n                \n            user_df_start = user_df[(end_time < user_df.start_time)]\n            if len(user_df_start) > 0:\n                user_df_start = user_df_start[(user_df_start.start_time-end_time<time_threshold_user)]\n                if len(user_df_start) > 0:\n                    nearest_startpoint = user_df_start.loc[user_df_start.start_time.idxmin()]\n                    out_df.loc[(out_df.path == path_id) & (out_df.timestamp == \n                        out_df[out_df.path == path_id].timestamp.max()), 'x'] = nearest_startpoint.start_wp_x\n                    out_df.loc[(out_df.path == path_id) & (out_df.timestamp == \n                        out_df[out_df.path == path_id].timestamp.max()), 'y'] = nearest_startpoint.start_wp_y\n                    counter_end += 1    \n            else:\n                site_df_start = site_df[(end_time < site_df.start_time)&(site_df.start_time-end_time<time_threshold_site)]        \n                if len(site_df_start) > 0:\n                    nearest_startpoint = site_df_start.loc[site_df_start.start_time.idxmin()]\n                    out_df.loc[(out_df.path == path_id) & (out_df.timestamp == \n                        out_df[out_df.path == path_id].timestamp.max()), 'x'] = nearest_startpoint.start_wp_x\n                    out_df.loc[(out_df.path == path_id) & (out_df.timestamp == \n                        out_df[out_df.path == path_id].timestamp.max()), 'y'] = nearest_startpoint.start_wp_y\n                    counter_end += 1\n            \n    out_df = out_df.drop([\"site\", \"path\", \"timestamp\"], axis=1)\n    print(str(counter_start) + ' start points are postprocessed.')\n    print(str(counter_end) + ' end points are postprocessed.')\n    print(str(counter_floor) + ' floors are postprocessed.')\n    return out_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Postprocessing based on Dijkstra's algorithm\nI searched the path with minimal [cost](https://www.kaggle.com/saitodevel01/indoor-post-processing-by-cost-minimization).","metadata":{}},{"cell_type":"code","source":"# https://note.nkmk.me/python-scipy-shortest-path/\ndef get_path(start, goal, pred):\n    return get_path_row(start, goal, pred[start])\n\n\ndef get_path_row(start, goal, pred_row):\n    path = []\n    i = goal\n    while i != start and i >= 0:\n        path.append(i)\n        i = pred_row[i]\n    if i < 0:\n        return []\n    path.append(i)\n    return path[::-1]\n\n\ndef shortest_path_postprocessing(input_wps, feature, wps,buffer_x=5, buffer_y=5, abs_ratio=0.05, sensor_mode='both'):\n    acce_datas = feature.accelerometer.copy()\n    acce_datas = acce_datas.values[:, :4]\n    waypoint_timestamps = feature.waypoint.timestamp\n\n    if (sensor_mode == 'rot') or (sensor_mode == 'both'):\n        ahrs_datas = feature.rotation_vector.copy()\n        ahrs_datas = ahrs_datas.values[:, :4]\n        step_timestamps, step_indexs, step_acce_max_mins = compute_steps(\n            acce_datas)\n        headings = compute_headings(ahrs_datas)\n        stride_lengths = compute_stride_length(step_acce_max_mins)\n        step_headings = compute_step_heading(step_timestamps, headings)\n        rel_pos_rot = compute_rel_positions(stride_lengths, step_headings)\n        rel_pos_rot = np.concatenate(\n            [np.array([[acce_datas[0, 0], 0, 0]]), rel_pos_rot])\n        rel_cumsum_rot = np.cumsum(rel_pos_rot, axis=0)\n        rel_t_rot = rel_pos_rot[:, 0]\n        rel_x_rot = rel_cumsum_rot[:, 1]\n        rel_y_rot = rel_cumsum_rot[:, 2]\n        rel_wx_rot = np.interp(waypoint_timestamps, rel_t_rot, rel_x_rot)\n        rel_wy_rot = np.interp(waypoint_timestamps, rel_t_rot, rel_y_rot)\n        rel_w = np.array([rel_wx_rot, rel_wy_rot]).transpose([1, 0])\n\n    if (sensor_mode == 'mag') or (sensor_mode == 'both'):\n        magn_datas = feature.magnetic_field.copy()\n        magn_datas = magn_datas.values[:, :4]\n        rel_pos_mag = steps_compute_rel_positions(acce_datas, magn_datas)\n        rel_pos_mag = np.concatenate(\n            [np.array([[acce_datas[0, 0], 0, 0]]), rel_pos_mag])\n        rel_cumsum_mag = np.cumsum(rel_pos_mag, axis=0)\n        rel_t_mag = rel_pos_mag[:, 0]\n        rel_x_mag = rel_cumsum_mag[:, 1]\n        rel_y_mag = rel_cumsum_mag[:, 2]\n        rel_wx_mag = np.interp(waypoint_timestamps, rel_t_mag, rel_x_mag)\n        rel_wy_mag = np.interp(waypoint_timestamps, rel_t_mag, rel_y_mag)\n        rel_w = np.array([rel_wx_mag, rel_wy_mag]).transpose([1, 0])\n\n    if sensor_mode == 'both':\n        rel1 = rel_pos_rot.copy()\n        rel2 = rel_pos_mag.copy()\n        rel1[:, 1:] = rel_pos_rot[:, 1:] / 2\n        rel2[:, 1:] = rel_pos_mag[:, 1:] / 2\n        rel_positions = np.vstack([rel1, rel2])\n        rel_pos_rm = rel_positions[np.argsort(rel_positions[:, 0])]\n        rel_pos_rm = np.concatenate(\n            [np.array([[acce_datas[0, 0], 0, 0]]), rel_pos_rm])\n        rel_cumsum_rm = np.cumsum(rel_pos_rm, axis=0)\n        rel_t_rm = rel_pos_rm[:, 0]\n        rel_x_rm = rel_cumsum_rm[:, 1]\n        rel_y_rm = rel_cumsum_rm[:, 2]\n        rel_wx_rm = np.interp(waypoint_timestamps, rel_t_rm, rel_x_rm)\n        rel_wy_rm = np.interp(waypoint_timestamps, rel_t_rm, rel_y_rm)\n        rel_w = np.array([rel_wx_rm, rel_wy_rm]).transpose([1, 0])\n\n    rel_input_wps = np.diff(rel_w, axis=0)\n\n    min_x = input_wps.min(axis=0)[0]-buffer_x\n    max_x = input_wps.max(axis=0)[0]+buffer_x\n    min_y = input_wps.min(axis=0)[1]-buffer_y\n    max_y = input_wps.max(axis=0)[1]+buffer_y\n    wps_sub = wps[(wps[:, 0] > min_x) & (max_x > wps[:, 0]) &\n                  (wps[:, 1] > min_y) & (max_y > wps[:, 1])]\n    if len(wps_sub) < 2:\n        min_x = input_wps.min(axis=0)[0]-buffer_x*10\n        max_x = input_wps.max(axis=0)[0]+buffer_x*10\n        min_y = input_wps.min(axis=0)[1]-buffer_y*10\n        max_y = input_wps.max(axis=0)[1]+buffer_y*10\n        wps_sub = wps[(wps[:, 0] > min_x) & (max_x > wps[:, 0])\n                      & (wps[:, 1] > min_y) & (max_y > wps[:, 1])]\n        if len(wps_sub) > 1:\n            print('No candidate waypoints for ' +\n                  feature.path_id+'. Use buffer x 10. Number of waypoints now: '+str(len(wps_sub)))\n        else:\n            print('No candidate waypoints for ' +\n                  feature.path_id+'. Use all waypoints')\n            wps_sub = wps\n    len_input_wps = len(input_wps)\n    len_wps_sub = len(wps_sub)\n\n    rel_wps = np.tile(wps_sub, (len_wps_sub, 1)) - \\\n        np.repeat(wps_sub, len_wps_sub, axis=0)\n    dist_abs = distance.cdist(input_wps, wps_sub, metric='euclidean')\n    dist_rel = distance.cdist(rel_input_wps, rel_wps, metric='euclidean')\n    dist_abs = dist_abs**2\n    dist_rel = dist_rel**2\n\n    cost_mat_abs = np.zeros(\n        [len_input_wps*len_wps_sub+2, len_input_wps*len_wps_sub+2])\n    cost_mat_abs[0, 1:len_wps_sub+1] = dist_abs[0, :]\n    cost_mat_abs[(len_input_wps-1)*len_wps_sub+1:(len_input_wps)\n                 * len_wps_sub+1, len_input_wps*len_wps_sub+1] = 1\n    cost_mat_rel = np.zeros(\n        [len_input_wps*len_wps_sub+2, len_input_wps*len_wps_sub+2])\n\n    for i in range(len_input_wps-1):\n        cost_mat_abs[i*len_wps_sub+1:(i+1)*len_wps_sub+1, (i+1)*len_wps_sub+1:(\n            i+2)*len_wps_sub+1] = np.tile(dist_abs[i+1, :], (len_wps_sub, 1)).transpose([1, 0])\n        cost_mat_rel[i*len_wps_sub+1:(i+1)*len_wps_sub+1, (i+1)*len_wps_sub+1:(\n            i+2)*len_wps_sub+1] = dist_rel[i, :].reshape([len_wps_sub, len_wps_sub])\n    cost_mat = cost_mat_abs*abs_ratio+cost_mat_rel*(1-abs_ratio)\n\n    d, p = shortest_path(cost_mat, indices=[\n                         0, len_input_wps*len_wps_sub+1], return_predecessors=True)\n\n    path = get_path(0, len_input_wps*len_wps_sub+1, p)\n    output_wps = input_wps.copy()\n    for i in range(len(output_wps)):\n        output_wps[i, :] = wps_sub[(path[i+1]-1) % len_wps_sub]\n    return output_wps, rel_w\n\n\ndef plot_shortest_path_postprocessing(input_wps, output_wps, rel_w, ground_truth, feature, wps, aug_wps=np.empty([0,2])):\n    plt.plot(rel_w[:, 0]+input_wps[0, 0],\n             rel_w[:, 1]+input_wps[0, 1], label=\"sensor\")\n    plt.scatter(wps[:, 0], wps[:, 1], label=\"wps\")\n    if len(aug_wps)>0:\n        plt.scatter(aug_wps[:, 0], aug_wps[:, 1], label=\"aug_wps\")\n    plt.plot(input_wps[:, 0], input_wps[:, 1], label=\"input\")\n    plt.plot(output_wps[:, 0]-0.25, output_wps[:, 1]-0.25, label=\"output\")\n    if len(ground_truth)>0:\n        plt.plot(ground_truth[:, 0]+0.25, ground_truth[:, 1]+0.25, label=\"truth\")\n        plt.axis([ground_truth.min(axis=0)[0]-20, ground_truth.max(axis=0)[0]+20,\n                  ground_truth.min(axis=0)[1]-20, ground_truth.max(axis=0)[1]+20])\n        error_input = np.mean(np.sqrt(np.sum((input_wps-ground_truth)**2, axis=1)))\n        error_output = np.mean(\n            np.sqrt(np.sum((output_wps-ground_truth)**2, axis=1)))\n        plt.title(f\"{feature.path_id}: {error_input:.5f} -> {error_output:.5f}\")\n    else:\n        temp_arr=np.vstack([input_wps,rel_w+input_wps[0, :]])\n        plt.axis([temp_arr.min(axis=0)[0]-20, temp_arr.max(axis=0)[0]+20,\n                  temp_arr.min(axis=0)[1]-20, temp_arr.max(axis=0)[1]+20])\n        plt.title(f\"{feature.path_id}\")        \n    plt.legend(loc=\"upper left\")\n    plt.show()\n\n\ndef evaluate_pp(path_ids, wps_dict, aug_wps_dict, buffer_x, buffer_y, abs_ratio, sensor_mode, noize_std, sample_wps=True, sample_wps_rate=0.9, augment_wps=False):\n    error_inputs = []\n    error_outputs = []\n    error_inputs_p = []\n    error_outputs_p = []\n    for path_id in tqdm(path_ids):\n        feature = pickle_load_dill(out_dir+'features/'+path_id+'.pickle')\n        ground_truth = feature.waypoint.values[:, 1:]\n        #input_wps = ground_truth + \\\n        #    (np.random.normal(0, noize_std, (len(ground_truth), 2)))\n        input_wps = ground_truth + \\\n            (np.random.normal(0, noize_std, (len(ground_truth), 2)))+np.ones([len(ground_truth), 2])*np.random.normal(0, noize_std, (1, 2))\n        site_id = feature.site_id\n        n_floor = feature.n_floor\n        wps = wps_dict[site_id][n_floor]\n        if sample_wps:\n            idx = np.random.choice(np.arange(len(wps)), size=int(len(wps)*sample_wps_rate), replace = False)\n            wps=wps[idx,:]            \n        if augment_wps:\n            aug_wps=aug_wps_dict[site_id][n_floor]\n            wps=np.vstack([wps,aug_wps])\n        else:\n            aug_wps=np.array([0,2])\n        output_wps, _ = shortest_path_postprocessing(input_wps, feature, wps, buffer_x, buffer_y, abs_ratio, sensor_mode)\n        error_input = np.sqrt(np.sum((input_wps-ground_truth)**2, axis=1))\n        error_output = np.sqrt(np.sum((output_wps-ground_truth)**2, axis=1))\n        error_input_p = np.mean(\n            np.sqrt(np.sum((input_wps-ground_truth)**2, axis=1)))\n        error_output_p = np.mean(\n            np.sqrt(np.sum((output_wps-ground_truth)**2, axis=1)))\n        error_inputs.append(error_input)\n        error_outputs.append(error_output)\n        error_inputs_p.append([error_input_p])\n        error_outputs_p.append([error_input_p])\n\n    error_inputs = np.concatenate(error_inputs)\n    error_outputs = np.concatenate(error_outputs)\n    error_inputs_p = np.concatenate([error_inputs_p])\n    error_outputs_p = np.concatenate([error_outputs_p])\n    return error_inputs.mean(), error_outputs.mean(), error_inputs_p, error_outputs_p\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count_num_wps_df=pd.DataFrame(test_meta.value_counts('num_wps')).reset_index()\ncount_num_wps_df.columns=['num_wps','num_wps_counts']\ntrain_meta_sub=fast_left_merge(train_meta_sub, count_num_wps_df, key='num_wps')\npath_ids = train_meta_sub.sample(100, replace=False, weights=train_meta_sub.num_wps_counts).path_id.values","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"These are the examples of the results of shortest path postprocessing","metadata":{}},{"cell_type":"code","source":"sample_wps=True\nsample_wps_rate=0.9\naugment_wps=True\nnoize_std=7\nbuffer_x=5\nbuffer_y=5\nabs_ratio=0.05 \nsensor_mode='both'\n#for noize_std in noize_stds:\nfor path_id in path_ids[:10]:\n#path_ids = train_meta_sub.sample(1000, replace=False, weights=train_meta_sub.num_wps_counts).path_id.values\n#for path_id in path_ids:\n    feature = pickle_load_dill(out_dir+'features/'+path_id+'.pickle')\n    ground_truth = feature.waypoint.values[:, 1:]\n    site_id = feature.site_id\n    n_floor = feature.n_floor\n    wps = wps_dict[site_id][n_floor]\n    if sample_wps:\n        idx = np.random.choice(np.arange(len(wps)), size=int(len(wps)*sample_wps_rate), replace = False)\n        wps=wps[idx,:]\n    input_wps = ground_truth + \\\n        (np.random.normal(0, noize_std, (len(ground_truth), 2)))\n    \n    if augment_wps:\n        aug_wps=aug_wps_dict[feature.site_id][feature.n_floor]\n        wps=np.vstack([wps,aug_wps])\n    else:\n        aug_wps=np.array([0,2])\n    output_wps, rel_w = shortest_path_postprocessing(input_wps, feature, wps, buffer_x, buffer_y, abs_ratio, sensor_mode)\n\n    plot_shortest_path_postprocessing(\n        input_wps, output_wps, rel_w, ground_truth, feature, wps, aug_wps)\n\n    #print(path_id)\n    #print(input_wps)\n    #print(output_wps)\n    #print(ground_truth)\n    #error_input = np.mean(np.sqrt(np.sum((input_wps-ground_truth)**2, axis=1)))\n    #error_output = np.mean(\n    #    np.sqrt(np.sum((output_wps-ground_truth)**2, axis=1)))\n    #print(f\"{path_id}: {error_input:.5f} -> {error_output:.5f}\")","metadata":{"_kg_hide-output":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def correct_pred_shortest_path(sub_df, wps_dict, aug_wps_dict=aug_wps_dict, buffer_x=5, buffer_y=5, abs_ratio=0.05, sensor_mode='both', plot_results=True, augment_wps=False):\n    tmp = sub_df['site_path_timestamp'].apply(\n        lambda s: pd.Series(s.split('_')))\n    sub_df['site'] = tmp[0]\n    sub_df['path'] = tmp[1]\n    sub_df['timestamp'] = tmp[2].astype(float)\n\n    dfs = []\n    for path_id, df in tqdm(sub_df.groupby('path')):\n        input_wps = df[['x', 'y']].values.copy()\n        feature = pickle_load_dill(out_dir+'features/'+path_id+'.pickle')\n        feature.path_id=path_id\n        site_id = feature.site_id\n        n_floor = int(df.floor.mean())\n        wps = wps_dict[site_id][n_floor]\n        if augment_wps:\n            aug_wps=aug_wps_dict[site_id][n_floor]\n            wps=np.vstack([wps,aug_wps])\n        else:\n            aug_wps=np.empty([0,2])\n        output_wps, rel_w = shortest_path_postprocessing(input_wps, feature, wps, buffer_x, buffer_y, abs_ratio, sensor_mode)\n        if plot_results:\n            plot_shortest_path_postprocessing(input_wps, output_wps, rel_w, np.empty([0,2]), feature, wps, aug_wps)\n        df[['x', 'y']] = output_wps\n        dfs.append(df)\n    processed_df = pd.concat(dfs).sort_values('site_path_timestamp')\n    processed_df = processed_df.drop([\"site\", \"path\", \"timestamp\"], axis=1)\n    return processed_df\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nigel_pred_df=pd.read_csv('../input/simple-99-accurate-floor-model/submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = pd.read_csv('../input/ensemble-best-lbs/submission.csv')\nsubmission_df_l_f=correct_pred_leak(submission_df,df_user,nigel_pred_df,correct_start_end=False,correct_floor=True, if_all_count_one='nigel',time_threshold_site=10000,time_threshold_user=10000)\nsubmission_df_l_f_pp = correct_pred_shortest_path(submission_df_l_f, wps_dict, plot_results=False,augment_wps=False)\nsubmission_df_l_f_pp.to_csv('submission_df_l_f_pp.csv', index=False)\nsubmission_df_l_f_pp_aug = correct_pred_shortest_path(submission_df_l_f, wps_dict, plot_results=False,augment_wps=True)\nsubmission_df_l_f_pp_aug.to_csv('submission_df_l_f_pp_aug.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualization\n","metadata":{}},{"cell_type":"code","source":"train_waypoints = pd.read_csv('/kaggle/input/indoor-location-train-waypoints/train_waypoints.csv')\nsub_df=submission_df\ntmp = sub_df['site_path_timestamp'].apply(\n    lambda s: pd.Series(s.split('_')))\nsub_df['site'] = tmp[0]\nsub_df['path'] = tmp[1]\nsub_df['timestamp'] = tmp[2].astype(float)\nall_sites = generate_target_sites(submission_df)\nsite_floors_dict = generate_site_floors_dict(submission_df)\nsite_floors_dict\n\nsubmission_df = sub_process(submission_df, train_waypoints)\nsubmission_df_l_f_pp = sub_process(submission_df_l_f_pp, train_waypoints)\nsubmission_df_l_f_pp_aug = sub_process(submission_df_l_f_pp_aug, train_waypoints)\n\nsites1 = all_sites[:8]\nsites2 = all_sites[8:16]\nsites3 = all_sites[16:]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_sites(sites1, submission_df, submission_df_l_f_pp, submission_df_l_f_pp_aug)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_sites(sites2, submission_df, submission_df_l_f_pp, submission_df_l_f_pp_aug)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_sites(sites3, submission_df, submission_df_l_f_pp, submission_df_l_f_pp_aug)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}