{"cells":[{"metadata":{},"cell_type":"markdown","source":"This kernel is introducing feature store for Indoor Location & Navigation competition, because the dataset of this competition is much comlicated and raw data. So it is useful for easy to retrieve data.\n\nThis script is not finalized. It will be updated for retrieve much more information."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import json\nimport re\nimport gc\nimport pickle\nimport itertools\nimport pandas as pd\nimport numpy as np\nfrom glob import glob\nfrom datetime import datetime as dt\nfrom pathlib import Path\nfrom tqdm import tqdm\nimport datetime\nts_conv = np.vectorize(datetime.datetime.fromtimestamp) # ut(10 digit) -> date\n\n# pandas settings -----------------------------------------\npd.set_option(\"display.max_colwidth\", 100)\npd.set_option(\"display.max_rows\", None)\npd.set_option(\"display.max_columns\", None)\npd.options.display.float_format = '{:,.5f}'.format\n\n# Graph drawing -------------------------------------------\nimport matplotlib\nfrom matplotlib import font_manager\nimport matplotlib.pyplot as plt\nimport matplotlib.cm as cm\nfrom matplotlib import rc\nfrom matplotlib_venn import venn2, venn2_circles\nfrom matplotlib import animation as ani\nfrom IPython.display import Image\nfrom pylab import imread\n\nplt.rcParams[\"patch.force_edgecolor\"] = True\nfrom IPython.display import display # Allows the use of display() for DataFrames\nimport seaborn as sns\nsns.set(style=\"whitegrid\", palette=\"muted\", color_codes=True)\nsns.set_style(\"whitegrid\", {'grid.linestyle': '--'})\nred = sns.xkcd_rgb[\"light red\"]\ngreen = sns.xkcd_rgb[\"medium green\"]\nblue = sns.xkcd_rgb[\"denim blue\"]\n\n%matplotlib inline\n%config InlineBackend.figure_format='retina'\n\n# ML -------------------------------------------\nfrom sklearn.preprocessing import LabelEncoder","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def unpickle(filename):\n    with open(filename, 'rb') as fo:\n        p = pickle.load(fo)\n    return p\n\ndef to_pickle(filename, obj):\n    with open(filename, 'wb') as f:\n        pickle.dump(obj, f, -1)\n\n\n\nclass FeatureStore():\n    \n    # necessayr to re-check\n    floor_convert = {'1F' :  0, '2F' : 1, '3F' : 2, '4F' : 3, '5F' : 4, \n                     '6F' : 5, '7F' : 6, '8F' : 7, '9F' : 8,\n                     'B'  : -1, 'B1' : -1, 'B2' : -2, 'B3' : -3, \n                     'BF' : -1, 'BM' : -1, \n                     'F1' : 0, 'F2' : 1, 'F3' : 2, 'F4' : 3, 'F5' : 4, \n                     'F6' : 5, 'F7' : 6, 'F8' : 7, 'F9' : 8, 'F10': 9,\n                     'L1' : 0, 'L2' : 1, 'L3' : 2, 'L4' : 3, 'L5' : 4, \n                     'L6' : 5, 'L7' : 6, 'L8' : 7, 'L9' : 8, 'L10': 9, \n                     'L11': 10,\n                     'G'  : 0, 'LG1': 0, 'LG2': 1, 'LM' : 0, 'M'  : 0, \n                     'P1' : 0, 'P2' : 1,}\n    \n    df_types = ['accelerometer',\n                'accelerometer_uncalibrated',\n                'beacon',\n                'gyroscope',\n                'gyroscope_uncalibrated',\n                'magnetic_field',\n                'magnetic_field_uncalibrated',\n                'rotation_vector',\n                'waypoint',\n                'wifi']\n    \n    # https://github.com/location-competition/indoor-location-competition-20\n    df_type_cols = {'accelerometer': [\"timestamp\", \"x\", \"y\", \"z\", \"accuracy\"],\n                'accelerometer_uncalibrated': [\"timestamp\", \"x\", \"y\", \"z\", \n                                               \"x2\", \"y2\", \"z2\", \"accuracy\" ],\n                'beacon': [\"timestamp\", \"uuid\", \"major_id\", \"minor_id\", \"tx_power\", \n                           \"rssi\", \"distance\", \"mac_addr\", \"timestamp2\"],\n                'gyroscope': [\"timestamp\", \"x\", \"y\", \"z\", \"accuracy\"],\n                'gyroscope_uncalibrated': [\"timestamp\", \"x\", \"y\", \"z\", \n                                           \"x2\", \"y2\", \"z2\", \"accuracy\" ],\n                'magnetic_field': [\"timestamp\", \"x\", \"y\", \"z\", \"accuracy\"],\n                'magnetic_field_uncalibrated': [\"timestamp\", \"x\", \"y\", \"z\", \n                                                \"x2\", \"y2\", \"z2\", \"accuracy\" ],\n                'rotation_vector': [\"timestamp\", \"x\", \"y\", \"z\", \"accuracy\"],\n                'waypoint': [\"timestamp\", \"x\", \"y\"],\n                'wifi': [\"timestamp\", \"ssid\", \"bssid\",\"rssi\",\"frequency\",\n                         \"last_seen_timestamp\",]}\n\n    dtype_dict = {}\n    dtype_dict[\"accelerometer\"] = {\"timestamp\":int, \"x\":float, \"y\":float, \"z\":float, \n                                   \"accuracy\":int}\n    dtype_dict[\"accelerometer_uncalibrated\"] = {\"timestamp\":int, \"x\":float, \"y\":float, \n                                                \"z\":float, \"x2\":float, \"y2\":float, \n                                                \"z2\":float, \"accuracy\":int}\n    dtype_dict[\"beacon\"] = {\"timestamp\":int, \"uuid\":str, \"major_id\":str, \n                            \"minor_id\":str, \"tx_power\":int,  \"rssi\":int, \n                            \"distance\":float, \"mac_addr\":str, \"timestamp2\":int}\n    dtype_dict[\"gyroscope\"] = {\"timestamp\":int, \"x\":float, \"y\":float, \"z\":float, \n                               \"accuracy\":int}\n    dtype_dict[\"gyroscope_uncalibrated\"] = {\"timestamp\":int, \"x\":float, \"y\":float, \n                                            \"z\":float, \"x2\":float, \"y2\":float, \n                                            \"z2\":float, \"accuracy\":int}\n    dtype_dict[\"magnetic_field\"] = {\"timestamp\":int, \"x\":float, \"y\":float, \n                                    \"z\":float, \"accuracy\":int}\n    dtype_dict[\"magnetic_field_uncalibrated\"] = {\"timestamp\":int, \"x\":float, \n                                                 \"y\":float, \"z\":float, \"x2\":float, \n                                                 \"y2\":float, \"z2\":float, \"accuracy\":int}\n    dtype_dict[\"rotation_vector\"] = {\"timestamp\":int, \"x\":float, \"y\":float, \n                                     \"z\":float, \"accuracy\":int}\n    dtype_dict[\"waypoint\"] = {\"timestamp\":int, \"x\":float, \"y\":float, \"z\":float}\n    dtype_dict[\"wifi\"] = {\"timestamp\":int, \"ssid\":str, \"bssid\":str,\n                          \"rssi\":int,\"frequency\":int, \"last_seen_timestamp\":int}\n\n    def __init__(self, site_id, floor, path_id, \n                 input_path=\"../input/indoor-location-navigation/\",\n                 save_path=\"../mid\"):\n        self.site_id = site_id.strip()\n        self.floor = floor.strip()\n        self.n_floor = self.floor_convert[self.floor]\n        self.path_id = path_id.strip()\n        \n        self.input_path = input_path\n        assert Path(input_path).exists(), f\"input_path do not exist: {input_path}\"\n        \n        self.save_path = save_path\n        Path(save_path).mkdir(parents=True, exist_ok=True)\n        \n        self.site_info = SiteInfo(site_id=self.site_id, floor=self.floor, input_path=self.input_path)\n        \n    def _flatten(self, l):\n        return list(itertools.chain.from_iterable(l))\n    \n    def multi_line_spliter(self, s):\n        matches = re.finditer(\"TYPE_\", s)\n        matches_positions = [match.start() for match in matches]\n        split_idx = [0] + [matches_positions[i]-14 for i in range(1, len(matches_positions))] + [len(s)]\n        return [s[split_idx[i]:split_idx[i+1]] for i in range(len(split_idx)-1)]\n    \n    def load_df(self, ):\n        path = str(Path(self.input_path)/f\"train/{self.site_id}/{self.floor}/{self.path_id}.txt\")\n        with open(path) as f:\n            data = f.readlines()\n        \n        modified_data = []\n        for s in data:\n            if s.count(\"TYPE_\")>1:\n                lines = self.multi_line_spliter(s)\n                modified_data.extend(lines)\n            else:\n                modified_data.append(s)\n        del data\n        self.meta_info_len = len([d for d in modified_data if d[0]==\"#\"])\n        self.meta_info_df = pd.DataFrame([m.replace(\"\\n\", \"\").split(\":\") \n                                          for m in self._flatten([d.split(\"\\t\") \n                                                                  for d in modified_data if d[0]==\"#\"]) if m!=\"#\"])\n\n        data_df = pd.DataFrame([d.replace(\"\\n\", \"\").split(\"\\t\") for d in modified_data if d[0]!=\"#\"])\n        for dt in self.df_types:\n            # select data type\n            df_s = data_df[data_df[1]==f\"TYPE_{dt.upper()}\"]\n            if len(df_s)==0:\n                setattr(self, dt, pd.DataFrame(columns=self.df_type_cols[dt]))\n            else:\n                # remove empty cols\n                na_info = df_s.isna().sum(axis=0) == len(df_s)\n                df_s = df_s[[i for i in na_info[na_info==False].index if i!=1]].reset_index(drop=True)\n                \n                if len(df_s.columns)!=len(self.df_type_cols[dt]):\n                    df_s.columns = self.df_type_cols[dt][:len(df_s.columns)]\n                else:\n                    df_s.columns = self.df_type_cols[dt]\n            \n                # set dtype          \n                for c in df_s.columns:\n                    df_s[c] = df_s[c].astype(self.dtype_dict[dt][c])\n                                     \n                # set DataFrame to attr\n                setattr(self, dt, df_s)\n    \n    def get_site_info(self, keep_raw=False):\n        self.site_info.get_site_info(keep_raw=keep_raw)\n            \n    def load_all_data(self, keep_raw=False):     \n        self.load_df()\n        self.get_site_info(keep_raw=keep_raw)\n        \n    def __getitem__(self, item):\n        if item in self.df_types:\n            return getattr(self, item)\n        else:\n            return None\n    \n    def save(self, ):\n        # to be implemented\n        pass\n    \n    \nclass SiteInfo():\n    def __init__(self, site_id, floor, input_path=\"../input/indoor-location-navigation/\"):\n        self.site_id = site_id\n        self.floor = floor\n        self.input_path = input_path\n        assert Path(input_path).exists(), f\"input_path do not exist: {input_path}\"\n        \n    def get_site_info(self, keep_raw=False):\n        floor_info_path = f\"{self.input_path}/metadata/{self.site_id}/{self.floor}/floor_info.json\"\n        with open(floor_info_path, \"r\") as f:\n            self.floor_info = json.loads(f.read())\n            self.site_height = self.floor_info[\"map_info\"][\"height\"]\n            self.site_width = self.floor_info[\"map_info\"][\"width\"]\n            if not keep_raw:\n                del self.floor_info\n            \n        geojson_map_path = f\"{self.input_path}/metadata/{self.site_id}/{self.floor}/geojson_map.json\"\n        with open(geojson_map_path, \"r\") as f:\n            self.geojson_map = json.loads(f.read())\n            self.map_type = self.geojson_map[\"type\"]\n            self.features = self.geojson_map[\"features\"]\n            \n            self.floor_coordinates = self.features[0][\"geometry\"][\"coordinates\"]\n            self.store_coordinates = [self.features[i][\"geometry\"][\"coordinates\"] \n                                          for i in range(1, len(self.features))]\n                \n            if not keep_raw:\n                del self.geojson_map\n    \n    def show_site_image(self):\n        path = f\"{self.input_path}/metadata/{self.site_id}/{self.floor}/floor_image.png\"\n        plt.imshow(imread(path), extent=[0, self.site_width, 0, self.site_height])\n\n    def draw_polygon(self, size=8, only_floor=False):\n\n        fig = plt.figure()\n        ax = plt.subplot(111)\n            \n        xmax, xmin, ymax, ymin = self._draw(self.floor_coordinates, ax, calc_minmax=True)\n        if not only_floor:\n            self._draw(self.store_coordinates, ax, fill=True)\n        plt.legend([])\n        \n        xrange = xmax - xmin\n        yrange = ymax - ymin\n        ratio = yrange / xrange\n        \n        self.x_size = size\n        self.y_size = size*ratio\n\n        fig.set_figwidth(size)\n        fig.set_figheight(size*ratio)\n        # plt.show()\n        return ax\n        \n    def _draw(self, coordinates, ax, fill=False, calc_minmax=False):\n        xmax, ymax = -np.inf, -np.inf\n        xmin, ymin = np.inf, np.inf\n        for i in range(len(coordinates)):\n            ndim = np.ndim(coordinates[i])\n            if ndim==2:\n                corrd_df = pd.DataFrame(coordinates[i])\n                if fill:\n                    ax.fill(corrd_df[0], corrd_df[1], alpha=0.7)\n                else:\n                    corrd_df.plot.line(x=0, y=1, style=\"-\", ax=ax)\n                        \n                if calc_minmax:\n                    xmax = max(xmax, corrd_df[0].max())\n                    xmin = min(xmin, corrd_df[0].min())\n\n                    ymax = max(ymax, corrd_df[1].max())\n                    ymin = min(ymin, corrd_df[1].min())\n            elif ndim==3:\n                for j in range(len(coordinates[i])):\n                    corrd_df = pd.DataFrame(coordinates[i][j])\n                    if fill:\n                        ax.fill(corrd_df[0], corrd_df[1], alpha=0.6)\n                    else:\n                        corrd_df.plot.line(x=0, y=1, style=\"-\", ax=ax)\n                        \n                    if calc_minmax:\n                        xmax = max(xmax, corrd_df[0].max())\n                        xmin = min(xmin, corrd_df[0].min())\n\n                        ymax = max(ymax, corrd_df[1].max())\n                        ymin = min(ymin, corrd_df[1].min())\n            else:\n                assert False, f\"ndim of coordinates should be 2 or 3: {ndim}\"\n        if calc_minmax:\n            return xmax, xmin, ymax, ymin\n        else:\n            return None\n        ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# prepare meta information"},{"metadata":{"trusted":true},"cell_type":"code","source":"# site_meta_data\nsite_meta_data = pd.DataFrame([[p.split(\"/\")[-2], p.split(\"/\")[-1]] for p in glob(\"../input/indoor-location-navigation/metadata/**/*\")])\nsite_meta_data.columns = [\"site_id\", \"floor\"]\nsite_meta_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_meta_data\ntrain_meta = glob(\"../input/indoor-location-navigation/train/*/*/*\")\ntrain_meta_org = pd.DataFrame(train_meta)\ntrain_meta = train_meta_org[0].str.split(\"/\", expand=True)[[4, 5, 6]]\ntrain_meta.columns = [\"site_id\", \"floor\", \"path_id\"]\ntrain_meta[\"path_id\"] = train_meta[\"path_id\"].str.replace(\".txt\", \"\")\ntrain_meta[\"path\"] = train_meta_org[0]\ntrain_meta.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# train data"},{"metadata":{"trusted":true},"cell_type":"code","source":"t = train_meta.iloc[512]\nprint(f\"site_id: {t.site_id}, floor: {t.floor}, path_id: {t.path_id}\")\nfeature = FeatureStore(site_id=t.site_id, floor=t.floor, path_id=t.path_id)\nfeature.load_all_data()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# attributes of feature store\n[c for c in dir(feature) if c[0]!=\"_\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# attributes of site info\n[c for c in dir(feature.site_info) if c[0]!=\"_\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature.site_info.site_height, feature.site_info.site_width","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for d in feature.df_types:\n    print(d)\n    print(feature[d].info())\n    display(feature[d].head())\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature.meta_info_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10, 10*feature.site_info.site_height / feature.site_info.site_width))\nfeature.site_info.show_site_image()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature.site_info.draw_polygon()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature.site_info.floor_coordinates[0][:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature.site_info.store_coordinates[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# Plot waypoint for a site."},{"metadata":{"trusted":true},"cell_type":"code","source":"# targe site\nsite_id = \"5da138764db8ce0c98bcaa46\"\nfloor = \"F1\"\n\n# get all waypoint\nwaypoint_list = []\ntrain_meta_s = train_meta.query(f\"site_id=='{site_id}' and floor == '{floor}'\")\nfor i, r in tqdm(train_meta_s.iterrows(), total=len(train_meta_s)):\n    if \"feature\" in globals():\n        del feature\n        gc.collect()\n    feature = FeatureStore(site_id=r.site_id, floor=r.floor, path_id=r.path_id)\n    feature.load_all_data()\n    feature.waypoint[\"path_id\"] = r.path_id\n    waypoint_list.append(feature.waypoint.copy())\n\nwaypoint_all = pd.concat(waypoint_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10, 10*feature.site_info.site_height / feature.site_info.site_width))\nfeature.site_info.show_site_image()\n\nfor i, g in waypoint_all.groupby(\"path_id\"):\n    plt.plot(g.x, g.y, \"-o\", alpha=0.6,  zorder=100, )\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}