{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import json\nimport re\nimport gc\nimport pickle\nimport itertools\nimport pandas as pd\nimport numpy as np\nfrom glob import glob\nfrom datetime import datetime as dt\nfrom pathlib import Path\nfrom tqdm import tqdm\nimport datetime\nts_conv = np.vectorize(datetime.datetime.fromtimestamp) # ut(10 digit) -> date\n\n# pandas settings -----------------------------------------\npd.set_option(\"display.max_colwidth\", 100)\npd.set_option(\"display.max_rows\", None)\npd.set_option(\"display.max_columns\", None)\npd.options.display.float_format = '{:,.5f}'.format\n\n# Graph drawing -------------------------------------------\nimport matplotlib\nfrom matplotlib import font_manager\nimport matplotlib.pyplot as plt\nimport matplotlib.cm as cm\nfrom matplotlib import rc\nfrom matplotlib_venn import venn2, venn2_circles\nfrom matplotlib import animation as ani\nfrom IPython.display import Image\nfrom pylab import imread\nfrom IPython.display import HTML\n\nplt.rcParams[\"patch.force_edgecolor\"] = True\nfrom IPython.display import display # Allows the use of display() for DataFrames\nimport seaborn as sns\nsns.set(style=\"whitegrid\", palette=\"muted\", color_codes=True)\nsns.set_style(\"whitegrid\", {'grid.linestyle': '--'})\nred = sns.xkcd_rgb[\"light red\"]\ngreen = sns.xkcd_rgb[\"medium green\"]\nblue = sns.xkcd_rgb[\"denim blue\"]\n\n%matplotlib inline\n%config InlineBackend.figure_format='retina'\n\n# ML -------------------------------------------\nfrom sklearn.preprocessing import LabelEncoder\n\ndef unpickle(filename):\n    with open(filename, 'rb') as fo:\n        p = pickle.load(fo)\n    return p\n\ndef to_pickle(filename, obj):\n    with open(filename, 'wb') as f:\n        pickle.dump(obj, f, -1)\n\n\n\nclass FeatureStore():\n    \n    # necessayr to re-check\n    floor_convert = {'1F' :  0, '2F' : 1, '3F' : 2, '4F' : 3, '5F' : 4, \n                     '6F' : 5, '7F' : 6, '8F' : 7, '9F' : 8,\n                     'B'  : -1, 'B1' : -1, 'B2' : -2, 'B3' : -3, \n                     'BF' : -1, 'BM' : -1, \n                     'F1' : 0, 'F2' : 1, 'F3' : 2, 'F4' : 3, 'F5' : 4, \n                     'F6' : 5, 'F7' : 6, 'F8' : 7, 'F9' : 8, 'F10': 9,\n                     'L1' : 0, 'L2' : 1, 'L3' : 2, 'L4' : 3, 'L5' : 4, \n                     'L6' : 5, 'L7' : 6, 'L8' : 7, 'L9' : 8, 'L10': 9, \n                     'L11': 10,\n                     'G'  : 0, 'LG1': 0, 'LG2': 1, 'LM' : 0, 'M'  : 0, \n                     'P1' : 0, 'P2' : 1,}\n    \n    df_types = ['accelerometer',\n                'accelerometer_uncalibrated',\n                'beacon',\n                'gyroscope',\n                'gyroscope_uncalibrated',\n                'magnetic_field',\n                'magnetic_field_uncalibrated',\n                'rotation_vector',\n                'waypoint',\n                'wifi']\n    \n    # https://github.com/location-competition/indoor-location-competition-20\n    df_type_cols = {'accelerometer': [\"timestamp\", \"x\", \"y\", \"z\", \"accuracy\"],\n                'accelerometer_uncalibrated': [\"timestamp\", \"x\", \"y\", \"z\", \n                                               \"x2\", \"y2\", \"z2\", \"accuracy\" ],\n                'beacon': [\"timestamp\", \"uuid\", \"major_id\", \"minor_id\", \"tx_power\", \n                           \"rssi\", \"distance\", \"mac_addr\", \"timestamp2\"],\n                'gyroscope': [\"timestamp\", \"x\", \"y\", \"z\", \"accuracy\"],\n                'gyroscope_uncalibrated': [\"timestamp\", \"x\", \"y\", \"z\", \n                                           \"x2\", \"y2\", \"z2\", \"accuracy\" ],\n                'magnetic_field': [\"timestamp\", \"x\", \"y\", \"z\", \"accuracy\"],\n                'magnetic_field_uncalibrated': [\"timestamp\", \"x\", \"y\", \"z\", \n                                                \"x2\", \"y2\", \"z2\", \"accuracy\" ],\n                'rotation_vector': [\"timestamp\", \"x\", \"y\", \"z\", \"accuracy\"],\n                'waypoint': [\"timestamp\", \"x\", \"y\"],\n                'wifi': [\"timestamp\", \"ssid\", \"bssid\",\"rssi\",\"frequency\",\n                         \"last_seen_timestamp\",]}\n\n    dtype_dict = {}\n    dtype_dict[\"accelerometer\"] = {\"timestamp\":int, \"x\":float, \"y\":float, \"z\":float, \n                                   \"accuracy\":int}\n    dtype_dict[\"accelerometer_uncalibrated\"] = {\"timestamp\":int, \"x\":float, \"y\":float, \n                                                \"z\":float, \"x2\":float, \"y2\":float, \n                                                \"z2\":float, \"accuracy\":int}\n    dtype_dict[\"beacon\"] = {\"timestamp\":int, \"uuid\":str, \"major_id\":str, \n                            \"minor_id\":str, \"tx_power\":int,  \"rssi\":int, \n                            \"distance\":float, \"mac_addr\":str, \"timestamp2\":int}\n    dtype_dict[\"gyroscope\"] = {\"timestamp\":int, \"x\":float, \"y\":float, \"z\":float, \n                               \"accuracy\":int}\n    dtype_dict[\"gyroscope_uncalibrated\"] = {\"timestamp\":int, \"x\":float, \"y\":float, \n                                            \"z\":float, \"x2\":float, \"y2\":float, \n                                            \"z2\":float, \"accuracy\":int}\n    dtype_dict[\"magnetic_field\"] = {\"timestamp\":int, \"x\":float, \"y\":float, \n                                    \"z\":float, \"accuracy\":int}\n    dtype_dict[\"magnetic_field_uncalibrated\"] = {\"timestamp\":int, \"x\":float, \n                                                 \"y\":float, \"z\":float, \"x2\":float, \n                                                 \"y2\":float, \"z2\":float, \"accuracy\":int}\n    dtype_dict[\"rotation_vector\"] = {\"timestamp\":int, \"x\":float, \"y\":float, \n                                     \"z\":float, \"accuracy\":int}\n    dtype_dict[\"waypoint\"] = {\"timestamp\":int, \"x\":float, \"y\":float, \"z\":float}\n    dtype_dict[\"wifi\"] = {\"timestamp\":int, \"ssid\":str, \"bssid\":str,\n                          \"rssi\":int,\"frequency\":int, \"last_seen_timestamp\":int}\n\n    def __init__(self, site_id, floor, path_id, \n                 input_path=\"../input/indoor-location-navigation/\",\n                 save_path=\"../mid\"):\n        self.site_id = site_id.strip()\n        self.floor = floor.strip()\n        self.n_floor = self.floor_convert[self.floor]\n        self.path_id = path_id.strip()\n        \n        self.input_path = input_path\n        assert Path(input_path).exists(), f\"input_path do not exist: {input_path}\"\n        \n        self.save_path = save_path\n        Path(save_path).mkdir(parents=True, exist_ok=True)\n        \n        self.site_info = SiteInfo(site_id=self.site_id, floor=self.floor, input_path=self.input_path)\n        \n    def _flatten(self, l):\n        return list(itertools.chain.from_iterable(l))\n    \n    def multi_line_spliter(self, s):\n        matches = re.finditer(\"TYPE_\", s)\n        matches_positions = [match.start() for match in matches]\n        split_idx = [0] + [matches_positions[i]-14 for i in range(1, len(matches_positions))] + [len(s)]\n        return [s[split_idx[i]:split_idx[i+1]] for i in range(len(split_idx)-1)]\n    \n    def load_df(self, ):\n        path = str(Path(self.input_path)/f\"train/{self.site_id}/{self.floor}/{self.path_id}.txt\")\n        with open(path) as f:\n            data = f.readlines()\n        \n        modified_data = []\n        for s in data:\n            if s.count(\"TYPE_\")>1:\n                lines = self.multi_line_spliter(s)\n                modified_data.extend(lines)\n            else:\n                modified_data.append(s)\n        del data\n        self.meta_info_len = len([d for d in modified_data if d[0]==\"#\"])\n        self.meta_info_df = pd.DataFrame([m.replace(\"\\n\", \"\").split(\":\") \n                                          for m in self._flatten([d.split(\"\\t\") \n                                                                  for d in modified_data if d[0]==\"#\"]) if m!=\"#\"])\n\n        data_df = pd.DataFrame([d.replace(\"\\n\", \"\").split(\"\\t\") for d in modified_data if d[0]!=\"#\"])\n        for dt in self.df_types:\n            # select data type\n            df_s = data_df[data_df[1]==f\"TYPE_{dt.upper()}\"]\n            if len(df_s)==0:\n                setattr(self, dt, pd.DataFrame(columns=self.df_type_cols[dt]))\n            else:\n                # remove empty cols\n                na_info = df_s.isna().sum(axis=0) == len(df_s)\n                df_s = df_s[[i for i in na_info[na_info==False].index if i!=1]].reset_index(drop=True)\n                \n                if len(df_s.columns)!=len(self.df_type_cols[dt]):\n                    df_s.columns = self.df_type_cols[dt][:len(df_s.columns)]\n                else:\n                    df_s.columns = self.df_type_cols[dt]\n            \n                # set dtype          \n                for c in df_s.columns:\n                    df_s[c] = df_s[c].astype(self.dtype_dict[dt][c])\n                                     \n                # set DataFrame to attr\n                setattr(self, dt, df_s)\n    \n    def get_site_info(self, keep_raw=False):\n        self.site_info.get_site_info(keep_raw=keep_raw)\n            \n    def load_all_data(self, keep_raw=False):     \n        self.load_df()\n        self.get_site_info(keep_raw=keep_raw)\n        \n    def __getitem__(self, item):\n        if item in self.df_types:\n            return getattr(self, item)\n        elif item==\"sensors\":\n            try:\n                return getattr(self, \"sensor_df\")\n            except:\n                self.sensor_df = pd.concat([feature[\"magnetic_field\"].set_index(\"timestamp\"), \n                                       feature[\"accelerometer\"].set_index(\"timestamp\"), \n                                       feature[\"gyroscope\"].set_index(\"timestamp\")], axis=1)\n                if self.sensor_df.shape[1]==12:\n                    self.sensor_df.columns = [\"mag_x\", \"mag_y\", \"mag_z\", \"mag_acc\", \n                                              \"acc_x\", \"acc_y\", \"acc_z\", \"acc_acc\",\n                                              \"gyr_x\", \"gyr_y\", \"gyr_z\", \"gyr_acc\", ]\n                else:\n                    self.sensor_df.columns = [\"mag_x\", \"mag_y\", \"mag_z\", \n                                              \"acc_x\", \"acc_y\", \"acc_z\", \n                                              \"gyr_x\", \"gyr_y\", \"gyr_z\",  ]\n                return self.sensor_df\n        else:\n            return None\n    \n    def save(self, ):\n        # to be implemented\n        pass\n    \n    \nclass SiteInfo():\n    def __init__(self, site_id, floor, input_path=\"../input/indoor-location-navigation/\"):\n        self.site_id = site_id\n        self.floor = floor\n        self.input_path = input_path\n        assert Path(input_path).exists(), f\"input_path do not exist: {input_path}\"\n        \n    def get_site_info(self, keep_raw=False):\n        floor_info_path = f\"{self.input_path}/metadata/{self.site_id}/{self.floor}/floor_info.json\"\n        with open(floor_info_path, \"r\") as f:\n            self.floor_info = json.loads(f.read())\n            self.site_height = self.floor_info[\"map_info\"][\"height\"]\n            self.site_width = self.floor_info[\"map_info\"][\"width\"]\n            if not keep_raw:\n                del self.floor_info\n            \n        geojson_map_path = f\"{self.input_path}/metadata/{self.site_id}/{self.floor}/geojson_map.json\"\n        with open(geojson_map_path, \"r\") as f:\n            self.geojson_map = json.loads(f.read())\n            self.map_type = self.geojson_map[\"type\"]\n            self.features = self.geojson_map[\"features\"]\n            \n            self.floor_coordinates = self.features[0][\"geometry\"][\"coordinates\"]\n            self.store_coordinates = [self.features[i][\"geometry\"][\"coordinates\"] \n                                          for i in range(1, len(self.features))]\n                \n            if not keep_raw:\n                del self.geojson_map\n    \n    def show_site_image(self, ax):\n        path = f\"{self.input_path}/metadata/{self.site_id}/{self.floor}/floor_image.png\"\n        ax.imshow(imread(path), extent=[0, self.site_width, 0, self.site_height])\n\n    def draw_polygon(self, size=8, only_floor=False):\n\n        fig = plt.figure()\n        ax = plt.subplot(111)\n            \n        xmax, xmin, ymax, ymin = self._draw(self.floor_coordinates, ax, calc_minmax=True)\n        if not only_floor:\n            self._draw(self.store_coordinates, ax, fill=True)\n        plt.legend([])\n        \n        xrange = xmax - xmin\n        yrange = ymax - ymin\n        ratio = yrange / xrange\n        \n        self.x_size = size\n        self.y_size = size*ratio\n\n        fig.set_figwidth(size)\n        fig.set_figheight(size*ratio)\n        # plt.show()\n        return ax\n        \n    def _draw(self, coordinates, ax, fill=False, calc_minmax=False):\n        xmax, ymax = -np.inf, -np.inf\n        xmin, ymin = np.inf, np.inf\n        for i in range(len(coordinates)):\n            ndim = np.ndim(coordinates[i])\n            if ndim==2:\n                corrd_df = pd.DataFrame(coordinates[i])\n                if fill:\n                    ax.fill(corrd_df[0], corrd_df[1], alpha=0.7)\n                else:\n                    corrd_df.plot.line(x=0, y=1, style=\"-\", ax=ax)\n                        \n                if calc_minmax:\n                    xmax = max(xmax, corrd_df[0].max())\n                    xmin = min(xmin, corrd_df[0].min())\n\n                    ymax = max(ymax, corrd_df[1].max())\n                    ymin = min(ymin, corrd_df[1].min())\n            elif ndim==3:\n                for j in range(len(coordinates[i])):\n                    corrd_df = pd.DataFrame(coordinates[i][j])\n                    if fill:\n                        ax.fill(corrd_df[0], corrd_df[1], alpha=0.6)\n                    else:\n                        corrd_df.plot.line(x=0, y=1, style=\"-\", ax=ax)\n                        \n                    if calc_minmax:\n                        xmax = max(xmax, corrd_df[0].max())\n                        xmin = min(xmin, corrd_df[0].min())\n\n                        ymax = max(ymax, corrd_df[1].max())\n                        ymin = min(ymin, corrd_df[1].min())\n            else:\n                assert False, f\"ndim of coordinates should be 2 or 3: {ndim}\"\n        if calc_minmax:\n            return xmax, xmin, ymax, ymin\n        else:\n            return None\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from matplotlib import animation as ani\nimport functools\nfrom IPython.display import Image\nplt.rcParams[\"font.size\"] = 7\n\ndef animate(axes, nframe):\n    global num_frame\n    \n    ts = feature.waypoint.timestamp.values[nframe]\n    \n    # axes[0]------------------------------\n    wp =feature.waypoint.iloc[nframe]\n    axes[0].text(x=wp.x, y=wp.y, s=f\"{nframe}\", fontsize=6)\n    \n    # axes[1]------------------------------\n    axes[1].vlines(ts, -3.5, 3.5, \"k\", lw=1)\n    axes[1].text(ts, -1, f\"{nframe}\", fontsize=8)\n    \n    # axes[2]------------------------------\n    axes[2].vlines(ts, -40, 40, \"k\", lw=1)\n    axes[2].text(ts, -20, f\"{nframe}\", fontsize=8)\n\n\ndef drow_path_gyr_mag(feature, gif_name):    \n    ts_min = feature.waypoint.timestamp.min()\n    ts_max = feature.waypoint.timestamp.max()\n    query = f\"{ts_min} <= timestamp and timestamp <= {ts_max}\"\n    sensor_df = feature[\"sensors\"].reset_index().query(query)\n    sensor_df[\"ts_diff\"] = sensor_df.timestamp.diff() / 1000\n    sensor_df[\"x_pos\"] = 0\n    sensor_df[\"y_pos\"] = 0\n\n    sensor_df[\"gyr_z_dif_ts\"] = sensor_df[\"gyr_z\"] * sensor_df[\"ts_diff\"].fillna(0.02)\n    fig, axes = plt.subplots(3, 1,\n        gridspec_kw=dict(width_ratios=[1], \n                         height_ratios=[4,1,1], \n                         wspace=0.1, \n                         hspace=0.3),)\n    # axes[0]------------------------------\n    feature.site_info.show_site_image(ax=axes[0])\n    axes[0].grid(False)\n    axes[0].tick_params(labelsize=7)\n    # axes[1]------------------------------\n    gyro_data = sensor_df.set_index(\"timestamp\").gyr_z_dif_ts.cumsum()\n    gyro_data.plot(lw=1, label=\"gyr_z_dif_ts\", ax=axes[1])\n    axes[1].hlines(np.pi, ts_min, ts_max, \"gray\", linestyle=\"--\", lw=0.5)\n    axes[1].hlines(-np.pi, ts_min, ts_max, \"gray\", linestyle=\"--\", lw=0.5)\n    axes[1].legend(loc=\"best\", fontsize=8)\n    axes[1].set_title(\"gyr_z\", fontsize=8)\n    axes[1].set_xlabel('timestamp', fontsize=8)\n    axes[1].tick_params(labelsize=7)\n    # axes[2]------------------------------\n    sensor_df.set_index(\"timestamp\").mag_x.plot(lw=1, label=\"mag_x\", ax=axes[2])\n    sensor_df.set_index(\"timestamp\").mag_y.plot(lw=1, label=\"mag_y\", ax=axes[2])\n    axes[2].legend(loc=\"best\", fontsize=8)\n    axes[2].set_title(\"magnetic_field_x, y\", fontsize=8)\n    axes[2].set_xlabel('timestamp', fontsize=8)\n    axes[2].tick_params(labelsize=7)\n    \n    animate_ = functools.partial(animate, axes)\n    anim = ani.FuncAnimation(fig, animate_, frames=len(feature.waypoint))\n\n    fig.set_figwidth(8)\n    fig.set_figheight(12)\n    anim.save(gif_name, fps=2, dpi=256)\n    plt.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature = FeatureStore(site_id='5da138764db8ce0c98bcaa46', floor='F1', path_id=\"5daae24718410e00067e6d44\")\nfeature.load_all_data()\ndrow_path_gyr_mag(feature, 'path_anim01.gif')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Image('path_anim01.gif')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"![](./path_anim01.gif)"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature = FeatureStore(site_id='5cdbc652853bc856e89a8694', floor='F1', path_id=\"5d0c45df22c7770008a7580a\")\nfeature.load_all_data()\ndrow_path_gyr_mag(feature, 'path_anim02.gif')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Image('path_anim02.gif')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"![](./path_anim02.gif)"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature = FeatureStore(site_id='5cd56b5ae2acfd2d33b5854a', floor='F1', path_id=\"5d076ecf0e86b60008036124\")\nfeature.load_all_data()\ndrow_path_gyr_mag(feature, 'path_anim03.gif')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Image('path_anim03.gif')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"![](./path_anim03.gif)"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}