{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport os\nfrom tensorflow.keras import layers\nimport numpy as np\nimport pandas as pd\n%matplotlib inline\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport matplotlib.dates as mdates\nfrom datetime import datetime\nfrom time import time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class SensorActions:\n    def __init__(self):\n        self.columns = ['site_id', 'floor_id', 'floor', 'x_waypoint', 'y_waypoint', \n                        'timestamp', 'type', 'x', 'y' , 'z' ]\n        self.extra_columns = ['xbias', 'ybias', 'zbias', 'accuracy']\n        self.sensor_cols = ['x','y', 'z']\n        self.sensor_extra_cols = self.extra_columns\n        self.num_cols = 10\n        self.max_slen = 0\n        self.df = None\n    \n    def set_max_len(self, data):\n        self.max_slen = max(map(len, data))\n    \n    def convert_to_df(self, data):\n        if self.max_slen > self.num_cols:\n            self.columns += self.extra_columns\n        self.df = pd.DataFrame(data, columns=self.columns)\n        if self.max_slen > self.num_cols:  # if uncalibrated sensor include xbias , ybias, zbias, accuracy\n            #rectify unknown column\n            rectify_col = ['TYPE_ACCELEROMETER', 'TYPE_MAGNETIC_FIELD', 'TYPE_GYROSCOPE', 'TYPE_ROTATION_VECTOR']\n            self.df.loc[self.df['type'].isin(rectify_col),'accuracy'] = self.df.loc[self.df['type'].isin(rectify_col),'xbias']\n            self.df.loc[self.df['type'].isin(rectify_col),'xbias'] = np.nan\n            self.df.fillna(value=np.nan, inplace=True)\n        self.df['timestamp'] = pd.to_datetime(self.df['timestamp'].astype('int64'), unit='ms')\n        self.df.set_index(['site_id', 'floor_id', 'timestamp', 'floor', 'x_waypoint', 'y_waypoint'], inplace=True)  \n        #sort by timestamp and site_id and floor\n        self.df.sort_index(ascending=True, inplace=True)\n    \n    def sensor_data_by_ts(self):\n        if self.max_slen > self.num_cols:  # if uncalibrated sensor include xbias , ybias, zbias\n            self.sensor_cols += self.sensor_extra_cols\n        self.df = self.df.pivot_table(values=self.sensor_cols, index=self.df.index.values,\n                                  columns='type', aggfunc='first').reset_index()\n        self.df.columns = [s1 + '_' + s2  for (s1, s2) in self.df.columns.tolist()]\n        self.df[['site_id', 'floor_id', 'timestamp','floor', 'x_waypoint', 'y_waypoint']] = pd.DataFrame(self.df.index_.tolist(), index=self.df.index)\n        self.df.set_index(['site_id','timestamp'], inplace=True)\n        self.df.drop(columns='index_', inplace=True)\n        return self.df\n    \n    def convert_to_numeric(self):\n        #convert object dtype to float64 for processing\n        for col in self.df.columns:\n            if col not in ['floor', 'floor_id']:\n                self.df[col] = pd.to_numeric(self.df[col], errors='coerce')\n        return self.df\n    \n    def store_data(self, filename):\n        self.df.to_csv(os.path.join(BUFFER_DIR, 'sensor_{}.csv'.format(filename)))\n        \nclass WIFIActions():\n    def __init__(self):\n        self.df = None\n        self.cols = ['site_id', 'floor_id','floor',\n                     'waypoint_x', 'waypoint_y', 'timestamp', 'type',\n                     'ssid', 'pssid', 'rssi', 'network_type', 'last_seen']\n    def convert_to_df(self, data):\n        self.df = pd.DataFrame(data, columns=self.cols)\n\n    def store_data(self, filename):\n        self.df.to_csv(os.path.join(BUFFER_DIR, 'wifi_{}.csv'.format(filename)))\n\nclass BEACONActions():\n    def __init__(self):\n        self.df = None\n        self.cols = ['site_id', 'floor_id','floor', 'waypoint_x', 'waypoint_y', 'timestamp',\n                      'type', 'uuid', 'minor_id' , 'major_id' ,'broadcasting_power', 'rssi',\n                     'distance', 'mac_id','loopkup_timestamp']\n    def convert_to_df(self, data):\n        self.df = pd.DataFrame(data, columns=self.cols)\n    def store_data(self, filename):\n        self.df.to_csv(os.path.join(BUFFER_DIR, 'beacon_{}.csv'.format(filename)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"SOURCE_DIR = '/kaggle/input/indoor-location-navigation'\nBUFFER_DIR = '/kaggle/working'\nTRAIN = 'train'\nMETADATA = 'metadata'\nMAX_FILES = 10\nANALYSIS_MODE = False\nSENSOR_DATA_MODE = True\nWIFI_DATA_MODE = True\nBEACON_DATA_MODE = True\nSITE_MAP_MODE = False\n\nimport re\n\nfloor_regex = re.compile('#\\s+SiteID:(\\w+)\\s+SiteName:(.+)\\s+FloorId:(\\w+)\\s+FloorName:(.+)')\ndevice_regex = re.compile('#\\s+Brand:(\\w+)\\s+Model:(.+)\\s+AndroidName:([\\d\\.]+)\\s+APILevel:(\\d+)')\nhardware_regex = re.compile('#\\s+type:(\\d+)\\s+name:(.*)\\s+version:(\\w+)\\s+vendor:(\\w+)\\s+resolution:([\\w.]+)\\s+power:([\\d.]+)\\s+maximumRange:([\\d.]+)')\nsensor_type = ['TYPE_ACCELEROMETER', 'TYPE_MAGNETIC_FIELD', 'TYPE_GYROSCOPE', 'TYPE_ROTATION_VECTOR',\n              'TYPE_MAGNETIC_FIELD_UNCALIBRATED', 'TYPE_GYROSCOPE_UNCALIBRATED', 'TYPE_ACCELEROMETER_UNCALIBRATED']\nsensor_nav_regex = re.compile('|'.join(sensor_type))\nsensor_data = []\nbeacon_data = []\nwifi_data = []\n\n#from files we can assume multiple sensor reading after a waypoint are associated with waypoint\n# add waypoint co-ordinates and floor as per time interval\n# so the user arrives at a waypoint and then takes sensor reading\n\nnum_floor = 0\nnum_file = 0\nsite_map = []\nBACTH_LIMIT = 500 # files\nSAMPLE_SITE_ID = '5cd56b5ae2acfd2d33b58546'\n\ndef process_sensor_data(data, site_id):\n    sa = SensorActions()\n    sa.set_max_len(data)\n    sa.convert_to_df(data)\n    sa.sensor_data_by_ts()\n    sa.convert_to_numeric()\n    sa.store_data(site_id)\n    del sa # free memory\n\ndef process_wifi_data(data, site_id):\n    wa = WIFIActions()\n    wa.convert_to_df(data)\n    wa.store_data(site_id)\n    del wa\n\ndef process_beacon_data(data, site_id):\n    ba = BEACONActions()\n    ba.convert_to_df(data)\n    ba.store_data(site_id)\n    del ba\n\nfor (root,dirs,files) in os.walk(os.path.join(*[SOURCE_DIR, TRAIN, SAMPLE_SITE_ID]), topdown=True):\n    if files:\n        floor_level = root.split('/')[-1] # this is floor level F1, F2, B1\n        for detailf in files:\n            num_file = num_file + 1\n            if num_file % 1000 == 0:# memory limit\n                process_sensor_data(sensor_data, SAMPLE_SITE_ID or site_id)\n                sensor_data = [] #free memory\n                process_wifi_data(wifi_data, SAMPLE_SITE_ID or site_id)\n                wifi_data = [] #free memory\n                process_beacon_data(beacon_data, SAMPLE_SITE_ID or site_id)\n                beacon_data = [] #free memory\n                print('processed {} files'.format(num_file))\n\n            if num_file % BACTH_LIMIT == 0 and not ANALYSIS_MODE:\n                print('storing data...')\n                process_sensor_data()\n\n            if num_file == MAX_FILES and ANALYSIS_MODE and not SITE_MAP_MODE: # stop while analysis\n                continue\n            with open(os.path.join(root, detailf), 'r') as f:\n                line_no = 0\n                waypoint_x = None # make waypoint varaible global for the file\n                waypoint_y = None\n                for line in f:\n                    line_no = line_no + 1\n                    if line.startswith('#'):\n                        if 'Time' in line or 'Version' in line:\n                            continue # skip start and end timestamp\n                        elif 'FloorName' in line:\n                            floor_details = floor_regex.search(line)\n                            if floor_details:\n                                site_id = floor_details.group(1)\n                                site_name = floor_details.group(2)\n                                floor_id = floor_details.group(3)\n                                floor_name = floor_details.group(4)\n                                if SITE_MAP_MODE:\n                                    site_map.append([site_id, floor_id, floor_level])\n                                    break\n                        elif 'Brand' in line:\n                            device_details = device_regex.search(line)\n                            if device_details:\n                                device_brand = device_details.group(1)\n                                device_model = device_details.group(2)\n                                andriod_ver = device_details.group(3)\n                                device_api = device_details.group(4)\n                        else:\n                            hardware_details = hardware_regex.search(line)\n                            if hardware_details:\n                                hardware_type = hardware_details.group(1)\n                                hardware_name = hardware_details.group(2)\n                                hardware_version = hardware_details.group(3)\n                                hardware_vendor = hardware_details.group(4)\n                                hardware_resolution = hardware_details.group(5)\n                                hardware_power = hardware_details.group(6)\n                                hardware_max_range = hardware_details.group(7)\n                    elif 'TYPE_WAYPOINT' in line:\n                            waypoint = line.replace('\\n', '').split('\\t')\n                            if len(waypoint) == 4:\n                                waypoint_x = waypoint[2]\n                                waypoint_y = waypoint[3]\n                    elif sensor_nav_regex.search(line) and SENSOR_DATA_MODE:\n                        #skip records where '\\n' character is missing leading to multiple records in same line\n                        if len(line.strip().split('\\t')) > 9:\n                            continue\n                        # timestamp, type, x, y , z ,xbias, ybias, zbias, accuracy\n                        sensor_data.append([site_id, floor_id, floor_level]+ [waypoint_x, waypoint_y] + line.strip().split('\\t'))\n                    elif 'TYPE_BEACON' in line and BEACON_DATA_MODE:\n                        #handle 1 beacon and wifi data in one line\n                        if len(re.findall('TYPE_BEACON', line)) == 1 and 'TYPE_WIFI' in line:\n                            beacon_line, wifi_line = line.strip().split('TYPE_WIFI')\n                            beacon_line, wifi_line = beacon_line.strip(), wifi_line.strip()\n                            # restructure becon line\n                            loopkup_timestamp = wifi_line.split('\\t')[-1]\n                            beacon_data.append([site_id, floor_id, floor_level]+ [waypoint_x, waypoint_y] + beacon_line.split('\\t') + [loopkup_timestamp])\n                            # restructure wifi line\n                            timestamp = beacon_line.replace('\\n', '').split('\\t')[0]\n                            wifi_data.append([site_id, floor_id, floor_level] + [waypoint_x, waypoint_y] + [timestamp] + wifi_line.split('\\t')[:len(wifi_line.split('\\t')) - 1])\n\n                        elif len(re.findall('TYPE_BEACON', line)) == 1 and 'TYPE_WIFI' not in line:\n                            # timestamp, type, uuid, minor_id , major_id , broadcasting_power, rssi, distance, mac_id, loopkup_timestamp\n                            beacon_data.append([site_id, floor_id, floor_level]+ [waypoint_x, waypoint_y] + line.strip().split('\\t'))\n                        else:\n                            # TODO handle multiple becon data with multiple wifi data in one line\n                            pass\n                    elif 'TYPE_WIFI' in line and WIFI_DATA_MODE:\n                        if len(re.findall('TYPE_WIFI', line)) == 1:\n                            # timestamp, type, wifi1, wifi2, latitute, longitude\n                            wifi_data.append([site_id, floor_id, floor_level] + [waypoint_x, waypoint_y]+ line.strip().split('\\t'))\n\n#save remainder data\nif sensor_data and not ANALYSIS_MODE:\n    process_sensor_data(sensor_data, SAMPLE_SITE_ID or site_id)\n    sensor_data = [] #free memory\n    process_wifi_data(wifi_data, SAMPLE_SITE_ID or site_id)\n    wifi_data = [] #free memory\n    process_beacon_data(beacon_data, SAMPLE_SITE_ID or site_id)\n    beacon_data = [] #free memory","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sensor_df = pd.read_csv(os.path.join(BUFFER_DIR, 'sensor_{}.csv'.format(SAMPLE_SITE_ID)))\nsensor_df['timestamp'] = pd.to_datetime(sensor_df['timestamp'])\nsensor_df.set_index(['site_id', 'timestamp'], inplace=True)\nsensor_df.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"wifi_df = pd.read_csv(os.path.join(BUFFER_DIR, 'wifi_{}.csv'.format(SAMPLE_SITE_ID)))\nwifi_df.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"beacon_df = pd.read_csv(os.path.join(BUFFER_DIR, 'beacon_{}.csv'.format(SAMPLE_SITE_ID)))\nbeacon_df.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sensor_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if SITE_MAP_MODE:\n    site_map_df = pd.DataFrame(site_map, columns=['site_id', 'floor_id', 'floor'])\n    site_map_df.info()\n    site_map_df.groupby('site_id').count()\n#!os.remove(\"/kaggle/working/indoor-location-nav_5cd56b5ae2acfd2d33b58546.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#there are Three predictions to be made:\n#1) floor level (B1, F1) \n#2) site_path (floor_id + site_id) where site_id is provided in test files\n#3) waypoints\n# There are three dataset given\n#1) sensor_dataset\n#2) wifi_dataset\n#3) becon_dataset\n# I will be using sensor_dataset for waypoints\n# wifi_dataset for site_id\n# becon_dataset for floor level\n\n#analysing sensor_dataset for waypoints\nprint(sensor_df['floor'][0])\nsample_floor_id = sensor_df['floor_id'][0]\nall_record_axis = ['x', 'y', 'z']\nsensors = ['TYPE_GYROSCOPE', 'TYPE_ACCELEROMETER', 'TYPE_MAGNETIC_FIELD', 'TYPE_ROTATION_VECTOR', 'waypoint']\nfig, ax  = plt.subplots(len(sensors),1,figsize=(15,15), sharex=True)\ncolor = ['green', 'blue', 'red']\nfor s_no, s in enumerate(sensors):\n    for a_no, a in enumerate(all_record_axis):\n        if s == 'waypoint' and a == 'z':\n            continue\n        rolmean = sensor_df[sensor_df['floor_id'] == sample_floor_id]['{}_{}'.format(a, s)].resample('1000ms', level='timestamp').mean() #every second data\n        ax[s_no].plot(rolmean, color=color[a_no])\n    ax[s_no].set_xlabel('Timestamp')\n    ax[s_no].set_ylabel(s) \n    \n\n    locator = mdates.MinuteLocator(interval=2)\n    ax[s_no].xaxis.set_major_locator(locator)\n    formatter = mdates.DateFormatter('%H:%M:%S')\n    ax[s_no].xaxis.set_major_formatter(formatter)\n\nplt.xticks(rotation=70)\nplt.grid(True)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# as analysing only one site drop site id from index\nsensor_df.reset_index(level=0, drop=True, inplace=True)\nsensor_df['timestamp'] = sensor_df.index\nsensor_df.sort_index(inplace=True)\nfloor_df = sensor_df.groupby('floor').agg({'timestamp': ['first','last']})\nfloor_df = floor_df.reset_index()\nfloor_df.columns = ['floor', 'entry', 'exit']\nfloor_df['total_time_spent'] = floor_df['exit'] - floor_df['entry']\nfloor_df.sort_values(by='entry')\n# customer navigated from floor F3 -> F4 -> F2 -> F1-> B1\n# customer spend most time on floor F1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"wp_df = sensor_df.groupby(['floor', 'x_waypoint', 'y_waypoint']).size().reset_index(name='counts')\nprint('there are total {} waypoints in site {}'.format(wp_df.shape[0], SAMPLE_SITE_ID))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"floor_wp_map = {}\nfor floor in sensor_df['floor'].unique():\n    wp_f_df = sensor_df[sensor_df['floor'] == floor].groupby(['x_waypoint', 'y_waypoint']).size().reset_index(name='counts')\n    floor_wp_map[floor] = wp_f_df.shape[0]\n    print('floor {} has {} waypoints'.format(floor, wp_f_df.shape[0]))\n    \nfig, ax = plt.subplots(figsize=(10, 10))\nax.bar(floor_wp_map.keys(), floor_wp_map.values())\nax.set_xlabel('floor')\nax.set_ylabel('number of waypoints')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#sensor records per waypoint\nfig, axs = plt.subplots(5, 1, figsize=(10, 20))\n\nfor i, floor in enumerate(sensor_df['floor'].unique()):\n    wp_f_df = sensor_df[sensor_df['floor'] == floor].groupby(['x_waypoint', 'y_waypoint']).size().reset_index(name='sensor_record_counts')\n    wp_f_df['waypoint'] = wp_f_df['x_waypoint'].astype(str) + ',' + wp_f_df['y_waypoint'].astype(str)\n    wp_f_df.plot(x='waypoint', y='sensor_record_counts', ax=axs[i], kind='bar', rot=75)\n\nfig.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#analysis of single waypoint\nsample_x_waypoint = sensor_df['x_waypoint'][1]\nsample_y_waypoint = sensor_df['y_waypoint'][1]\n\nsample_waypoint_df = sensor_df[(sensor_df['floor_id'] == sample_floor_id) &\n          np.isclose(sensor_df['x_waypoint'], sample_x_waypoint) &\n          np.isclose(sensor_df['y_waypoint'], sample_y_waypoint)]\n\nsample_waypoint_df.reset_index(level=0, drop=True, inplace=True)\n\nfig, ax  = plt.subplots(len(sensors),1,figsize=(15,15))\ncolor = ['green', 'blue', 'red']\nfor s_no, s in enumerate(sensors):\n    for a_no, a in enumerate(all_record_axis):\n        if s == 'waypoint' and a == 'z':\n            continue\n        ax[s_no].plot(sample_waypoint_df['{}_{}'.format(a, s)], color=color[a_no])\n    ax[s_no].set_xlabel('Seconds')\n    ax[s_no].set_ylabel(s) \n    \n\n    locator = mdates.SecondLocator(interval=1)\n    ax[s_no].xaxis.set_major_locator(locator)\n    formatter = mdates.DateFormatter('%S')\n    ax[s_no].xaxis.set_major_formatter(formatter)\n    \nplt.grid(True)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#analysis of customer waypoint on floor F1 image\nfor (root,dirs,files) in os.walk(os.path.join(*[SOURCE_DIR, METADATA, SAMPLE_SITE_ID, 'F1']), topdown=True):\n    print(files)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}