{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"
\n

Snap to Grid & Fix the timestamps - Part(3)

\n

Identify the position of a smartphone in a shopping mall

\n

By: Somayyeh Gholami & Mehran Kazeminia

\n
","metadata":{}},{"cell_type":"markdown","source":"
\n
","metadata":{}},{"cell_type":"markdown","source":"# Description:","metadata":{}},{"cell_type":"markdown","source":"### - In this notebook, we want to improve the score of our previous notebook (No. 2). We chose \"generated6\", which has a score of \"5.265\". The address of our previous notebook is as follows:\n\nhttps://www.kaggle.com/mehrankazeminia/2-3-indoor-navigation-comparative-method\n\n### - We have used the following notebook codes in this notebook. Thanks again for sharing this great notebook. \"Data Visualization\" is of particular importance in this challenge. Because the location of the corridors is important :)\n\nhttps://www.kaggle.com/robikscube/indoor-navigation-snap-to-grid-post-processing\n\n### - Next, we used the following excellent notebook for \"Fix the timestamps\". \n\nhttps://www.kaggle.com/tomooinubushi/postprocessing-based-on-leakage\n\n### =======================================================\n\n### For more information, you can refer to the following address:\n\nhttps://www.kaggle.com/c/indoor-location-navigation/discussion/230153\n\n## >>> Good Luck <<<\n\n","metadata":{}},{"cell_type":"markdown","source":"
\n
","metadata":{}},{"cell_type":"markdown","source":"# If you find this work useful, please don't forget upvoting :)","metadata":{}},{"cell_type":"markdown","source":"
\n
","metadata":{}},{"cell_type":"markdown","source":"# Import ","metadata":{}},{"cell_type":"code","source":"import json\nimport numpy as np\nimport pandas as pd \nimport matplotlib.pylab as plt\n\nfrom scipy.spatial.distance import cdist\n\n%matplotlib inline","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n
","metadata":{}},{"cell_type":"markdown","source":"# Helper Functions","metadata":{}},{"cell_type":"code","source":"def split_col(df):\n df = pd.concat([\n df['site_path_timestamp'].str.split('_', expand=True) \\\n .rename(columns={0:'site',\n 1:'path',\n 2:'timestamp'}),\n df\n ], axis=1).copy()\n return df\n\nfloor_map = {\"B2\":-2, \"B1\":-1, \"F1\":0, \"F2\": 1, \"F3\":2,\n \"F4\":3, \"F5\":4, \"F6\":5, \"F7\":6,\"F8\":7,\"F9\":8,\n \"1F\":0, \"2F\":1, \"3F\":2, \"4F\":3, \"5F\":4, \"6F\":5,\n \"7F\":6, \"8F\": 7, \"9F\":8}\n\n\ndef plot_preds(\n site,\n floorNo,\n sub=None,\n true_locs=None,\n base=\"../input/indoor-location-navigation\",\n show_train=True,\n show_preds=True,\n fix_labels=True,\n map_floor=None\n):\n \"\"\"\n Plots predictions on floorplan map.\n \n map_floor : use a different floor's map\n \"\"\"\n if map_floor is None:\n map_floor = floorNo\n # Prepare width_meter & height_meter (taken from the .json file)\n floor_plan_filename = f\"{base}/metadata/{site}/{map_floor}/floor_image.png\"\n json_plan_filename = f\"{base}/metadata/{site}/{map_floor}/floor_info.json\"\n with open(json_plan_filename) as json_file:\n json_data = json.load(json_file)\n\n width_meter = json_data[\"map_info\"][\"width\"]\n height_meter = json_data[\"map_info\"][\"height\"]\n\n floor_img = plt.imread(f\"{base}/metadata/{site}/{map_floor}/floor_image.png\")\n\n fig, ax = plt.subplots(figsize=(12, 12))\n plt.imshow(floor_img)\n\n if show_train:\n true_locs = true_locs.query('site == @site and floorNo == @map_floor').copy()\n true_locs[\"x_\"] = true_locs[\"x\"] * floor_img.shape[0] / height_meter\n true_locs[\"y_\"] = (\n true_locs[\"y\"] * -1 * floor_img.shape[1] / width_meter\n ) + floor_img.shape[0]\n true_locs.query(\"site == @site and floorNo == @map_floor\").groupby(\"path\").plot(\n x=\"x_\",\n y=\"y_\",\n style=\"+\",\n ax=ax,\n label=\"train waypoint location\",\n color=\"grey\",\n alpha=0.5,\n )\n\n if show_preds:\n sub = sub.query('site == @site and floorNo == @floorNo').copy()\n sub[\"x_\"] = sub[\"x\"] * floor_img.shape[0] / height_meter\n sub[\"y_\"] = (\n sub[\"y\"] * -1 * floor_img.shape[1] / width_meter\n ) + floor_img.shape[0]\n for path, path_data in sub.query(\n \"site == @site and floorNo == @floorNo\"\n ).groupby(\"path\"):\n path_data.plot(\n x=\"x_\",\n y=\"y_\",\n style=\".-\",\n ax=ax,\n title=f\"{site} - floor - {floorNo}\",\n alpha=1,\n label=path,\n )\n if fix_labels:\n handles, labels = ax.get_legend_handles_labels()\n by_label = dict(zip(labels, handles))\n plt.legend(\n by_label.values(), by_label.keys(), loc=\"center left\", bbox_to_anchor=(1, 0.5)\n )\n return fig, ax\n\ndef sub_process(sub, train_waypoints):\n train_waypoints['isTrainWaypoint'] = True\n sub = split_col(sub[['site_path_timestamp','floor','x','y']]).copy()\n sub = sub.merge(train_waypoints[['site','floorNo','floor']].drop_duplicates(), how='left')\n sub = sub.merge(\n train_waypoints[['x','y','site','floor','isTrainWaypoint']].drop_duplicates(),\n how='left',\n on=['site','x','y','floor']\n )\n sub['isTrainWaypoint'] = sub['isTrainWaypoint'].fillna(False)\n return sub.copy()\n","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n
","metadata":{}},{"cell_type":"markdown","source":"# Data Set","metadata":{}},{"cell_type":"code","source":"train_waypoints = pd.read_csv('../input/indoor-location-train-waypoints/train_waypoints.csv')\nsub = sub_process(pd.read_csv('../input/2-3-indoor-navigation-comparative-method/generated6.csv'),\n train_waypoints)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n
","metadata":{}},{"cell_type":"markdown","source":"# Find the closest \"grid\" point for each prediction.","metadata":{}},{"cell_type":"code","source":"def add_xy(df):\n df['xy'] = [(x, y) for x,y in zip(df['x'], df['y'])]\n return df\n\ndef closest_point(point, points):\n \"\"\" Find closest point from a list of points. \"\"\"\n return points[cdist([point], points).argmin()]\n\nsub = add_xy(sub)\ntrain_waypoints = add_xy(train_waypoints)\n\nds = []\nfor (site, myfloor), d in sub.groupby(['site','floor']):\n true_floor_locs = train_waypoints.loc[(train_waypoints['floor'] == myfloor) &\n (train_waypoints['site'] == site)] \\\n .reset_index(drop=True)\n if len(true_floor_locs) == 0:\n print(f'Skipping {site} {myfloor}')\n continue\n d['matched_point'] = [closest_point(x, list(true_floor_locs['xy'])) for x in d['xy']]\n d['x_'] = d['matched_point'].apply(lambda x: x[0])\n d['y_'] = d['matched_point'].apply(lambda x: x[1])\n ds.append(d)\n\nsub = pd.concat(ds)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n
","metadata":{}},{"cell_type":"markdown","source":"# Apply a Threshold and \"Snap to Grid\"","metadata":{}},{"cell_type":"code","source":"def snap_to_grid(sub, threshold):\n \"\"\"\n Snap to grid if within a threshold.\n \n x, y are the predicted points.\n x_, y_ are the closest grid points.\n _x_, _y_ are the new predictions after post processing.\n \"\"\"\n sub['_x_'] = sub['x']\n sub['_y_'] = sub['y']\n sub.loc[sub['dist'] < threshold, '_x_'] = sub.loc[sub['dist'] < threshold]['x_']\n sub.loc[sub['dist'] < threshold, '_y_'] = sub.loc[sub['dist'] < threshold]['y_']\n return sub.copy()\n\n# Calculate the distances\nsub['dist'] = np.sqrt( (sub.x-sub.x_)**2 + (sub.y-sub.y_)**2 )\n\nsub_pp = snap_to_grid(sub, threshold=7.55)\n\nsub_pp = sub_pp[['site_path_timestamp','floor','_x_','_y_','site','path','floorNo']] \\\n .rename(columns={'_x_':'x', '_y_':'y'})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n
","metadata":{}},{"cell_type":"markdown","source":"# Save Post Processed Submission.","metadata":{}},{"cell_type":"code","source":"sub_pp[['site_path_timestamp','floor','x','y']] \\\n .to_csv('submission_snap_to_grid.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_snap_to_grid = sub_pp[['site_path_timestamp','floor','x','y']]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n
","metadata":{}},{"cell_type":"markdown","source":"# Postprocessing with leaked feature","metadata":{}},{"cell_type":"code","source":"import json\nimport re\nimport gc\nimport pickle\nimport itertools\nimport pandas as pd\nimport numpy as np\nfrom glob import glob\nfrom datetime import datetime as dt\nfrom pathlib import Path\nfrom tqdm import tqdm\nimport datetime\nts_conv = np.vectorize(datetime.datetime.fromtimestamp) # ut(10 digit) -> date\n\n# pandas settings -----------------------------------------\npd.set_option(\"display.max_colwidth\", 100)\npd.set_option(\"display.max_rows\", None)\npd.set_option(\"display.max_columns\", None)\npd.options.display.float_format = '{:,.5f}'.format\n\n# Graph drawing -------------------------------------------\nimport matplotlib\nfrom matplotlib import font_manager\nimport matplotlib.pyplot as plt\nimport matplotlib.cm as cm\nfrom matplotlib import rc\nfrom matplotlib_venn import venn2, venn2_circles\nfrom matplotlib import animation as ani\nfrom IPython.display import Image\nfrom pylab import imread\n\nplt.rcParams[\"patch.force_edgecolor\"] = True\nfrom IPython.display import display # Allows the use of display() for DataFrames\nimport seaborn as sns\nsns.set(style=\"whitegrid\", palette=\"muted\", color_codes=True)\nsns.set_style(\"whitegrid\", {'grid.linestyle': '--'})\nred = sns.xkcd_rgb[\"light red\"]\ngreen = sns.xkcd_rgb[\"medium green\"]\nblue = sns.xkcd_rgb[\"denim blue\"]\n\n%matplotlib inline\n%config InlineBackend.figure_format='retina'\n\n# ML -------------------------------------------\nfrom sklearn.preprocessing import LabelEncoder\n\n\nimport dill\nfrom collections import defaultdict, OrderedDict\nfrom scipy.spatial import distance","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def unpickle(filename):\n with open(filename, 'rb') as fo:\n p = pickle.load(fo)\n return p\n\ndef to_pickle(filename, obj):\n with open(filename, 'wb') as f:\n pickle.dump(obj, f, -1)\n\n\n\nclass FeatureStore():\n \n # necessayr to re-check\n floor_convert = {'1F' : 0, '2F' : 1, '3F' : 2, '4F' : 3, '5F' : 4, \n '6F' : 5, '7F' : 6, '8F' : 7, '9F' : 8,\n 'B' : -1, 'B1' : -1, 'B2' : -2, 'B3' : -3, \n 'BF' : -1, 'BM' : -1, \n 'F1' : 0, 'F2' : 1, 'F3' : 2, 'F4' : 3, 'F5' : 4, \n 'F6' : 5, 'F7' : 6, 'F8' : 7, 'F9' : 8, 'F10': 9,\n 'L1' : 0, 'L2' : 1, 'L3' : 2, 'L4' : 3, 'L5' : 4, \n 'L6' : 5, 'L7' : 6, 'L8' : 7, 'L9' : 8, 'L10': 9, \n 'L11': 10,\n 'G' : 0, 'LG1': 0, 'LG2': 1, 'LM' : 0, 'M' : 0, \n 'P1' : 0, 'P2' : 1,}\n \n df_types = ['accelerometer',\n 'accelerometer_uncalibrated',\n 'beacon',\n 'gyroscope',\n 'gyroscope_uncalibrated',\n 'magnetic_field',\n 'magnetic_field_uncalibrated',\n 'rotation_vector',\n 'waypoint',\n 'wifi']\n \n # https://github.com/location-competition/indoor-location-competition-20\n df_type_cols = {'accelerometer': [\"timestamp\", \"x\", \"y\", \"z\", \"accuracy\"],\n 'accelerometer_uncalibrated': [\"timestamp\", \"x\", \"y\", \"z\", \n \"x2\", \"y2\", \"z2\", \"accuracy\" ],\n 'beacon': [\"timestamp\", \"uuid\", \"major_id\", \"minor_id\", \"tx_power\", \n \"rssi\", \"distance\", \"mac_addr\", \"timestamp2\"],\n 'gyroscope': [\"timestamp\", \"x\", \"y\", \"z\", \"accuracy\"],\n 'gyroscope_uncalibrated': [\"timestamp\", \"x\", \"y\", \"z\", \n \"x2\", \"y2\", \"z2\", \"accuracy\" ],\n 'magnetic_field': [\"timestamp\", \"x\", \"y\", \"z\", \"accuracy\"],\n 'magnetic_field_uncalibrated': [\"timestamp\", \"x\", \"y\", \"z\", \n \"x2\", \"y2\", \"z2\", \"accuracy\" ],\n 'rotation_vector': [\"timestamp\", \"x\", \"y\", \"z\", \"accuracy\"],\n 'waypoint': [\"timestamp\", \"x\", \"y\"],\n 'wifi': [\"timestamp\", \"ssid\", \"bssid\",\"rssi\",\"frequency\",\n \"last_seen_timestamp\",]}\n\n dtype_dict = {}\n dtype_dict[\"accelerometer\"] = {\"timestamp\":int, \"x\":float, \"y\":float, \"z\":float, \n \"accuracy\":int}\n dtype_dict[\"accelerometer_uncalibrated\"] = {\"timestamp\":int, \"x\":float, \"y\":float, \n \"z\":float, \"x2\":float, \"y2\":float, \n \"z2\":float, \"accuracy\":int}\n dtype_dict[\"beacon\"] = {\"timestamp\":int, \"uuid\":str, \"major_id\":str, \n \"minor_id\":str, \"tx_power\":int, \"rssi\":int, \n \"distance\":float, \"mac_addr\":str, \"timestamp2\":int}\n dtype_dict[\"gyroscope\"] = {\"timestamp\":int, \"x\":float, \"y\":float, \"z\":float, \n \"accuracy\":int}\n dtype_dict[\"gyroscope_uncalibrated\"] = {\"timestamp\":int, \"x\":float, \"y\":float, \n \"z\":float, \"x2\":float, \"y2\":float, \n \"z2\":float, \"accuracy\":int}\n dtype_dict[\"magnetic_field\"] = {\"timestamp\":int, \"x\":float, \"y\":float, \n \"z\":float, \"accuracy\":int}\n dtype_dict[\"magnetic_field_uncalibrated\"] = {\"timestamp\":int, \"x\":float, \n \"y\":float, \"z\":float, \"x2\":float, \n \"y2\":float, \"z2\":float, \"accuracy\":int}\n dtype_dict[\"rotation_vector\"] = {\"timestamp\":int, \"x\":float, \"y\":float, \n \"z\":float, \"accuracy\":int}\n dtype_dict[\"waypoint\"] = {\"timestamp\":int, \"x\":float, \"y\":float, \"z\":float}\n dtype_dict[\"wifi\"] = {\"timestamp\":int, \"ssid\":str, \"bssid\":str,\n \"rssi\":int,\"frequency\":int, \"last_seen_timestamp\":int}\n\n def __init__(self, site_id, floor, path_id, \n input_path=\"../input/indoor-location-navigation/\",\n save_path=\"../mid\"):\n self.site_id = site_id.strip()\n self.floor = floor.strip()\n self.n_floor = self.floor_convert[self.floor]\n self.path_id = path_id.strip()\n \n self.input_path = input_path\n assert Path(input_path).exists(), f\"input_path do not exist: {input_path}\"\n \n self.save_path = save_path\n Path(save_path).mkdir(parents=True, exist_ok=True)\n \n self.site_info = SiteInfo(site_id=self.site_id, floor=self.floor, input_path=self.input_path)\n \n def _flatten(self, l):\n return list(itertools.chain.from_iterable(l))\n \n def multi_line_spliter(self, s):\n matches = re.finditer(\"TYPE_\", s)\n matches_positions = [match.start() for match in matches]\n split_idx = [0] + [matches_positions[i]-14 for i in range(1, len(matches_positions))] + [len(s)]\n return [s[split_idx[i]:split_idx[i+1]] for i in range(len(split_idx)-1)]\n \n def load_df(self, ):\n path = str(Path(self.input_path)/f\"train/{self.site_id}/{self.floor}/{self.path_id}.txt\")\n with open(path) as f:\n data = f.readlines()\n \n modified_data = []\n for s in data:\n if s.count(\"TYPE_\")>1:\n lines = self.multi_line_spliter(s)\n modified_data.extend(lines)\n else:\n modified_data.append(s)\n del data\n self.meta_info_len = len([d for d in modified_data if d[0]==\"#\"])\n self.meta_info_df = pd.DataFrame([m.replace(\"\\n\", \"\").split(\":\") \n for m in self._flatten([d.split(\"\\t\") \n for d in modified_data if d[0]==\"#\"]) if m!=\"#\"])\n\n data_df = pd.DataFrame([d.replace(\"\\n\", \"\").split(\"\\t\") for d in modified_data if d[0]!=\"#\"])\n for dt in self.df_types:\n # select data type\n df_s = data_df[data_df[1]==f\"TYPE_{dt.upper()}\"]\n if len(df_s)==0:\n setattr(self, dt, pd.DataFrame(columns=self.df_type_cols[dt]))\n else:\n # remove empty cols\n na_info = df_s.isna().sum(axis=0) == len(df_s)\n df_s = df_s[[i for i in na_info[na_info==False].index if i!=1]].reset_index(drop=True)\n \n if len(df_s.columns)!=len(self.df_type_cols[dt]):\n df_s.columns = self.df_type_cols[dt][:len(df_s.columns)]\n else:\n df_s.columns = self.df_type_cols[dt]\n \n # set dtype \n for c in df_s.columns:\n df_s[c] = df_s[c].astype(self.dtype_dict[dt][c])\n \n # set DataFrame to attr\n setattr(self, dt, df_s)\n \n def get_site_info(self, keep_raw=False):\n self.site_info.get_site_info(keep_raw=keep_raw)\n \n def load_all_data(self, keep_raw=False): \n self.load_df()\n self.get_site_info(keep_raw=keep_raw)\n \n def __getitem__(self, item):\n if item in self.df_types:\n return getattr(self, item)\n else:\n return None\n \n def save(self, ):\n # to be implemented\n pass\n \n \nclass SiteInfo():\n def __init__(self, site_id, floor, input_path=\"../input/indoor-location-navigation/\"):\n self.site_id = site_id\n self.floor = floor\n self.input_path = input_path\n assert Path(input_path).exists(), f\"input_path do not exist: {input_path}\"\n \n def get_site_info(self, keep_raw=False):\n floor_info_path = f\"{self.input_path}/metadata/{self.site_id}/{self.floor}/floor_info.json\"\n with open(floor_info_path, \"r\") as f:\n self.floor_info = json.loads(f.read())\n self.site_height = self.floor_info[\"map_info\"][\"height\"]\n self.site_width = self.floor_info[\"map_info\"][\"width\"]\n if not keep_raw:\n del self.floor_info\n \n geojson_map_path = f\"{self.input_path}/metadata/{self.site_id}/{self.floor}/geojson_map.json\"\n with open(geojson_map_path, \"r\") as f:\n self.geojson_map = json.loads(f.read())\n self.map_type = self.geojson_map[\"type\"]\n self.features = self.geojson_map[\"features\"]\n \n self.floor_coordinates = self.features[0][\"geometry\"][\"coordinates\"]\n self.store_coordinates = [self.features[i][\"geometry\"][\"coordinates\"] \n for i in range(1, len(self.features))]\n \n if not keep_raw:\n del self.geojson_map\n \n def show_site_image(self):\n path = f\"{self.input_path}/metadata/{self.site_id}/{self.floor}/floor_image.png\"\n plt.imshow(imread(path), extent=[0, self.site_width, 0, self.site_height])\n\n def draw_polygon(self, size=8, only_floor=False):\n\n fig = plt.figure()\n ax = plt.subplot(111)\n \n xmax, xmin, ymax, ymin = self._draw(self.floor_coordinates, ax, calc_minmax=True)\n if not only_floor:\n self._draw(self.store_coordinates, ax, fill=True)\n plt.legend([])\n \n xrange = xmax - xmin\n yrange = ymax - ymin\n ratio = yrange / xrange\n \n self.x_size = size\n self.y_size = size*ratio\n\n fig.set_figwidth(size)\n fig.set_figheight(size*ratio)\n # plt.show()\n return ax\n \n def _draw(self, coordinates, ax, fill=False, calc_minmax=False):\n xmax, ymax = -np.inf, -np.inf\n xmin, ymin = np.inf, np.inf\n for i in range(len(coordinates)):\n ndim = np.ndim(coordinates[i])\n if ndim==2:\n corrd_df = pd.DataFrame(coordinates[i])\n if fill:\n ax.fill(corrd_df[0], corrd_df[1], alpha=0.7)\n else:\n corrd_df.plot.line(x=0, y=1, style=\"-\", ax=ax)\n \n if calc_minmax:\n xmax = max(xmax, corrd_df[0].max())\n xmin = min(xmin, corrd_df[0].min())\n\n ymax = max(ymax, corrd_df[1].max())\n ymin = min(ymin, corrd_df[1].min())\n elif ndim==3:\n for j in range(len(coordinates[i])):\n corrd_df = pd.DataFrame(coordinates[i][j])\n if fill:\n ax.fill(corrd_df[0], corrd_df[1], alpha=0.6)\n else:\n corrd_df.plot.line(x=0, y=1, style=\"-\", ax=ax)\n \n if calc_minmax:\n xmax = max(xmax, corrd_df[0].max())\n xmin = min(xmin, corrd_df[0].min())\n\n ymax = max(ymax, corrd_df[1].max())\n ymin = min(ymin, corrd_df[1].min())\n else:\n assert False, f\"ndim of coordinates should be 2 or 3: {ndim}\"\n if calc_minmax:\n return xmax, xmin, ymax, ymin\n else:\n return None\n \n","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_meta_data\ntrain_meta = glob(\"../input/indoor-location-navigation/train/*/*/*\")\ntrain_meta_org = pd.DataFrame(train_meta)\ntrain_meta = train_meta_org[0].str.split(\"/\", expand=True)[[4, 5, 6]]\ntrain_meta.columns = [\"site_id\", \"floor\", \"path_id\"]\ntrain_meta[\"path_id\"] = train_meta[\"path_id\"].str.replace(\".txt\", \"\")\ntrain_meta[\"path\"] = train_meta_org[0]\n#train_meta.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pickle_dump_dill(obj, path):\n with open(path, mode='wb') as f:\n dill.dump(obj, f)\n\n\ndef pickle_load_dill(path):\n with open(path, mode='rb') as f:\n data = dill.load(f)\n return data","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub = pd.read_csv('../input/indoor-location-navigation/sample_submission.csv')\ntest_sites = sample_sub.site_path_timestamp.apply(lambda x: pd.Series(x.split(\"_\")))[0].unique().tolist()\n\ntest_meta = sample_sub[\"site_path_timestamp\"].apply(\n lambda x: pd.Series(x.split(\"_\")))\ntest_meta.columns = [\"site_id\", \"path_id\", \"timestamp\"]\ntest_meta=test_meta.drop('timestamp', axis=1)\ntest_meta = test_meta.drop_duplicates(subset=[\"site_id\", \"path_id\"]).reset_index(drop=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get first and last waypoints in train dataset","metadata":{}},{"cell_type":"code","source":"create_train_meta_sub=False\nif create_train_meta_sub:\n train_meta_sub=train_meta[train_meta['site_id'].isin(test_sites)].reset_index(drop=True)\n train_meta_sub['start_time']=0\n train_meta_sub['end_time']=0\n train_meta_sub['start_wp_time']=0\n train_meta_sub['start_wp_x']=0\n train_meta_sub['start_wp_y']=0\n train_meta_sub['end_wp_time']=0\n train_meta_sub['end_wp_x']=0\n train_meta_sub['end_wp_y']=0\n train_meta_sub['n_floor']=0\n for i in tqdm(range(len(train_meta_sub))):\n t = train_meta_sub.iloc[i]\n n_floor = FeatureStore.floor_convert[t.floor]\n feature = FeatureStore(\n site_id=t.site_id, floor=t.floor, path_id=t.path_id)\n feature.load_all_data() \n start_time=int(feature.meta_info_df[feature.meta_info_df[0]=='startTime'][1])\n end_time=int(feature.meta_info_df[feature.meta_info_df[0]=='endTime'][1])\n train_meta_sub.loc[i,'start_time']=start_time\n train_meta_sub.loc[i,'start_wp_time']=feature.waypoint.iloc[0]['timestamp']\n train_meta_sub.loc[i,'start_wp_x']=feature.waypoint.iloc[0]['x']\n train_meta_sub.loc[i,'start_wp_y']=feature.waypoint.iloc[0]['y']\n train_meta_sub.loc[i,'end_time']=end_time\n train_meta_sub.loc[i,'end_wp_time']=feature.waypoint.iloc[-1]['timestamp']\n train_meta_sub.loc[i,'end_wp_x']=feature.waypoint.iloc[-1]['x']\n train_meta_sub.loc[i,'end_wp_y']=feature.waypoint.iloc[-1]['y']\n train_meta_sub.loc[i,'n_floor']=feature.n_floor\n train_meta_sub.to_csv('train_meta_sub.csv', index=False)\nelse:\n train_meta_sub = pd.read_csv('../input/indoor-public/train_meta_sub.csv')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_meta_sub[:50]","metadata":{"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nfor test_site in test_sites:\n plt.figure()\n sns.boxplot(x='floor', y='start_time', data=train_meta_sub[train_meta_sub.site_id==test_site])","metadata":{"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_txt(file):\n with open(file) as f:\n txt = f.readlines()\n\n modified_data = []\n for s in txt:\n if s.count(\"TYPE_\") > 1:\n lines = multi_line_spliter(s)\n modified_data.extend(lines)\n else:\n modified_data.append(s)\n return modified_data\n\n\ndef _flatten(l):\n return list(itertools.chain.from_iterable(l))\n\n\ndef get_feature_test(site_id, path_id, input_path, sample_sub):\n file = f\"{input_path}/test/{path_id}.txt\"\n content = read_txt(file)\n data_df = pd.DataFrame([d.replace(\"\\n\", \"\").split(\"\\t\")\n for d in content if d[0] != \"#\"])\n data_dict = OrderedDict()\n for dt in FeatureStore.df_types:\n # select data type\n df_s = data_df[data_df[1] == f\"TYPE_{dt.upper()}\"]\n if len(df_s) == 0:\n setattr(data_dict, dt, pd.DataFrame(\n columns=FeatureStore.df_type_cols[dt]))\n else:\n # remove empty cols\n na_info = df_s.isna().sum(axis=0) == len(df_s)\n df_s = df_s[[i for i in na_info[na_info ==\n False].index if i != 1]].reset_index(drop=True)\n\n if len(df_s.columns) != len(FeatureStore.df_type_cols[dt]):\n df_s.columns = FeatureStore.df_type_cols[dt][:len(\n df_s.columns)]\n else:\n df_s.columns = FeatureStore.df_type_cols[dt]\n\n # set dtype\n for c in df_s.columns:\n df_s[c] = df_s[c].astype(FeatureStore.dtype_dict[dt][c])\n setattr(data_dict, dt, df_s)\n data_dict.meta_info_df = pd.DataFrame([m.replace(\"\\n\", \"\").split(\":\")\n for m in _flatten([d.split(\"\\t\")\n for d in content if d[0] == \"#\"]) if m != \"#\"])\n startTime_ind = int(np.where(data_dict.meta_info_df[0] == 'startTime')[0])\n endTime_ind = int(np.where(data_dict.meta_info_df[0] == 'endTime')[0])\n data_dict.meta_info_df.loc[startTime_ind,\n 1] = data_dict.meta_info_df.loc[startTime_ind+1, 0]\n data_dict.meta_info_df.loc[endTime_ind,\n 1] = data_dict.meta_info_df.loc[endTime_ind+1, 0]\n\n data_dict.waypoint['timestamp'] = sample_sub[sample_sub.path_id ==\n path_id].timestamp.values.astype(int)\n data_dict.waypoint['x'] = 0\n data_dict.waypoint['y'] = 0\n data_dict.n_floor = 0\n data_dict.site_id = site_id\n if len(data_dict.beacon) > 0:\n gap = data_dict.beacon.loc[0, 'timestamp2'] + \\\n data_dict.beacon.loc[0, 'timestamp']\n else:\n gap = (data_dict.wifi.last_seen_timestamp.values -\n data_dict.wifi.timestamp.values).max()+210.14426803816337 # from mean gap\n data_dict.wifi.last_seen_timestamp = data_dict.wifi.last_seen_timestamp-gap\n return data_dict","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Postprocessing based on leaked feature.","metadata":{}},{"cell_type":"code","source":"def leak_postprocessing(submission_df,train_meta, postprocess_start=True, postprocess_end=True, postprocess_floor=True,start_threshold=5500,end_threshold=6500):\n submission_df[[\"site_id\", \"path_id\", \"timestamp\"]] = submission_df[\"site_path_timestamp\"].apply(\n lambda x: pd.Series(x.split(\"_\")))\n start_counter = 0\n end_counter = 0\n floor_counter = 0\n input_path='/kaggle/input/indoor-location-navigation/'\n sample_sub = pd.read_csv(f\"{input_path}/sample_submission.csv\")\n sample_sub = sample_sub[\"site_path_timestamp\"].apply(\n lambda x: pd.Series(x.split(\"_\")))\n sample_sub.columns = [\"site_id\", \"path_id\", \"timestamp\"]\n submission_df_unique=submission_df.drop_duplicates(\n subset=[\"site_id\", \"path_id\"]).reset_index(drop=True)\n for i in tqdm(range(len(submission_df_unique.path_id))):\n t = submission_df_unique.iloc[i]\n site_id=t.site_id\n path_id=t.path_id\n feature = get_feature_test(site_id, path_id, input_path, sample_sub)\n if feature.meta_info_df[feature.meta_info_df[0] == 'startTime'][1].values == None:\n start_time = int(np.nanmin([feature.accelerometer.timestamp.min(\n ), feature.wifi.timestamp.min(), feature.beacon.timestamp.min()]))\n else:\n start_time = int(\n feature.meta_info_df[feature.meta_info_df[0] == 'startTime'][1])\n if (len(feature.meta_info_df[feature.meta_info_df[0] == 'endTime']) == 0) or (feature.meta_info_df[feature.meta_info_df[0] == 'endTime'][1].values == None):\n end_time = int(np.nanmax([feature.accelerometer.timestamp.max(\n ), feature.wifi.timestamp.max(), feature.beacon.timestamp.max()]))\n else:\n end_time = int(\n feature.meta_info_df[feature.meta_info_df[0] == 'endTime'][1])\n if len(feature.beacon) > 0:\n gap = feature.beacon.loc[0, 'timestamp2'] + \\\n feature.beacon.loc[0, 'timestamp']\n else:\n gap = (feature.wifi.last_seen_timestamp.values -\n feature.wifi.timestamp.values).max()+210.14426803816337 # from mean gap\n site_id = feature.site_id\n train_meta_site = train_meta[train_meta.site_id == site_id]\n \n #postprocess start point based on leakage\n train_meta_site_end = train_meta_site[(\n start_time+gap) > train_meta_site.end_time]\n if len(train_meta_site_end) > 0:\n nearest_endpoint = train_meta_site_end.loc[train_meta_site_end.end_time.idxmax(\n )]\n if postprocess_start and (start_time + gap - nearest_endpoint.end_time < start_threshold):\n submission_df.loc[(submission_df.path_id == path_id) & (submission_df.timestamp == \n submission_df[submission_df.path_id == path_id].timestamp.min()), 'x'] = nearest_endpoint.end_wp_x\n submission_df.loc[(submission_df.path_id == path_id) & (submission_df.timestamp == \n submission_df[submission_df.path_id == path_id].timestamp.min()), 'y'] = nearest_endpoint.end_wp_y\n start_counter += 1\n \n #postprocess end point based on leakage\n train_meta_site_start = train_meta_site[train_meta_site.start_time > (\n end_time+gap)]\n if len(train_meta_site_start) > 0:\n nearest_startpoint = train_meta_site_start.loc[train_meta_site_start.start_time.idxmin(\n )]\n if postprocess_end and (nearest_startpoint.start_time - end_time - gap < end_threshold):\n submission_df.loc[(submission_df.path_id == path_id) & (submission_df.timestamp == \n submission_df[submission_df.path_id == path_id].timestamp.max()), 'x'] = nearest_startpoint.start_wp_x\n submission_df.loc[(submission_df.path_id == path_id) & (submission_df.timestamp == \n submission_df[submission_df.path_id == path_id].timestamp.max()), 'y'] = nearest_startpoint.start_wp_y\n end_counter += 1\n \n #postprocess floor based on leakage\n if postprocess_floor:\n if (len(train_meta_site_end) > 0) and (len(train_meta_site_start) > 0) and (nearest_endpoint.n_floor == nearest_startpoint.n_floor):\n submission_df.loc[(submission_df.path_id == path_id),\n 'floor'] = nearest_endpoint.n_floor\n floor_counter += (submission_df.path_id == path_id).sum()\n elif (len(train_meta_site_end) > 0) and (len(train_meta_site_start) > 0):\n diff_start_time = start_time - nearest_endpoint.end_time\n diff_end_time = nearest_startpoint.start_time - end_time\n if diff_start_time < diff_end_time:\n submission_df.loc[(submission_df.path_id == path_id),\n 'floor'] = nearest_endpoint.n_floor\n floor_counter += (submission_df.path_id == path_id).sum()\n if diff_end_time < diff_start_time:\n submission_df.loc[(submission_df.path_id == path_id),\n 'floor'] = nearest_startpoint.n_floor\n floor_counter += (submission_df.path_id == path_id).sum()\n elif len(train_meta_site_end) > 0:\n submission_df.loc[(submission_df.path_id == path_id),\n 'floor'] = nearest_endpoint.n_floor\n floor_counter += (submission_df.path_id == path_id).sum()\n elif len(train_meta_site_start) > 0:\n submission_df.loc[(submission_df.path_id == path_id),\n 'floor'] = nearest_startpoint.n_floor\n floor_counter += (submission_df.path_id == path_id).sum()\n\n print(str(start_counter) + ' start points are postprocessed.')\n print(str(end_counter) + ' end points are postprocessed.')\n print(str(floor_counter) + ' floors are postprocessed.')\n submission_df = submission_df.drop(\n [\"site_id\", \"path_id\", \"timestamp\"], axis=1)\n return submission_df","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = submission_snap_to_grid","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df_leak_start = leak_postprocessing(submission_df,train_meta_sub, postprocess_start=True, postprocess_end=False, postprocess_floor=False)\nsubmission_df_leak_start.to_csv(\n 'submission_df_leak_start.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df_leak_end = leak_postprocessing(submission_df,train_meta_sub, postprocess_start=False, postprocess_end=True, postprocess_floor=False)\nsubmission_df_leak_end.to_csv(\n 'submission_df_leak_end.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df_leak_floor = leak_postprocessing(submission_df,train_meta_sub, postprocess_start=False, postprocess_end=False, postprocess_floor=True)\nsubmission_df_leak_floor.to_csv(\n 'submission_df_leak_floor.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df_leak_all = leak_postprocessing(submission_df,train_meta_sub, postprocess_start=True, postprocess_end=True, postprocess_floor=True)\nsubmission_df_leak_all.to_csv(\n 'submission_df_leak_all.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Visualization\n\nWe just draw the \"End point\" option. The best public score will be created with this option.","metadata":{}},{"cell_type":"code","source":"def split_col(df):\n \"\"\"\n Split submission site/path/timestamp into individual columns.\n \"\"\"\n df = pd.concat(\n [\n df[\"site_path_timestamp\"]\n .str.split(\"_\", expand=True)\n .rename(columns={0: \"site\", 1: \"path\", 2: \"timestamp\"}),\n df,\n ],\n axis=1,\n ).copy()\n return df\n\n\ndef plot_preds(\n site,\n floorNo,\n sub=None,\n true_locs=None,\n base=\"../input/indoor-location-navigation\",\n show_train=True,\n show_preds=True,\n):\n \"\"\"\n Plots predictions on floorplan map.\n \"\"\"\n # Prepare width_meter & height_meter (taken from the .json file)\n floor_plan_filename = f\"{base}/metadata/{site}/{floorNo}/floor_image.png\"\n json_plan_filename = f\"{base}/metadata/{site}/{floorNo}/floor_info.json\"\n with open(json_plan_filename) as json_file:\n json_data = json.load(json_file)\n\n width_meter = json_data[\"map_info\"][\"width\"]\n height_meter = json_data[\"map_info\"][\"height\"]\n\n floor_img = plt.imread(f\"{base}/metadata/{site}/{floorNo}/floor_image.png\")\n\n fig, ax = plt.subplots(figsize=(12, 12))\n plt.imshow(floor_img)\n\n if show_train:\n true_locs[\"x_\"] = true_locs[\"x\"] * floor_img.shape[0] / height_meter\n true_locs[\"y_\"] = (\n true_locs[\"y\"] * -1 * floor_img.shape[1] / width_meter\n ) + floor_img.shape[0]\n true_locs.query(\"site == @site and floorNo == @floorNo\").groupby(\"path\").plot(\n x=\"x_\",\n y=\"y_\",\n style=\"+\",\n ax=ax,\n label=\"train waypoint location\",\n color=\"grey\",\n alpha=0.5,\n )\n\n if show_preds:\n sub[\"x_\"] = sub[\"x\"] * floor_img.shape[0] / height_meter\n sub[\"y_\"] = (\n sub[\"y\"] * -1 * floor_img.shape[1] / width_meter\n ) + floor_img.shape[0]\n for path, path_data in sub.query(\n \"site == @site and floorNo == @floorNo\"\n ).groupby(\"path\"):\n path_data.plot(\n x=\"x_\",\n y=\"y_\",\n style=\".-\",\n ax=ax,\n title=f\"{site} - floor - {floorNo}\",\n alpha=1,\n label=path,\n )\n return fig, ax\n","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = split_col(submission_df_leak_end)\n\ntrue_locs = pd.read_csv(\"../input/indoor-location-train-waypoints/train_waypoints.csv\")\n\n# Add floor No to sub file\nsub = sub.merge(true_locs[[\"site\", \"floor\", \"floorNo\"]].drop_duplicates())\n\n\nfor (site, floorNo), d in sub.groupby([\"site\", \"floorNo\"]):\n fig, ax = plot_preds(site, floorNo, sub, true_locs)\n # Remove duplicate labels\n handles, labels = ax.get_legend_handles_labels()\n by_label = dict(zip(labels, handles))\n plt.legend(\n by_label.values(), by_label.keys(), loc=\"center left\", bbox_to_anchor=(1, 0.5)\n )\n plt.show()\n ","metadata":{"_kg_hide-output":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n
","metadata":{}},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"sub = submission_df_leak_end\n\nsub.to_csv(\"submission.csv\", index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n
","metadata":{}}]}