{"cells":[{"metadata":{},"cell_type":"markdown","source":"In this notebook, I'm going to explore the coverage of the training data of the site-surveyor, then train a simple LightGDM model to predict and validate data by visualization."},{"metadata":{},"cell_type":"markdown","source":"### Libraries📚"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install pytorch-tabnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport glob\nimport math\nimport json\n\nfrom dataclasses import dataclass\n\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport matplotlib.pyplot as plt\nimport plotly.graph_objs as go\nfrom PIL import Image","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Reading in the data"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"!cp -r /kaggle/input/github-dataset/* ./","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":false,"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# Import custom function from the repository\nfrom io_f import read_data_file\n\n# How 1 path looks\nbase = '../input/indoor-location-navigation'\npath = f'{base}/train/5a0546857ecc773753327266/B1/5e15730aa280850006f3d005.txt'\n\n# Read in 1 random example\nsample_file = read_data_file(path)\n\n# You can access the information for each variable:\nprint(\"~~~ Example ~~~\")\nprint(\"acce: {}\".format(sample_file.acce.shape), \"\\n\" +\n      \"acacce_uncalice: {}\".format(sample_file.acce_uncali.shape), \"\\n\" +\n      \"ahrs: {}\".format(sample_file.ahrs.shape), \"\\n\" +\n      \"gyro: {}\".format(sample_file.gyro.shape), \"\\n\" +\n      \"gyro_uncali: {}\".format(sample_file.gyro_uncali.shape), \"\\n\" +\n      \"ibeacon: {}\".format(sample_file.ibeacon.shape), \"\\n\" +\n      \"magn: {}\".format(sample_file.magn.shape), \"\\n\" +\n      \"magn_uncali: {}\".format(sample_file.magn_uncali.shape), \"\\n\" +\n      \"waypoint: {}\".format(sample_file.waypoint.shape), \"\\n\" +\n      \"wifi: {}\".format(sample_file.wifi.shape))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# All waypoint on site 0\n\nLet's find out the data coverage on 1 of the buildings."},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"floorplans = sorted(glob.glob(f\"{base}/train/*/*\"))\nprint(\"Number of floor plans:\", len(floorplans))\nfloorplans[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"paths = {fp:glob.glob(f\"{fp}/*.txt\") for fp in floorplans}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def visualize_trajectories(trajectories, floor_plan_filename, width_meter, height_meter, title=None, mode='lines + markers + text', show=False):\n    fig = go.Figure()\n\n    # add trajectory\n    for trajectory in trajectories:\n        size_list = [6] * trajectory.shape[0]\n        size_list[0] = 10\n        size_list[-1] = 10\n\n        color_list = ['rgba(4, 174, 4, 0.5)'] * trajectory.shape[0]\n        color_list[0] = 'rgba(12, 5, 235, 1)'\n        color_list[-1] = 'rgba(235, 5, 5, 1)'\n\n        position_count = {}\n        text_list = []\n        for i in range(trajectory.shape[0]):\n            if str(trajectory[i]) in position_count:\n                position_count[str(trajectory[i])] += 1\n            else:\n                position_count[str(trajectory[i])] = 0\n            text_list.append('        ' * position_count[str(trajectory[i])] + f'{i}')\n        text_list[0] = 'Start Point: 0'\n        text_list[-1] = f'End Point: {trajectory.shape[0] - 1}'\n\n        fig.add_trace(\n            go.Scattergl(\n                x=trajectory[:, 0],\n                y=trajectory[:, 1],\n                mode=mode,\n                marker=dict(size=size_list, color=color_list),\n                line=dict(shape='linear', color='rgb(100, 10, 100)', width=2, dash='dot'),\n                text=text_list,\n                textposition=\"top center\",\n                name='trajectory',\n            ))\n\n    # add floor plan\n    floor_plan = Image.open(floor_plan_filename)\n    fig.update_layout(images=[\n        go.layout.Image(\n            source=floor_plan,\n            xref=\"x\",\n            yref=\"y\",\n            x=0,\n            y=height_meter,\n            sizex=width_meter,\n            sizey=height_meter,\n            sizing=\"contain\",\n            opacity=1,\n            layer=\"below\",\n        )\n    ])\n\n    # configure\n    fig.update_xaxes(autorange=False, range=[0, width_meter])\n    fig.update_yaxes(autorange=False, range=[0, height_meter], scaleanchor=\"x\", scaleratio=1)\n    fig.update_layout(\n        title=go.layout.Title(\n            text=title or \"No title.\",\n            xref=\"paper\",\n            x=0,\n        ),\n        autosize=True,\n        width=900,\n        height=200 + 900 * height_meter / width_meter,\n        template=\"plotly_white\",\n    )\n\n    if show:\n        fig.show()\n\n    return fig","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import glob\nSITES = list(map(lambda x: x.split('/')[-1].split('_')[0], sorted(glob.glob('../input/generate-wifi-features-5-times-faster/*_train.csv'))))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"siteNo = 4\nsite = SITES[siteNo]\nfor floor in floorplans:\n    if floor.split('/')[-2] != site:\n        continue\n    floorNo = floor.split('/')[-1]\n\n    trajectories = list()\n    for path_filename in glob.glob(f'{base}/train/{site}/{floorNo}/*.txt'):\n\n        # Read in a sample\n        example = read_data_file(path_filename)\n\n        # ~~~~~~~~~\n\n        # Returns timestamp, x, y values\n        trajectory = example.waypoint\n        # Removes timestamp (we only need the coordinates)\n        trajectory = trajectory[:, 1:3]\n        trajectories.append(trajectory)\n\n    # Prepare floor_plan coresponding with our example\n    floor_plan_filename = f'{base}/metadata/{site}/{floorNo}/floor_image.png'\n\n    # Prepare width_meter & height_meter\n    ### (taken from the .json file)\n    json_plan_filename = f'{base}/metadata/{site}/{floorNo}/floor_info.json'\n    with open(json_plan_filename) as json_file:\n        json_data = json.load(json_file)\n\n    width_meter = json_data[\"map_info\"][\"width\"]\n    height_meter = json_data[\"map_info\"][\"height\"]\n\n    # Title\n    title = f\"All Waypoints {floorNo}\"\n\n    # ~~~~~~~~~\n    # Finally, let's plot\n    visualize_trajectories(trajectories = trajectories,\n                         floor_plan_filename = floor_plan_filename,\n                         width_meter = width_meter,\n                         height_meter = height_meter,\n                         title = title,\n                         show = True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Compare predicted waypoint with ground truth on site 0"},{"metadata":{},"cell_type":"markdown","source":"First, I'm gonna train a simple lightGBM Regressor to predict the position and floor.\n\n> 📌**Note**: Preprocessed data is from [this dataset](https://www.kaggle.com/devinanzelmo/indoor-navigation-and-location-wifi-features) by [Devin Anzelmo](https://www.kaggle.com/devinanzelmo)."},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def mean_position_error(x_pred, y_pred, f_pred, x_true, y_true, f_true, p=15):\n    '''Custom function to evaluate Mean Position Error.\n    x: x coordinate of the waypoint position; dtype list()\n    y: y coordinate of the waypoint position; dtype list()\n    f: exact floor or the building; dtype list()\n    p: floor penalty, set to 15 (always)'''\n    \n    N = len(x_true)\n    #1\n    formula = np.sqrt( np.power(x_pred - x_true, 2) + np.power(y_pred - y_true, 2) )\n    #2\n    formula = formula + p * np.absolute(f_pred - f_true)\n    #3\n    formula = formula.sum() / N\n    \n    return formula","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"trusted":true},"cell_type":"code","source":"N_A = 48\nsiteNo = 13\nSITE = SITES[siteNo]\n# Import Libraries\nfrom pytorch_tabnet.tab_model import TabNetRegressor ##Import Tabnet \n\nfeature_dir = \"/kaggle/input/generate-wifi-features-5-times-faster\"\ntrain_file = f\"{feature_dir}/{SITE}_train.csv\"\n\ntrain_df = pd.read_csv(train_file)\n\nloaded_clf = TabNetRegressor()\nloaded_clf.load_model(f'../input/tabnet-model-container/tabnet_{N_A}/tabnet_model_test_{siteNo}.zip')\n\nprediction_dict = loaded_clf.predict(train_df.drop(columns = ['x', 'y', 'f', 'path']).values) \n\npreds_x = prediction_dict[:, 0]\npreds_y = prediction_dict[:, 1]\npreds_f = prediction_dict[:, 2].round()\n\n#Accuracy\nprint('Accuracy floor of site {}: {}'.format(siteNo, (preds_f.shape[0] - np.abs(preds_f - train_df['f']).sum())/preds_f.shape[0] * 100))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"floor_error = preds_f - train_df['f']\nfloor_error[floor_error !=0].index #760","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Valid paths\npaths_valid = train_df.iloc[:, -1].unique()\npathNo_to_draw = np.where(paths_valid == train_df.loc[703]['path'])[0][0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Path in training data\n\nThen, I'm gonna try to trace back the path in the validation data set created earlier. Then draw the ground truth position using Plotly from this [GitHub repo](https://github.com/location-competition/indoor-location-competition-20)."},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# GitHub functions\npath_to_draw = paths_valid[pathNo_to_draw]\nfrom visualize_f import visualize_trajectory, visualize_heatmap\n\nbase = '../input/indoor-location-navigation'\nsite = SITE\npathNo = path_to_draw\nfloorNo = glob.glob(f'{base}/train/{site}/*/{path_to_draw}.txt')[0].split('/')[-2]\n\npath_filename = f'{base}/train/{site}/{floorNo}/{path_to_draw}.txt'\n\n# Read in a sample\nexample = read_data_file(path_filename)\n\n# ~~~~~~~~~\n\n# Returns timestamp, x, y values\ntrajectory = example.waypoint\n# Removes timestamp (we only need the coordinates)\ntrajectory = trajectory[:, 1:3]\n\ngt_trajectory = trajectory\ngt_pos_df = train_df[['x', 'y', 'path']]\n# gt_pos_df.columns = ['x', 'y', 'path']\ngt_pos_df_to_draw = gt_pos_df[gt_pos_df['path'] == path_to_draw]\ngt_pos_df_to_draw","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[train_df['path'] == path_to_draw].drop(columns = ['x', 'y', 'f', 'path'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.bincount(np.where(train_df[train_df['path'] == path_to_draw].drop(columns = ['x', 'y', 'f', 'path']).values != -999)[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Check for wifi signal\nnp.where(train_df[train_df['path'] == path_to_draw].drop(columns = ['x', 'y', 'f', 'path']).values != -999)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Check for wifi signal and beacon signal\nwith open(path_filename) as f:\n    for row in csv.reader(f, delimiter=\"\\t\", doublequote=True):\n        if row[1] == \"TYPE_WAYPOINT\":\n            print(row)\nwith open(path_filename) as f:\n    for row in csv.reader(f, delimiter=\"\\t\", doublequote=True):\n        if row[1] == \"TYPE_WIFI\":\n            print(row)\nwith open(path_filename) as f:\n    for row in csv.reader(f, delimiter=\"\\t\", doublequote=True):\n        if row[1] == \"TYPE_BEACON\":\n            print(row)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# Prepare floor_plan coresponding with our example\nfloor_plan_filename = f'{base}/metadata/{site}/{floorNo}/floor_image.png'\n\n# Prepare width_meter & height_meter\n### (taken from the .json file)\njson_plan_filename = f'{base}/metadata/{site}/{floorNo}/floor_info.json'\nwith open(json_plan_filename) as json_file:\n    json_data = json.load(json_file)\n    \nwidth_meter = json_data[\"map_info\"][\"width\"]\nheight_meter = json_data[\"map_info\"][\"height\"]\n\n# Title\ntitle = f\"Training Waypoint {floorNo}\"\n\n# ~~~~~~~~~\n\n# Finally, let's plot\nvisualize_trajectory(trajectory = gt_pos_df_to_draw.iloc[:,:2].to_numpy(),\n                     floor_plan_filename = floor_plan_filename,\n                     width_meter = width_meter,\n                     height_meter = height_meter,\n                     title = title)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Path in prediction\n\nFinally, I'm gonna do the same for the predicted path."},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"path_to_draw = paths_valid[pathNo_to_draw]\npred_pos_df = pd.DataFrame(np.array([preds_x.T, preds_y, preds_f, train_df.iloc[:, -1]])).T\npred_pos_df.columns = ['x', 'y', 'f', 'path']\npred_pos_df = pred_pos_df[pred_pos_df['path'] == path_to_draw]\npred_pos_df","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"from visualize_f import visualize_trajectory, visualize_heatmap\nfrom collections import Counter\n\nbase = '../input/indoor-location-navigation'\nsite = SITE\npathNo = path_to_draw\nfloor_int = Counter(pred_pos_df['f']).most_common(1)[0][0]\n\nfloor_map = {-2: [\"B2\"], -1: [\"B1\"], 0: [\"F1\", '1F'], 1: ['F2', '2F'], 2: ['F3', '3F'], 3: ['F4', '4F'], 4: ['F5', '5F'], 5: ['F6', '6F'], 6: ['F7' ,'7F'],\n                                    7: ['F8', '8F'], 8: ['F9', '9F']}\n\nfloorNolist = floor_map[floor_int]\nfloorNo = \"error\"\nfor floorNo_ in floorNolist:\n    if os.path.exists(os.path.join(base, 'metadata', site, floorNo_)):\n        floorNo = floorNo_\n        break\n\n# Removes timestamp (we only need the coordinates)\ntrajectory = pred_pos_df.iloc[:,:2].to_numpy()\n\n# Prepare floor_plan coresponding with our example\nfloor_plan_filename = f'{base}/metadata/{site}/{floorNo}/floor_image.png'\n\n# Prepare width_meter & height_meter\n### (taken from the .json file)\njson_plan_filename = f'{base}/metadata/{site}/{floorNo}/floor_info.json'\nwith open(json_plan_filename) as json_file:\n    json_data = json.load(json_file)\n    \nwidth_meter = json_data[\"map_info\"][\"width\"]\nheight_meter = json_data[\"map_info\"][\"height\"]\n\n# Title\ntitle = f\"Prediction on Waypoint {floorNo}\"\n\n# ~~~~~~~~~\n\n# Finally, let's plot\nvisualize_trajectory(trajectory = trajectory,\n                     floor_plan_filename = floor_plan_filename,\n                     width_meter = width_meter,\n                     height_meter = height_meter,\n                     title = title)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}