{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport sklearn \nimport json\nimport math\nimport math\nimport glob\nimport cv2\nimport scipy \nimport seaborn as sns\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport plotly.graph_objs as go\nfrom PIL import Image, ImageOps\nfrom skimage import io\nfrom skimage.color import rgba2rgb, rgb2xyz\nfrom tqdm import tqdm\nfrom dataclasses import dataclass\nfrom math import floor, ceil\nimport os","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-02T15:23:06.480108Z","iopub.execute_input":"2021-06-02T15:23:06.480664Z","iopub.status.idle":"2021-06-02T15:23:08.177332Z","shell.execute_reply.started":"2021-06-02T15:23:06.480551Z","shell.execute_reply":"2021-06-02T15:23:08.176407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"site = '5d27096c03f801723c31e5e0'\nfloorNo = 'F3'\n\nfloor = plt.imread(f\"../input/indoor-location-navigation/metadata/{site}/{floorNo}/floor_image.png\")\nbw = plt.imread(f\"../input/indoorlocationnavigationbwdist/metadata/{site}/{floorNo}/geojson_map_bw_filled_x16.png\")\ndist = plt.imread(f\"../input/indoorlocationnavigationbwdist/metadata/{site}/{floorNo}/geojson_map_cv_dist_normalized_x16.png\")","metadata":{"execution":{"iopub.status.busy":"2021-06-02T15:23:08.179703Z","iopub.execute_input":"2021-06-02T15:23:08.180043Z","iopub.status.idle":"2021-06-02T15:23:08.346061Z","shell.execute_reply.started":"2021-06-02T15:23:08.180013Z","shell.execute_reply":"2021-06-02T15:23:08.344603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Indoor Location Competition 2.0 GIT","metadata":{}},{"cell_type":"code","source":"!cp -r ../input/indoor-location-competition-20-git/* ./\n    # GitHub functions\nimport io_f \nimport visualize_f \nimport compute_f\nfrom main import calibrate_magnetic_wifi_ibeacon_to_position\nfrom main import extract_magnetic_strength\nfrom main import extract_wifi_rssi, extract_wifi_count\nfrom main import extract_ibeacon_rssi","metadata":{"execution":{"iopub.status.busy":"2021-06-02T15:23:08.34694Z","iopub.status.idle":"2021-06-02T15:23:08.347349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAINCOLS = ['site','path','timestamp',\n        'acce_x', 'acce_y', 'acce_z', 'acce_accuracy',\n        'acce_uncali_x', 'acce_uncali_y', 'acce_uncali_z', 'acce_bias_x', 'acce_bias_y', 'acce_bias_z', 'acce_uncali_accuracy',\n        'magn_x', 'magn_y', 'magn_z', 'magn_accuracy',\n        'magn_uncali_x', 'magn_uncali_y', 'magn_uncali_z', 'magn_bias_x', 'magn_bias_y', 'magn_bias_z', 'magn_uncali_accuracy',\n        'gyro_x', 'gyro_y', 'gyro_z', 'gyro_accuracy',\n        'gyro_uncali_x', 'gyro_uncali_y', 'gyro_uncali_z', 'gyro_bias_x', 'gyro_bias_y', 'gyro_bias_z', 'gyro_uncali_accuracy',\n        'ahrs_x', 'ahrs_y', 'ahrs_z', 'ahrs_accuracy',\n        'wifi_ssid', 'wifi_bssid', 'wifi_rssi', 'wifi_freq', 'wifi_ls_ts',\n        'uuid', 'majorid', 'minorid', 'txpow', 'beacon_rssi', 'distance', 'macaddr', 'unix_time',\n        'way_x', 'way_y', 'floor'\n       ]\nfloors = {'1F':0, '2F':1, '3F':2, '4F':3, '5F':4, '6F':5, '7F':6, '8F':7, '9F':8, \n          'B':-1, 'B1':-1,'B2':-2, 'B3':-3, 'BF':-1, 'BM':-1, \n          'F1':0, 'F2':1, 'F3':2, 'F4':3, 'F5':4, 'F6':5,'F7':6, 'F8':7, 'F9':8, 'F10':9,\n          'G':0, 'LG1':-1, 'LG2':-2, \n          \"L1\":0,\"L2\":1,\"L3\":2,\"L4\":3,\"L5\":4,\"L6\":5,\"L7\":6,\"L8\":7,\"L9\":8,\"L10\":9,'L11':10, \n          'LM':0, 'M':0, 'P1':-1, 'P2':-2,\n          \"地下一层\":-1,\"一层\":0,\"二层\":1,\"三层\":2,\"四层\":3}","metadata":{"execution":{"iopub.status.busy":"2021-06-01T15:22:29.847255Z","iopub.execute_input":"2021-06-01T15:22:29.847668Z","iopub.status.idle":"2021-06-01T15:22:29.861956Z","shell.execute_reply.started":"2021-06-01T15:22:29.847634Z","shell.execute_reply":"2021-06-01T15:22:29.860795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#sample path\nbase = '../input/indoor-location-navigation'\npath = f'{base}/train/5cd56b5ae2acfd2d33b58546/F1/5cf396145b96c60008d35a63.txt'\n#path = '../input/indoor-location-navigation/train/5d27096c03f801723c31e5e0/F2/5db116392aaa8f000651261f.txt'\n# Read in 1 random example\nsample_file = io_f.read_data_file(path)\nprint(\"~~\" + path)\nprint(\"acce: {}\".format(sample_file.acce.shape), \"\\n\" +\n      \"acacce_uncalice: {}\".format(sample_file.acce_uncali.shape), \"\\n\" +\n      \"ahrs: {}\".format(sample_file.ahrs.shape), \"\\n\" +\n      \"gyro: {}\".format(sample_file.gyro.shape), \"\\n\" +\n      \"gyro_uncali: {}\".format(sample_file.gyro_uncali.shape), \"\\n\" +\n      \"ibeacon: {}\".format(sample_file.ibeacon.shape), \"\\n\" +\n      \"magn: {}\".format(sample_file.magn.shape), \"\\n\" +\n      \"magn_uncali: {}\".format(sample_file.magn_uncali.shape), \"\\n\" +\n      \"waypoint: {}\".format(sample_file.waypoint.shape), \"\\n\" +\n      \"wifi: {}\".format(sample_file.wifi.shape))","metadata":{"execution":{"iopub.status.busy":"2021-06-02T15:23:08.348556Z","iopub.status.idle":"2021-06-02T15:23:08.348984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_waypoint = pd.DataFrame(sample_file.waypoint, columns = ['timestamp', 'way_x', 'way_y'])\nsample_waypoint['timestamp'] = sample_waypoint['timestamp'].astype(int)\n\nprint(\"Waypoint Data\")\ndisplay(sample_waypoint)\n\nsample_acce = pd.DataFrame(sample_file.acce, columns = ['timestamp', 'acce_x', 'acce_y', 'acce_z'])\nsample_gyro = pd.DataFrame(sample_file.gyro, columns = ['timestamp', 'gyro_x', 'gyro_y', 'gyro_z'])\nsample_magn = pd.DataFrame(sample_file.magn, columns = ['timestamp', 'magn_x', 'magn_y', 'magn_z'])\nsample_ahrs = pd.DataFrame(sample_file.ahrs, columns = ['timestamp', 'ahrs_x', 'ahrs_y', 'ahrs_z'])\ncom_imu_df = sample_acce.copy()\n\nfor df in [sample_acce, sample_gyro, sample_magn, sample_ahrs]:\n    \n    assert len(com_imu_df) == len(com_imu_df.merge(df, on = 'timestamp'))\n    com_imu_df = (com_imu_df.merge(df, on = 'timestamp', suffixes = (\"\", \"_uncali\")))\n\ncom_imu_df['timestamp'] = com_imu_df['timestamp'].astype(int)\n\nprint(\"Phone Sensors Data\")\n\ndisplay(com_imu_df.head())","metadata":{"execution":{"iopub.status.busy":"2021-06-01T15:42:06.232036Z","iopub.execute_input":"2021-06-01T15:42:06.232577Z","iopub.status.idle":"2021-06-01T15:42:06.311507Z","shell.execute_reply.started":"2021-06-01T15:42:06.232535Z","shell.execute_reply":"2021-06-01T15:42:06.310595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#sample_waypoint['timestamp'] = sample_waypoint['timestamp'] - sample_waypoint.timestamp.iloc[0]\n\nimu_df = sample_acce.copy()\nimu_df['timestamp'] = com_imu_df['timestamp'] - com_imu_df.timestamp.iloc[0]\nimu_df['timestamp'] = imu_df['timestamp'].astype(int)\n\nimu_df['acce_rms'] = np.sqrt(com_imu_df.acce_x**2 + com_imu_df.acce_y**2 + com_imu_df.acce_z**2) - 9.8\nnew_acce = scipy.ndimage.median_filter(imu_df.acce_rms, 10, mode='constant', cval=0.25)\nimu_df['acce_fil'] = new_acce\n\nimu_df['mag_rms'] = np.sqrt(com_imu_df.magn_x**2 + com_imu_df.magn_y**2 + com_imu_df.magn_z**2)\nnew_mag = scipy.ndimage.median_filter(imu_df.acce_rms, 10, mode='constant', cval=0.25)\nimu_df['mag_fil'] = new_mag\n\nimu_df['mag_ang'] = np.arctan(com_imu_df['magn_y']/com_imu_df['magn_x'])\nnew_mag = scipy.ndimage.median_filter(imu_df.mag_ang, 10, mode='constant', cval=0.25)\nimu_df['mag_ang'] = new_mag\n","metadata":{"execution":{"iopub.status.busy":"2021-06-01T15:42:19.060284Z","iopub.execute_input":"2021-06-01T15:42:19.060718Z","iopub.status.idle":"2021-06-01T15:42:19.083006Z","shell.execute_reply.started":"2021-06-01T15:42:19.06068Z","shell.execute_reply":"2021-06-01T15:42:19.082029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_freq = 50\nwindow_size = 22\nlow_acce_mag = 0.6\nstep_criterion = 1\ninterval_threshold = 250\n\nacce_max = np.zeros((2,))\nacce_min = np.zeros((2,))\nacce_binarys = np.zeros((window_size,), dtype=int)\nacce_mag_pre = 0\nstate_flag = 0","metadata":{"execution":{"iopub.status.busy":"2021-06-01T15:39:38.570709Z","iopub.execute_input":"2021-06-01T15:39:38.571061Z","iopub.status.idle":"2021-06-01T15:39:38.576919Z","shell.execute_reply.started":"2021-06-01T15:39:38.571032Z","shell.execute_reply":"2021-06-01T15:39:38.575962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def init_parameters_filter(sample_freq, warmup_data, cut_off_freq=2):\n    order = 4\n    filter_b, filter_a = signal.butter(order, cut_off_freq / (sample_freq / 2), 'low', False)\n    zf = signal.lfilter_zi(filter_b, filter_a)\n    _, zf = signal.lfilter(filter_b, filter_a, warmup_data, zi=zf)\n    _, filter_zf = signal.lfilter(filter_b, filter_a, warmup_data, zi=zf)\n\n    return filter_b, filter_a, filter_zf","metadata":{"execution":{"iopub.status.busy":"2021-06-01T15:29:48.139721Z","iopub.execute_input":"2021-06-01T15:29:48.140071Z","iopub.status.idle":"2021-06-01T15:29:48.149Z","shell.execute_reply.started":"2021-06-01T15:29:48.140041Z","shell.execute_reply":"2021-06-01T15:29:48.147882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.signal import find_peaks\nheight = 0.75\nwidth = 8\npeak_indexes, pi = find_peaks(imu_df.acce_fil, height, width=width)\ninv_data_y = imu_df.acce_fil*(-1) # Tried 1/data_y but not better.\nvalley_indexes, vi = find_peaks(inv_data_y, height, width=width)","metadata":{"execution":{"iopub.status.busy":"2021-06-01T15:43:07.150027Z","iopub.execute_input":"2021-06-01T15:43:07.150392Z","iopub.status.idle":"2021-06-01T15:43:07.157629Z","shell.execute_reply.started":"2021-06-01T15:43:07.150358Z","shell.execute_reply":"2021-06-01T15:43:07.156515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def synch_peak_vall(peak, valley, timestamp):\n    done = True\n    while done:\n        for i in range(len(valley)):\n            print(i, len(peak), len(valley), end='\\r')\n            if ((i == len(peak)-1) or (i == len(valley)-1)):\n                done = False\n            try:\n                time_diff = timestamp[valley[i]] - timestamp[peak[i]]\n            except IndexError:\n                break\n            if time_diff > 600:\n                peak = np.delete(peak, i)\n                #print('delete peak', i)\n                break\n            if time_diff < 0:\n                valley = np.delete(valley, i)\n                #print('delete valley', i)\n                break\n    if len(valley)>len(peak):\n        valley = np.delete(valley, -1)\n        \n    elif len(peak)>len(valley):\n        peak = np.delete(peak, -1)\n        \n    return peak, valley","metadata":{"execution":{"iopub.status.busy":"2021-06-01T15:43:08.089244Z","iopub.execute_input":"2021-06-01T15:43:08.08965Z","iopub.status.idle":"2021-06-01T15:43:08.097785Z","shell.execute_reply.started":"2021-06-01T15:43:08.089616Z","shell.execute_reply":"2021-06-01T15:43:08.096979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"peak_indexes, valley_indexes = synch_peak_vall(peak_indexes, valley_indexes, imu_df.timestamp)","metadata":{"execution":{"iopub.status.busy":"2021-06-01T15:43:11.008853Z","iopub.execute_input":"2021-06-01T15:43:11.009455Z","iopub.status.idle":"2021-06-01T15:43:11.080827Z","shell.execute_reply.started":"2021-06-01T15:43:11.009398Z","shell.execute_reply":"2021-06-01T15:43:11.079359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(imu_df)\n#text_peak = [str(j) for j in range(len(peak_indexes))]\n#text_valley = [str(j) for j in range(len(valley_indexes))]\nfig = go.Figure()\nfig.add_trace(go.Scatter(x=imu_df.timestamp, y=imu_df.acce_rms, mode='lines', name='Acce RMS'))\nfig.add_trace(go.Scatter(x=imu_df.timestamp, y=imu_df.acce_fil, mode='lines', name='Acce LP Filtered'))\nfig.add_trace(go.Scatter(x=[imu_df.timestamp[j] for j in peak_indexes], y=[imu_df.acce_fil[j] for j in peak_indexes], mode='markers',text=text_peak, textposition=\"top center\",marker=dict(size=8, color='green'),name='Peaks'))\nfig.add_trace(go.Scatter(x=[imu_df.timestamp[j] for j in valley_indexes], y=[imu_df.acce_fil[j] for j in valley_indexes], mode='markers',text=text_valley, textposition=\"top center\", marker=dict(size=8, color='red'),name='Valleys'))\n#fig.add_trace(go.Scatter(x=imu_df.timestamp, y=imu_df.mag, mode='lines', name='yaw'))\n#for wp in sample_waypoint.timestamp:\n#    fig.add_vline(x=wp, line_width=3, line_dash=\"dash\", line_color=\"green\")\n\nfig.update_layout(\n    xaxis_title=\"Time (ms)\",\n    yaxis_title=\"Acceleration (m/s<sup>2</sup>)\",\n    #paper_bgcolor='rgba(0,0,0,0)',\n) \n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-01T16:01:04.386768Z","iopub.execute_input":"2021-06-01T16:01:04.387187Z","iopub.status.idle":"2021-06-01T16:01:04.447172Z","shell.execute_reply.started":"2021-06-01T16:01:04.387149Z","shell.execute_reply":"2021-06-01T16:01:04.446357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"peak_ts = [imu_df.timestamp[j] for j in valley_indexes]\nall_spteps_in_wp = []\nspteps_in_wp = 0\nwp = 1\npeaks = 0\nfor s in peak_ts:\n    peaks += 1\n    if wp == len(sample_waypoint.timestamp):\n        break\n    elif s < sample_waypoint.timestamp[wp]:\n        spteps_in_wp += 1\n    else:\n        all_spteps_in_wp.append(spteps_in_wp)\n        spteps_in_wp = 1\n        wp += 1\n    if peaks == len(peak_ts):\n        all_spteps_in_wp.append(spteps_in_wp)","metadata":{"execution":{"iopub.status.busy":"2021-06-01T15:29:56.576677Z","iopub.execute_input":"2021-06-01T15:29:56.577027Z","iopub.status.idle":"2021-06-01T15:29:56.587353Z","shell.execute_reply.started":"2021-06-01T15:29:56.576996Z","shell.execute_reply":"2021-06-01T15:29:56.586059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_spteps_in_wp","metadata":{"execution":{"iopub.status.busy":"2021-06-01T15:29:57.573692Z","iopub.execute_input":"2021-06-01T15:29:57.574222Z","iopub.status.idle":"2021-06-01T15:29:57.579987Z","shell.execute_reply.started":"2021-06-01T15:29:57.574172Z","shell.execute_reply":"2021-06-01T15:29:57.579012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def step_len(peak, valley, acce, mag):\n    steps = []\n    for i in range(len(peak)):\n        K = 0.68-0.37*mag[peak[i]]+0.15*(mag[peak[i]]**2)\n        s_len = K*np.sqrt(np.sqrt(acce[peak[i]]-acce[valley[i]]))\n        steps.append(s_len)\n    return steps","metadata":{"execution":{"iopub.status.busy":"2021-06-01T15:30:00.403128Z","iopub.execute_input":"2021-06-01T15:30:00.403487Z","iopub.status.idle":"2021-06-01T15:30:00.409208Z","shell.execute_reply.started":"2021-06-01T15:30:00.403456Z","shell.execute_reply":"2021-06-01T15:30:00.408235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"step_len = step_len(peak_indexes, valley_indexes, imu_df.acce_fil, imu_df.mag_fil)","metadata":{"execution":{"iopub.status.busy":"2021-06-01T15:30:00.699772Z","iopub.execute_input":"2021-06-01T15:30:00.700281Z","iopub.status.idle":"2021-06-01T15:30:00.707543Z","shell.execute_reply.started":"2021-06-01T15:30:00.700249Z","shell.execute_reply":"2021-06-01T15:30:00.706254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example = read_data_file(path)\norg_wp = example.waypoint\norg_wp = org_wp[:, 1:3]\n\n# Prepare floor_plan coresponding with our example\nsite = path.split(\"/\")[4]\nfloorNo = path.split(\"/\")[5]\nfloor_plan_filename = f'{base}/metadata/{site}/{floorNo}/floor_image.png'\n\n# Prepare width_meter & height_meter\n### (taken from the .json file)\njson_plan_filename = f'{base}/metadata/{site}/{floorNo}/floor_info.json'\nwith open(json_plan_filename) as json_file:\n    json_data = json.load(json_file)\n    \nwidth_meter = json_data[\"map_info\"][\"width\"]\nheight_meter = json_data[\"map_info\"][\"height\"]\n# Title\n# Finally, let's plot\nvisualize_trajectory(trajectory = org_wp,\n                     floor_plan_filename = floor_plan_filename,\n                     width_meter = width_meter,\n                     height_meter = height_meter,\n                    title=\"Original Waypoints\")\n","metadata":{"execution":{"iopub.status.busy":"2021-05-20T17:22:38.635381Z","iopub.execute_input":"2021-05-20T17:22:38.63573Z","iopub.status.idle":"2021-05-20T17:22:39.032208Z","shell.execute_reply.started":"2021-05-20T17:22:38.6357Z","shell.execute_reply":"2021-05-20T17:22:39.031044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_trajectory(trajectory = new_way,\n                     floor_plan_filename = floor_plan_filename,\n                     width_meter = width_meter,\n                     height_meter = height_meter,\n                    title=\"Calculated Waypoints\")","metadata":{"execution":{"iopub.status.busy":"2021-05-20T17:23:27.705604Z","iopub.execute_input":"2021-05-20T17:23:27.705952Z","iopub.status.idle":"2021-05-20T17:23:27.913345Z","shell.execute_reply.started":"2021-05-20T17:23:27.705924Z","shell.execute_reply":"2021-05-20T17:23:27.912076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#yaw = [sample_ahrs.ahrs_fil[j] for j in valley_indexes]\nmed_step = np.mean(stride_lengths[:,1])\nx = []\ny = []\nx.append(org_wp[0][0])\ny.append(org_wp[0][1])\nfor i in range(len(org_wp)-1):\n    x.append(x[i]+med_step*all_spteps_in_wp[i]*np.cos(headings[i]))\n    y.append(y[i]+med_step*all_spteps_in_wp[i]*np.sin(headings[i]))\n    \nnew_way = list(zip(x,y))\nnew_way = np.array(new_way)\nnew_way","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.888052Z","iopub.status.idle":"2021-05-20T16:03:50.888501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"med_step","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.889303Z","iopub.status.idle":"2021-05-20T16:03:50.889745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def angleFromCoordinate(x1, y1, x2, y2):\n    brng = -math.atan2((x2-x1),(y2-y1))\n    return brng+1.55","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.890586Z","iopub.status.idle":"2021-05-20T16:03:50.891004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"headings = []\nfor i in range(1,len(org_wp),1):\n    angle = angleFromCoordinate(org_wp[i-1][0], org_wp[i-1][1], org_wp[i][0], org_wp[i][1])\n    headings.append(angle)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.891852Z","iopub.status.idle":"2021-05-20T16:03:50.89232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"headings","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.893304Z","iopub.status.idle":"2021-05-20T16:03:50.893732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.degrees(1.1763003931601608)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.894549Z","iopub.status.idle":"2021-05-20T16:03:50.894978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"angle = angleFromCoordinate(org_wp[0][0], org_wp[0][1], org_wp[1][0], org_wp[1][1])","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.895879Z","iopub.status.idle":"2021-05-20T16:03:50.896336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"swp = sample_file.waypoint[0:2,:]","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:12:12.6363Z","iopub.execute_input":"2021-05-20T16:12:12.636725Z","iopub.status.idle":"2021-05-20T16:12:12.641303Z","shell.execute_reply.started":"2021-05-20T16:12:12.63668Z","shell.execute_reply":"2021-05-20T16:12:12.640151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"swp","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:12:17.349119Z","iopub.execute_input":"2021-05-20T16:12:17.349511Z","iopub.status.idle":"2021-05-20T16:12:17.355656Z","shell.execute_reply.started":"2021-05-20T16:12:17.34948Z","shell.execute_reply":"2021-05-20T16:12:17.354552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acce_datas = sample_file.acce\nahrs_datas = sample_file.ahrs\nposi_datas = sample_file.waypoint\n\nstep_timestamps, step_indexs, step_acce_max_mins = compute_f.compute_steps(acce_datas)\nheadings = compute_f.compute_headings(ahrs_datas)\nstride_lengths = compute_f.compute_stride_length(step_acce_max_mins)\nstep_headings = compute_f.compute_step_heading(step_timestamps, headings)\nrel_positions = compute_f.compute_rel_positions(stride_lengths, step_headings)\nstep_positions = compute_f.correct_positions(rel_positions, posi_datas)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T17:23:21.968967Z","iopub.execute_input":"2021-05-20T17:23:21.969583Z","iopub.status.idle":"2021-05-20T17:23:22.757132Z","shell.execute_reply.started":"2021-05-20T17:23:21.969532Z","shell.execute_reply":"2021-05-20T17:23:22.756321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = go.Figure()\n#fig.add_trace(go.Scatter(x=sample_acce.timestamp, y=sample_acce.acce, mode='lines', name='original'))\nfig.add_trace(go.Scatter(x=imu_df.timestamp, y=imu_df.acce_rms, mode='lines', name='acce'))\nfig.add_trace(go.Scatter(x=[imu_df.timestamp[j] for j in step_indexs], y=[imu_df.acce_fil[j] for j in peak_indexes], mode='markers', \n                         text=text_peak, textposition=\"top center\",marker=dict(size=8, color='green'),name='Step'))\n#fig.add_trace(go.Scatter(x=imu_df.timestamp, y=imu_df.mag, mode='lines', name='yaw'))\nfor wp in sample_waypoint.timestamp:\n    fig.add_vline(x=wp, line_width=3, line_dash=\"dash\", line_color=\"green\")\n\nfig.update_layout(\n    xaxis_title=\"Time (ms)\",\n    yaxis_title=\"Acceleration (m/s^2)\",\n    #paper_bgcolor='rgba(0,0,0,0)',\n) \n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.89974Z","iopub.status.idle":"2021-05-20T16:03:50.900151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rel_positions[:,0]","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.901006Z","iopub.status.idle":"2021-05-20T16:03:50.901498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_way = list(zip(step_positions[:,1],step_positions[:,2]))\nnew_way = np.array(new_way)\nnew_way","metadata":{"execution":{"iopub.status.busy":"2021-05-20T17:23:25.235589Z","iopub.execute_input":"2021-05-20T17:23:25.235941Z","iopub.status.idle":"2021-05-20T17:23:25.246483Z","shell.execute_reply.started":"2021-05-20T17:23:25.235913Z","shell.execute_reply":"2021-05-20T17:23:25.245252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"org_wp","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.903678Z","iopub.status.idle":"2021-05-20T16:03:50.904091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### DATA IMPORT","metadata":{}},{"cell_type":"code","source":"base_dir = \"../input/indoor-navigation-and-location-wifi-features/wifi_features\"\ntrain_dir = \"/train/*_train.csv\"\ntest_dir = \"/test/*_test.csv\"\n\n# Paths for train & test files\ntrain_paths = sorted(glob.glob(base_dir + train_dir))\ntest_paths = sorted(glob.glob(base_dir + test_dir))\n\nprint(\"Len Train Files: {}\".format(len(train_paths)), \"\\n\" +\n      \"Len Test Files: {}\".format(len(test_paths)))","metadata":{"execution":{"iopub.status.busy":"2021-05-26T14:33:20.79976Z","iopub.execute_input":"2021-05-26T14:33:20.800121Z","iopub.status.idle":"2021-05-26T14:33:20.879306Z","shell.execute_reply.started":"2021-05-26T14:33:20.80009Z","shell.execute_reply":"2021-05-26T14:33:20.878407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Prepeare PySpark","metadata":{}},{"cell_type":"code","source":"!pip install pyspark","metadata":{"execution":{"iopub.status.busy":"2021-05-26T14:33:29.249432Z","iopub.execute_input":"2021-05-26T14:33:29.249774Z","iopub.status.idle":"2021-05-26T14:34:05.611957Z","shell.execute_reply.started":"2021-05-26T14:33:29.249745Z","shell.execute_reply":"2021-05-26T14:34:05.611111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Getting PySpark Ready\nimport pyspark\nfrom pyspark.sql import SparkSession, Row, SQLContext\nfrom pyspark import SparkContext, sql, SparkConf\nfrom pyspark.ml.linalg import Vectors\nfrom pyspark.ml.feature import VectorAssembler\nfrom pyspark.sql.functions import desc, col, split\nfrom pyspark.ml.classification import LogisticRegression\nfrom pyspark.ml.regression import LinearRegression\nconf = SparkConf().setAppName(\"Indoor Location & Navigation\")\nconf = (conf.setMaster('local[*]')\n        .set('spark.executor.memory', '10G')\n        .set('spark.driver.memory', '45G')\n        .set('spark.driver.maxResultSize', '10G')\n        .set(\"spark.sql.execution.arrow.enabled\", \"true\"))\nsc = SparkContext(conf=conf)\nsqlContext = SQLContext(sc);\nspark = sqlContext.sparkSession\n## Add shape funcionality to spark df\ndef spark_shape(self):\n    return (self.count(), len(self.columns))\npyspark.sql.dataframe.DataFrame.shape = spark_shape\n# Enable Arrow-based columnar data transfers\n#spark.conf.set(\"spark.sql.execution.arrow.enabled\", \"true\")","metadata":{"execution":{"iopub.status.busy":"2021-05-26T14:34:05.613376Z","iopub.execute_input":"2021-05-26T14:34:05.613631Z","iopub.status.idle":"2021-05-26T14:34:10.691932Z","shell.execute_reply.started":"2021-05-26T14:34:05.613605Z","shell.execute_reply":"2021-05-26T14:34:10.690967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Functions","metadata":{}},{"cell_type":"code","source":"def split_data(main_train: pd.DataFrame, train_perc: float):\n    ap_qty = main_train.shape[1]-4\n    print(\"~ Number of Paths:\", main_train.shape[0], \"| Number of APs:\", ap_qty)\n    main_train = main_train.sort_values('path')\n    max_floor = main_train['f'].max()\n    min_floor = main_train['f'].min()\n    print(\"~ Floors:\", min_floor, \"to\", max_floor)\n    \n    ## Skip if use all DF to train\n    if (train_perc >= 1):\n        print(\"~ No Split\")\n        return main_train, None, int(-min_floor)\\\n    \n    ##Find Index to Split by Path\n    paths_arr = np.asarray(main_train.groupby(['path']).size())\n    total_n = len(paths_arr)\n    train_n = int(total_n*0.75)\n    test_n = int(total_n-train_n)\n    print('~ Paths to train/test:',train_n ,'/',test_n)\n    train_path = paths_arr[:train_n]\n    test_path = paths_arr[-test_n:]\n    train_index = 0\n    test_index = 0\n    for i in range(train_n):\n        train_index = train_index + train_path[i]\n    for i in range(test_n):\n        test_index = test_index + test_path[i] \n    print(\"~ Train/test rows:\", train_index, \"/\", test_index)\n    \n    ## Split by Path\n    train = main_train.iloc[:train_index]\n    test = main_train.iloc[-test_index:]\n    \n    return train, test, int(-min_floor), int(ap_qty)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T14:56:27.927337Z","iopub.execute_input":"2021-05-26T14:56:27.92796Z","iopub.status.idle":"2021-05-26T14:56:27.934642Z","shell.execute_reply.started":"2021-05-26T14:56:27.927908Z","shell.execute_reply":"2021-05-26T14:56:27.933905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def assemble_features(pdf: pd.DataFrame, floor_mod: int):\n    #print(\"~ Converting Pandas DF to Spark DF...\")\n    sdf = spark.createDataFrame(pdf)\n    \n    #print(\"~ Assembling all features...\")\n    feature_cols = sdf.columns[0:-4]\n    vect_assem = VectorAssembler(inputCols=feature_cols,outputCol=\"features\")\n    features_sdf = vect_assem.transform(sdf)\n    \n    final_sdf = features_sdf.select([\"features\",\"x\",\"y\",\"f\",\"path\"])\n    final_sdf = final_sdf.withColumn(\"f\", final_sdf.f+floor_mod)\n    \n    #final_sdf.show()\n    #print(\"~ Done.\")\n    \n    return final_sdf","metadata":{"execution":{"iopub.status.busy":"2021-05-26T14:55:39.612397Z","iopub.execute_input":"2021-05-26T14:55:39.612787Z","iopub.status.idle":"2021-05-26T14:55:39.620067Z","shell.execute_reply.started":"2021-05-26T14:55:39.612751Z","shell.execute_reply":"2021-05-26T14:55:39.618639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_floors_spark_lr(train_sdf: spark.createDataFrame):\n    #print(\"~ Floor Logistic Regression Training...\")\n    f_lr = LogisticRegression(labelCol=\"f\", featuresCol=\"features\")\n    \n    #print(\"~~ Training Floor...\")\n    f_lr_model = f_lr.fit(train_sdf)\n    \n    print(\"~ Trainings Complite.\")\n    return f_lr_model","metadata":{"execution":{"iopub.status.busy":"2021-05-26T14:35:11.63994Z","iopub.execute_input":"2021-05-26T14:35:11.640284Z","iopub.status.idle":"2021-05-26T14:35:11.645299Z","shell.execute_reply.started":"2021-05-26T14:35:11.640251Z","shell.execute_reply":"2021-05-26T14:35:11.644177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_way_spark_lr(train_sdf: spark.createDataFrame):\n    print(\"~ X Y Linear Regression Training...\")\n    x_lr = LinearRegression(labelCol=\"x\", featuresCol=\"features\")\n    y_lr = LinearRegression(labelCol=\"y\", featuresCol=\"features\")\n    \n    print(\"~~ Training Way X...\")\n    x_lr_model = x_lr.fit(train_sdf)\n    x_sum = x_lr_model.summary\n    print(\"~~~ RMSE: %f\" % x_sum.rootMeanSquaredError)\n    print(\"~~~ r2: %f\" % x_sum.r2)\n    \n    print(\"~~ Training Way Y...\")\n    y_lr_model = y_lr.fit(train_sdf)\n    y_sum = y_lr_model.summary\n    print(\"~~~ RMSE: %f\" % y_sum.rootMeanSquaredError)\n    print(\"~~~ r2: %f\" % y_sum.r2)\n    \n    print(\"~~ Trainings Complite.\")\n    return x_lr_model, y_lr_model","metadata":{"execution":{"iopub.status.busy":"2021-05-26T14:35:13.863605Z","iopub.execute_input":"2021-05-26T14:35:13.863964Z","iopub.status.idle":"2021-05-26T14:35:13.870676Z","shell.execute_reply.started":"2021-05-26T14:35:13.86393Z","shell.execute_reply":"2021-05-26T14:35:13.869612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_way_spark_ir(train_sdf: spark.createDataFrame):\n    from pyspark.ml.regression import IsotonicRegression\n    print(\"~ X Y Isotonic Regression Training...\")\n    x = IsotonicRegression(labelCol=\"x\", featuresCol=\"features\")\n    y = IsotonicRegression(labelCol=\"y\", featuresCol=\"features\")\n    \n    print(\"~~ Training Way X...\")\n    x_model = x.fit(train_sdf)\n    x_sum = model.summary\n    print(\"~~~ Coefficient Standard Errors: \" + str(x_sum.coefficientStandardErrors))\n    print(\"~~ Training Way Y...\")\n    y_model = y.fit(train_sdf)\n    y_sum = model.summary\n    print(\"~~~ Coefficient Standard Errors: \" + str(y_sum.coefficientStandardErrors))\n    \n    print(\"~~ Trainings Complite.\")\n    return x_model, y_model","metadata":{"execution":{"iopub.status.busy":"2021-05-26T14:35:15.534757Z","iopub.execute_input":"2021-05-26T14:35:15.535129Z","iopub.status.idle":"2021-05-26T14:35:15.542529Z","shell.execute_reply.started":"2021-05-26T14:35:15.535098Z","shell.execute_reply":"2021-05-26T14:35:15.541383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_way_knn(train_pdf: pd.DataFrame, ap: int):\n    from sklearn.neighbors import KNeighborsRegressor\n    \n    features = train_pdf.iloc[:,:-4]\n    x_label = train_pdf.iloc[:,-4]\n    y_label = train_pdf.iloc[:,-3]\n    \n    print(\"~ X Y KNN Regression Training...\")\n    x = KNeighborsRegressor(n_neighbors=ap)\n    y = KNeighborsRegressor(n_neighbors=ap)\n    \n    print(\"~~ Training Way X...\")\n    x_model = x.fit(features, x_label)\n    x_r2 = x.score(features, x_label)\n    print(\"~~~ r2: %f\" % x_r2)\n    \n    print(\"~~ Training Way Y...\")\n    y_model = y.fit(features, y_label)\n    y_r2 = y.score(features, y_label)\n    print(\"~~~ r2: %f\" % y_r2)\n    \n    print(\"~~ Trainings Complite.\")\n    return x_model, y_model","metadata":{"execution":{"iopub.status.busy":"2021-05-26T14:35:18.120895Z","iopub.execute_input":"2021-05-26T14:35:18.121463Z","iopub.status.idle":"2021-05-26T14:35:18.13107Z","shell.execute_reply.started":"2021-05-26T14:35:18.121412Z","shell.execute_reply":"2021-05-26T14:35:18.129886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_way_spark_dtr(train_sdf: spark.createDataFrame):\n    print(\"~ X Y Decision Tree Regression Training...\")\n    from pyspark.ml import Pipeline\n    from pyspark.ml.regression import DecisionTreeRegressor\n    from pyspark.ml.feature import VectorIndexer\n    \n    # Scale features.\n    #featureIndexer = VectorIndexer(inputCol=\"features\", outputCol=\"indexedFeatures\", maxCategories=16).fit(train_sdf)\n\n    # Train a FM model.\n    print(\"~~ Training Way X...\")\n    x = DecisionTreeRegressor(labelCol=\"x\",featuresCol=\"features\")\n    #x_pl = Pipeline(stages=[featureIndexer, x])\n    x_model = x.fit(train_sdf)\n    \n    print(\"~~ Training Way Y...\")\n    y = DecisionTreeRegressor(labelCol=\"y\",featuresCol=\"features\")\n    #y_pl = Pipeline(stages=[featureIndexer, y])\n    y_model = y.fit(train_sdf)\n    \n    print(\"~~ Trainings Complite.\")\n    return x_model, y_model","metadata":{"execution":{"iopub.status.busy":"2021-05-26T14:35:21.223237Z","iopub.execute_input":"2021-05-26T14:35:21.223653Z","iopub.status.idle":"2021-05-26T14:35:21.230801Z","shell.execute_reply.started":"2021-05-26T14:35:21.223613Z","shell.execute_reply":"2021-05-26T14:35:21.22982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_way_spark_gbt(train_sdf: spark.createDataFrame):\n    print(\"~ X Y Gradient-Boosted Tree Regression Training...\")\n    from pyspark.ml import Pipeline\n    from pyspark.ml.regression import GBTRegressor\n    from pyspark.ml.feature import VectorIndexer\n    \n    # Scale features.\n    #featureIndexer = VectorIndexer(inputCol=\"features\", outputCol=\"indexedFeatures\", maxCategories=16).fit(train_sdf)\n\n    # Train a FM model.\n    print(\"~~ Training Way X...\")\n    x = GBTRegressor(labelCol=\"x\",featuresCol=\"features\")\n    #x_pl = Pipeline(stages=[featureIndexer, x_gbt])\n    x_model = x.fit(train_sdf)\n    \n    print(\"~~ Training Way Y...\")\n    y = GBTRegressor(labelCol=\"y\",featuresCol=\"features\")\n    #y_pl = Pipeline(stages=[featureIndexer, y_gbt])\n    y_model = y.fit(train_sdf)\n    \n    print(\"~~ Trainings Complite.\")\n    return x_model, y_model","metadata":{"execution":{"iopub.status.busy":"2021-05-26T14:35:24.135274Z","iopub.execute_input":"2021-05-26T14:35:24.135646Z","iopub.status.idle":"2021-05-26T14:35:24.142761Z","shell.execute_reply.started":"2021-05-26T14:35:24.135614Z","shell.execute_reply":"2021-05-26T14:35:24.141623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def validate_spark_models(test_sdf: spark.createDataFrame, x_model, y_model, f_model):\n    from pyspark.ml.evaluation import RegressionEvaluator\n    \n    print(\"~ Validating Way X...\")\n    x_val = x_model.transform(test_sdf)\n    x_sum = RegressionEvaluator(labelCol=\"x\", predictionCol=\"prediction\", metricName=\"r2\")\n    rmse = x_sum.evaluate(x_val)\n    print(\"~~~ r2: %g\" % rmse)\n    \n    print(\"~ Validating Way Y...\")\n    y_val = y_model.transform(test_sdf)\n    y_sum = RegressionEvaluator(labelCol=\"y\", predictionCol=\"prediction\", metricName=\"r2\")\n    rmse = y_sum.evaluate(y_val)\n    print(\"~~~ r2: %g\" % rmse)\n    \n    print(\"~ Validating Floors...\")\n    f_val = f_model.transform(test_sdf)\n    f_sum = f_model.summary\n    print(\"~~~ Accuracy: %f\" % f_sum.accuracy)\n    \n    print(\"~ Validation Complite.\\n\")\n    pred_x = x_val.select('prediction').collect()\n    pred_y = y_val.select('prediction').collect()\n    pred_f = f_val.select('prediction').collect()\n    \n    return pred_x, pred_y, pred_f","metadata":{"execution":{"iopub.status.busy":"2021-05-26T14:36:33.51788Z","iopub.execute_input":"2021-05-26T14:36:33.518223Z","iopub.status.idle":"2021-05-26T14:36:33.529794Z","shell.execute_reply.started":"2021-05-26T14:36:33.518194Z","shell.execute_reply":"2021-05-26T14:36:33.528926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def validate_spark_wp(test_sdf: pd.DataFrame, x_model, y_model):\n    from pyspark.ml.evaluation import RegressionEvaluator\n    \n    print(\"~ Validating Way X...\")\n    x_val = x_model.transform(test_sdf)\n    x_sum = RegressionEvaluator(labelCol=\"x\", predictionCol=\"prediction\", metricName=\"rmse\")\n    rmse_x = x_sum.evaluate(x_val)\n    x_sum = RegressionEvaluator(labelCol=\"x\", predictionCol=\"prediction\", metricName=\"r2\")\n    r2_x = x_sum.evaluate(x_val)\n    print(\"~~~ RMSE: %g\" % rmse_x)\n    print(\"~~~ R2: %g\" % r2_x)\n    \n    print(\"~ Validating Way Y...\")\n    y_val = y_model.transform(test_sdf)\n    y_sum = RegressionEvaluator(labelCol=\"y\", predictionCol=\"prediction\", metricName=\"rmse\")\n    rmse_y = y_sum.evaluate(y_val)\n    y_sum = RegressionEvaluator(labelCol=\"y\", predictionCol=\"prediction\", metricName=\"r2\")\n    r2_y = y_sum.evaluate(y_val)\n    print(\"~~~ RMSE: %g\" % rmse_y)\n    print(\"~~~ RMSE: %g\" % r2_y)\n    \n    print(\"~ Validation Complite.\\n\")\n    pred_x = x_val.select('prediction').collect()\n    pred_y = y_val.select('prediction').collect()\n    \n    return pred_x, pred_y, rmse_x, rmse_y","metadata":{"execution":{"iopub.status.busy":"2021-05-26T14:36:34.133648Z","iopub.execute_input":"2021-05-26T14:36:34.134037Z","iopub.status.idle":"2021-05-26T14:36:34.143744Z","shell.execute_reply.started":"2021-05-26T14:36:34.134003Z","shell.execute_reply":"2021-05-26T14:36:34.142608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def validate_wp(test_sdf: spark.createDataFrame, x_model, y_model):\n    from sklearn.metrics import mean_squared_error\n    from sklearn.metrics import r2_score\n    \n    features = test_pdf.iloc[:,:-4]\n    x_label = test_pdf.iloc[:,-4]\n    y_label = test_pdf.iloc[:,-3]\n    \n    print(\"~ Validating Way X...\")\n    pred_x = x_model.predict(features)\n    \n    rmse_x = mean_squared_error(x_label, pred_x)\n    r2_x = r2_score(x_label, pred_x)\n    print(\"~~~ RMSE: %g\" % rmse_x)\n    print(\"~~~ R2: %g\" % r2_x)\n    \n    print(\"~ Validating Way Y...\")\n    pred_y = y_model.predict(features)\n    \n    rmse_y = mean_squared_error(y_label, pred_y)\n    r2_y = r2_score(y_label, pred_y)\n    print(\"~~~ RMSE: %g\" % rmse_y)\n    print(\"~~~ RMSE: %g\" % r2_y)\n    \n    print(\"~ Validation Complite.\\n\")\n    \n    return pred_x, pred_y, r2_x, r2_y","metadata":{"execution":{"iopub.status.busy":"2021-05-26T14:36:37.443838Z","iopub.execute_input":"2021-05-26T14:36:37.444235Z","iopub.status.idle":"2021-05-26T14:36:37.45177Z","shell.execute_reply.started":"2021-05-26T14:36:37.444201Z","shell.execute_reply":"2021-05-26T14:36:37.451003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def validate_spark_floor(test_sdf: spark.createDataFrame, f_model):\n    from pyspark.ml.evaluation import MulticlassClassificationEvaluator \n    \n    print(\"~ Model Summary:\")\n    trainingSummary = f_model.summary\n    train_accuracy = trainingSummary.accuracy\n    falsePositiveRate = trainingSummary.weightedFalsePositiveRate\n    truePositiveRate = trainingSummary.weightedTruePositiveRate\n    fMeasure = trainingSummary.weightedFMeasure()\n    precision = trainingSummary.weightedPrecision\n    recall = trainingSummary.weightedRecall\n    print(\"~~ Accuracy: %s | FPR: %s | TPR: %s | F-measure: %s | Precision: %s | Recall: %s\"\n      % (train_accuracy, falsePositiveRate, truePositiveRate, fMeasure, precision, recall))\n    \n    print(\"~ Validating Floors...\")\n    f_val = f_model.transform(test_sdf)\n    \n    # Select example rows to display.\n    #f_val.select(\"prediction\", \"f\", \"path\").show(5)\n\n    # Select (prediction, true label) and compute test error\n    evaluator = MulticlassClassificationEvaluator(\n        labelCol=\"f\", predictionCol=\"prediction\", metricName=\"accuracy\")\n    test_accuracy = evaluator.evaluate(f_val)\n    print(\"~~ Test Error: %g \" % (1.0 - test_accuracy))\n    print(\"~~ Test Accuracy: %g \" % test_accuracy)\n    \n    print(\"~ Validation Complite.\\n\")\n    pred_f = f_val.select('prediction').collect()\n    \n    return pred_f, train_accuracy, test_accuracy","metadata":{"execution":{"iopub.status.busy":"2021-05-26T14:36:39.932877Z","iopub.execute_input":"2021-05-26T14:36:39.933542Z","iopub.status.idle":"2021-05-26T14:36:39.942622Z","shell.execute_reply.started":"2021-05-26T14:36:39.933503Z","shell.execute_reply":"2021-05-26T14:36:39.941558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def val_to_pdf(test_pdf: pd.DataFrame, pred_x: list, pred_y: list, pred_f: list, floor_mod: int):\n    print(\"~ Prepearing Predictions Table...\")\n    pd.set_option('mode.chained_assignment',None)\n    test_pdf = test_pdf[[\"path\",\"x\",\"y\",\"f\"]]\n    if pred_x is not None:\n        test_pdf['pred_x'] = np.array(pred_x)\n    if pred_y is not None:\n        test_pdf['pred_y'] = np.array(pred_y)\n    if pred_f is not None:\n        test_pdf['pred_f'] = np.array(pred_f)\n        test_pdf['pred_f'] = test_pdf['pred_f'].astype(int) - floor_mod\n    \n    print(\"~ Predictions Table Example:\")\n    display(test_pdf)\n    \n    return test_pdf","metadata":{"execution":{"iopub.status.busy":"2021-05-26T14:36:42.229235Z","iopub.execute_input":"2021-05-26T14:36:42.22979Z","iopub.status.idle":"2021-05-26T14:36:42.236709Z","shell.execute_reply.started":"2021-05-26T14:36:42.229758Z","shell.execute_reply":"2021-05-26T14:36:42.235784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"jkjk === 1","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.926057Z","iopub.status.idle":"2021-05-20T16:03:50.926534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Floor Training","metadata":{}},{"cell_type":"code","source":"train_perc = 0.75\ntrain_accuracy = []\ntest_accuracy = []\nap_qtys = []\nfor train_path in train_paths:\n    #train_path = train_paths[2]\n    print(\"Train Path:\", train_path)\n    train = pd.read_csv(train_path, index_col=0)\n    print(\"SPLITTING DATA\")\n    train_pdf, valid_pdf, floor_mod, ap_qty = split_data(train, train_perc)\n    print(\"FEATURES ASSAMBLE\")\n    train_sdf = assemble_features(train_pdf, floor_mod)\n    valid_sdf = assemble_features(valid_pdf, floor_mod)\n    print(\"TRAINING\")\n    model_spark_gbt_x, model_spark_gbt_y = train_way_spark_gbt(train_sdf)\n    print(\"VALIDATION\")\n    pred_x, pred_y, acc1, acc2 = validate_spark_wp(valid_sdf, model_spark_gbt_x, model_spark_gbt_y)\n    predictions_pdf = val_to_pdf(valid_pdf, pred_x, pred_y, None, floor_mod)\n    train_accuracy.append(acc1)\n    test_accuracy.append(acc2)\n    ap_qtys.append(ap_qty)\n    break","metadata":{"execution":{"iopub.status.busy":"2021-05-26T15:02:29.305583Z","iopub.execute_input":"2021-05-26T15:02:29.306223Z","iopub.status.idle":"2021-05-26T15:03:34.102702Z","shell.execute_reply.started":"2021-05-26T15:02:29.306186Z","shell.execute_reply":"2021-05-26T15:03:34.101865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2021-05-26T15:02:01.844734Z","iopub.execute_input":"2021-05-26T15:02:01.845093Z","iopub.status.idle":"2021-05-26T15:02:01.872574Z","shell.execute_reply.started":"2021-05-26T15:02:01.845065Z","shell.execute_reply":"2021-05-26T15:02:01.871081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s_ap_qtys = [str(i) for i in ap_qtys]","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.930851Z","iopub.status.idle":"2021-05-20T16:03:50.931379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = go.Figure()\nfig.add_trace(go.Bar(\n    x=s_ap_qtys,\n    y=train_accuracy,\n    text=train_accuracy,\n    textposition='auto',\n    name='Train Accuracy',\n    marker_color='indianred'\n))\nfig.add_trace(go.Bar(\n    x=s_ap_qtys,\n    y=test_accuracy,\n    text=test_accuracy,\n    textposition='auto',\n    name='Test Accuracy',\n    marker_color='lightsalmon'\n))\n\n# Here we modify the tickangle of the xaxis, resulting in rotated labels.\nfig.update_layout(barmode='group', xaxis_tickangle=-45)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.932191Z","iopub.status.idle":"2021-05-20T16:03:50.93262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.style.use('seaborn-deep')\n\nx = train_accuracy\ny = test_accuracy\nbins = np.arange(start=0, stop=24, step=1)\n\nplt.hist([x, y], bins, label=['train_accuracy', 'test_accuracy'])\nplt.legend(loc='upper right')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.933425Z","iopub.status.idle":"2021-05-20T16:03:50.933833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Waypoint Training","metadata":{}},{"cell_type":"code","source":"train_path = train_paths[3]\ntrain_perc = 0.75\nprint(\"Train Path:\", train_path)\ntrain = pd.read_csv(train_path, index_col=0)\nprint(\"SPLITTING DATA\")\ntrain_pdf, test_pdf, floor_mod, ap_qty = split_data(train, train_perc)\nprint(\"FEATURES ASSAMBLE\")\ntrain_sdf = assemble_features(train_pdf, floor_mod)\ntest_sdf = assemble_features(test_pdf, floor_mod)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:14:58.131515Z","iopub.execute_input":"2021-05-20T16:14:58.131908Z","iopub.status.idle":"2021-05-20T16:15:04.771895Z","shell.execute_reply.started":"2021-05-20T16:14:58.131873Z","shell.execute_reply":"2021-05-20T16:15:04.770723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"TRAINING\")\nmodel_f = train_floors_spark_lr(train_sdf)\nprint(\"VALIDATION\")\npred_f, acc1, acc2 = validate_spark_floor(test_sdf, model_f)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:22:02.394747Z","iopub.execute_input":"2021-05-20T16:22:02.395203Z","iopub.status.idle":"2021-05-20T16:22:10.759003Z","shell.execute_reply.started":"2021-05-20T16:22:02.395145Z","shell.execute_reply":"2021-05-20T16:22:10.757755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"SKLearn TRAINING\")\nmodel_x_knn, model_y_knn = train_way_knn(train_pdf, 80)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:15:09.262578Z","iopub.execute_input":"2021-05-20T16:15:09.26292Z","iopub.status.idle":"2021-05-20T16:15:12.108375Z","shell.execute_reply.started":"2021-05-20T16:15:09.262891Z","shell.execute_reply":"2021-05-20T16:15:12.107263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"VALIDATION\")\npred_x, pred_y, x_knn_r2, y_knn_r2 = validate_wp(test_pdf, model_x_knn, model_y_knn)\nknn_predictions_pdf = val_to_pdf(test_pdf, pred_x, pred_y, pred_f, floor_mod)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:22:27.360912Z","iopub.execute_input":"2021-05-20T16:22:27.361563Z","iopub.status.idle":"2021-05-20T16:22:28.197265Z","shell.execute_reply.started":"2021-05-20T16:22:27.361528Z","shell.execute_reply":"2021-05-20T16:22:28.196228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"one_path = knn_predictions_pdf.loc[knn_predictions_pdf['path'] == '5db116392aaa8f000651261f']\n#one_path = one_path.groupby(['path','x','y','f'],as_index=False)[[\"pred_x\",\"pred_y\"]].mean()\ndisplay(one_path.sort_values('x'))","metadata":{"execution":{"iopub.status.busy":"2021-05-20T17:24:06.728131Z","iopub.execute_input":"2021-05-20T17:24:06.728505Z","iopub.status.idle":"2021-05-20T17:24:06.74056Z","shell.execute_reply.started":"2021-05-20T17:24:06.728474Z","shell.execute_reply":"2021-05-20T17:24:06.739197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"knn_predictions_pdf['error'] = np.sqrt(np.power(knn_predictions_pdf.pred_x - knn_predictions_pdf.x, 2) + np.power(knn_predictions_pdf.pred_y - knn_predictions_pdf.y, 2))","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:44:56.031116Z","iopub.execute_input":"2021-05-20T16:44:56.031511Z","iopub.status.idle":"2021-05-20T16:44:56.038806Z","shell.execute_reply.started":"2021-05-20T16:44:56.031479Z","shell.execute_reply":"2021-05-20T16:44:56.037556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"knn_predictions_pdf['error'].median()","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.939939Z","iopub.status.idle":"2021-05-20T16:03:50.940405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len_arr = np.arange(len(knn_predictions_pdf.error))\n#layout = go.Layout( autosize=False, width=600, height=600)\n\nfig = go.Figure()\nfig.add_trace(go.Bar(\n    x=len_arr,\n    y=knn_predictions_pdf.error,\n    marker_color='lightsalmon'\n))\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.941132Z","iopub.status.idle":"2021-05-20T16:03:50.941565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"SPARK TRAINING\")\nmodel_x_lr, model_y_lr = train_way_spark_lr(train_sdf)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.9425Z","iopub.status.idle":"2021-05-20T16:03:50.942919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"VALIDATION\")\npred_x, pred_y, acc1, acc2 = validate_spark_wp(test_sdf, model_x_lr, model_y_lr)\npredictions_pdf = val_to_pdf(test_pdf, pred_x, pred_y, None, floor_mod)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.943655Z","iopub.status.idle":"2021-05-20T16:03:50.944061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions_pdf","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.944805Z","iopub.status.idle":"2021-05-20T16:03:50.945236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"one_path = knn_predictions_pdf.loc[knn_predictions_pdf['path'] == '5db2a33b5741f4000680a82b']\none_path = one_path.groupby(['path','x','y','f'],as_index=False)[[\"pred_x\",\"pred_y\"]].mean()\ndisplay(one_path)\none_path['vel'] = np.sqrt(one_path.x * one_path.y)\none_path['pred_vel'] = np.sqrt(one_path.pred_x * one_path.pred_x + one_path.pred_y * one_path.pred_y)\none_path['error'] = np.sqrt(np.power(one_path.pred_x - one_path.x, 2) + np.power(one_path.pred_y - one_path.y, 2))","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:24:23.918814Z","iopub.execute_input":"2021-05-20T16:24:23.919181Z","iopub.status.idle":"2021-05-20T16:24:23.933679Z","shell.execute_reply.started":"2021-05-20T16:24:23.919137Z","shell.execute_reply":"2021-05-20T16:24:23.932392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"one_path","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.947261Z","iopub.status.idle":"2021-05-20T16:03:50.947687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len_arr = np.arange(len(one_path.vel))\nlayout = go.Layout(\n    autosize=False,\n    width=600,\n    height=600)\n\nfig = go.Figure(layout=layout)\nfig.add_trace(go.Scatter(x=len_arr, y=one_path.vel, mode='markers', name='Test Waypoints'))\nfig.add_trace(go.Scatter(x=len_arr, y=one_path.pred_vel, mode='markers', name='Pred Waypoints'))\nfig.show()\n\nfig = go.Figure(layout=layout)\nfig.add_trace(go.Bar(\n    x=len_arr,\n    y=one_path.error,\n    marker_color='lightsalmon'\n))\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:24:33.620393Z","iopub.execute_input":"2021-05-20T16:24:33.620756Z","iopub.status.idle":"2021-05-20T16:24:33.655665Z","shell.execute_reply.started":"2021-05-20T16:24:33.620726Z","shell.execute_reply":"2021-05-20T16:24:33.654896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Train Acc: \", np.array(train_accuracy).mean(), \"Test Acc:\", np.array(test_accuracy).mean())","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.949879Z","iopub.status.idle":"2021-05-20T16:03:50.950365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"VALIDATION\")\npred_x, pred_y, pred_f = validate_spark_models(test_sdf, dtr_model_x, dtr_model_y, model_f)\npredictions_pdf = val_to_pdf(test_pdf, pred_x, pred_y, pred_f, floor_mod)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.951416Z","iopub.status.idle":"2021-05-20T16:03:50.951852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#../input/indoor-location-navigation/train/5a0546857ecc773753327266/F1/5e15a2e4f4c3420006d521d8.txt\npath_train = train.loc[train['path'] == '5db2a33b5741f4000680a82b']\npath_test_sdf = assemble_features(path_train, floor_mod)\npred_x, pred_y, pred_f = validate_spark_models(path_test_sdf, fmr_model_x, model_y, model_f)\npath_predictions = val_to_pdf(path_train, pred_x, pred_y, pred_f, floor_mod)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:24:48.472764Z","iopub.execute_input":"2021-05-20T16:24:48.473105Z","iopub.status.idle":"2021-05-20T16:24:49.148274Z","shell.execute_reply.started":"2021-05-20T16:24:48.473078Z","shell.execute_reply":"2021-05-20T16:24:49.146399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trajectory = one_path[[\"x\",\"y\"]].to_numpy()\ntrajectory_trained = one_path[[\"pred_x\",\"pred_y\"]].to_numpy()\n\n# Prepare floor_plan coresponding with our example\nsite = path.split(\"/\")[4]\nfloorNo = path.split(\"/\")[5]\nfloor_plan_filename = f'{base}/metadata/{site}/{floorNo}/floor_image.png'\n\n# Prepare width_meter & height_meter\n### (taken from the .json file)\njson_plan_filename = f'{base}/metadata/{site}/{floorNo}/floor_info.json'\nwith open(json_plan_filename) as json_file:\n    json_data = json.load(json_file)\n    \nwidth_meter = json_data[\"map_info\"][\"width\"]\nheight_meter = json_data[\"map_info\"][\"height\"]\n\n# Title\n# Finally, let's plot\ntitle = \"Original Waypoint\"\norg_wp = visualize_trajectory(trajectory = trajectory,\n                     floor_plan_filename = floor_plan_filename,\n                     width_meter = width_meter,\n                     height_meter = height_meter,\n                     title = title)\ntitle = \"Predicted Waypoint\"\npre_wp = visualize_trajectory(trajectory = trajectory_trained,\n                     floor_plan_filename = floor_plan_filename,\n                     width_meter = width_meter,\n                     height_meter = height_meter,\n                     title = title)\ndisplay(org_wp, pre_wp)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:45:22.142619Z","iopub.execute_input":"2021-05-20T16:45:22.143037Z","iopub.status.idle":"2021-05-20T16:45:22.497449Z","shell.execute_reply.started":"2021-05-20T16:45:22.142984Z","shell.execute_reply":"2021-05-20T16:45:22.496364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type(org_wp)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.955071Z","iopub.status.idle":"2021-05-20T16:03:50.955518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.io as pio\n\npio.show(org_wp)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.956332Z","iopub.status.idle":"2021-05-20T16:03:50.956748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from plotly.subplots import make_subplots\nimport plotly.graph_objects as go\n\nfig = make_subplots(rows=1, cols=2)\n\nfig.add_trace(\n    go.Figure(org_wp),\n    row=1, col=1\n)\n\nfig.add_trace(\n    go.Figure(pre_wp),\n    row=1, col=2\n)\n\nfig.update_layout(height=600, width=800, title_text=\"Side By Side Subplots\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.957628Z","iopub.status.idle":"2021-05-20T16:03:50.958063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trajectory","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.958901Z","iopub.status.idle":"2021-05-20T16:03:50.959342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trajectory_trained","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.96005Z","iopub.status.idle":"2021-05-20T16:03:50.960503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Run Wi-Fi Data","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_perc = 0.75\nfor train_path, test_path in zip(train_paths, test_paths):\n    # --- Read in data ---\n    print(\"Train Path:\", train_path)\n    train = pd.read_csv(train_path, index_col=0)\n    print(\"SPLITTING DATA\")\n    train_pdf, test_pdf, floor_mod = split_data(train, train_perc)\n    print(\"TRAINING\")\n    model_x, model_y, model_f = train_with_spark(train_pdf, floor_mod)\n    print(\"VALIDATION\")\n    final_predictions = validate_spark_models(test_pdf, model_x, model_y, model_f, floor_mod)\n    \n    #train_df = train_df.sample(frac=1, random_state=10)\n    # Erase last column (which is \"site_path_timestamp\")\n    #test_df = pd.read_csv(test_path, index_col=0)\n\n    #train_size = int(len(train_df) * train_perc)\n    \n    break","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.961532Z","iopub.status.idle":"2021-05-20T16:03:50.961964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### USEFULL CODE","metadata":{}},{"cell_type":"code","source":"# = final_train_data.filter(col(\"path\") == \"5dcf884a94e49000061256be\")\n","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.962887Z","iopub.status.idle":"2021-05-20T16:03:50.963361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### X DecisionTreeRegressor","metadata":{}},{"cell_type":"code","source":"from pyspark.ml.regression import DecisionTreeRegressor\ndtr = DecisionTreeRegressor(labelCol=\"x\", featuresCol=\"features\")","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.964233Z","iopub.status.idle":"2021-05-20T16:03:50.964656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train model.  This also runs the indexers.\nx_gbt_model = dtr.fit(train_df)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.965463Z","iopub.status.idle":"2021-05-20T16:03:50.965885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Make predictions.\nx_gbt_predictions = x_gbt_model.transform(test_df)\nx_gbt_predictions.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.966791Z","iopub.status.idle":"2021-05-20T16:03:50.967232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pyspark.ml.evaluation import RegressionEvaluator\n# Select (prediction, true label) and compute test error\nevaluator = RegressionEvaluator(\n    labelCol=\"x\", predictionCol=\"prediction\", metricName=\"r2\")\nrmse = evaluator.evaluate(x_gbt_predictions)\nprint(\"Root Mean Squared Error (R2) on test data = %g\" % rmse)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.968145Z","iopub.status.idle":"2021-05-20T16:03:50.968607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Wi-Fi Fingerprinting ALL BSSID\nwifi_dir = \"../input/indoor-navigation-and-location-wifi-features\"\ntrain_dir = \"/*_train.csv\"\ntest_dir = \"/*_test.csv\"\n\n\n# Paths for train & test files\ntrain_paths = sorted(glob.glob(wifi_dir + train_dir))\ntest_paths = sorted(glob.glob(wifi_dir + test_dir))\nsample_subm = pd.read_csv('../input/indoor-location-navigation/sample_submission.csv',\n                          index_col=0)\n\nprint(\"Len Train Files: {}\".format(len(train_paths)), \"\\n\" +\n      \"Len Test Files: {}\".format(len(test_paths)))","metadata":{"execution":{"iopub.status.busy":"2021-05-20T16:03:50.969379Z","iopub.status.idle":"2021-05-20T16:03:50.969792Z"},"trusted":true},"execution_count":null,"outputs":[]}]}