{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport os\nimport glob\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport itertools\nimport seaborn as sns\nfrom pandas_profiling import ProfileReport","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-20T15:13:43.862316Z","iopub.execute_input":"2021-11-20T15:13:43.862814Z","iopub.status.idle":"2021-11-20T15:13:43.867200Z","shell.execute_reply.started":"2021-11-20T15:13:43.862784Z","shell.execute_reply":"2021-11-20T15:13:43.866157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Read Data**\n#### reading in the data files","metadata":{}},{"cell_type":"code","source":"os.listdir('/kaggle/input/indoor-location-navigation')","metadata":{"execution":{"iopub.status.busy":"2021-11-20T15:13:43.868600Z","iopub.execute_input":"2021-11-20T15:13:43.869052Z","iopub.status.idle":"2021-11-20T15:13:43.884141Z","shell.execute_reply.started":"2021-11-20T15:13:43.869023Z","shell.execute_reply":"2021-11-20T15:13:43.883348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dir = \"../input/indoor-location-navigation/train\"\ntest_dir = \"../input/indoor-location-navigation/test\"\nmeta_dir = \"../input/indoor-location-navigation/metadata\"\nss = \"../input/indoor-location-navigation/sample_submission.csv\"","metadata":{"execution":{"iopub.status.busy":"2021-11-20T15:13:43.885943Z","iopub.execute_input":"2021-11-20T15:13:43.886510Z","iopub.status.idle":"2021-11-20T15:13:43.890400Z","shell.execute_reply.started":"2021-11-20T15:13:43.886461Z","shell.execute_reply":"2021-11-20T15:13:43.889814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_names = ['Time', 'Type'] + ['reading_'+str(x)+'_' for x in range(1,9)]\ntest_names = ['Time', 'Type'] + ['reading_'+str(x)+'_' for x in range(1,9)]\n\ntrain_files = glob.glob(os.path.join(train_dir, \"**/*.txt\"), recursive=True)\ntest_files = glob.glob(os.path.join(test_dir, \"**/*.txt\"), recursive=True)\ntrain_files = train_files[0:9]\ntest_files = test_files[0:9]\n\ndef find_floor(FloorName):\n    floor_type = FloorName[:1]\n    floor_number = int(FloorName[1:])\n    if floor_type == 'B':\n        floor_level = -floor_number\n    elif floor_type == 'F':\n        floor_level = floor_number-1\n    else:\n        floor_level = -99\n    return floor_level\n\ndef read_files(files, names):\n    full_df = pd.DataFrame(columns= ['WalkID', 'SiteID', 'Floor']+names)\n    for file in files:\n        file_df = pd.read_csv(file, sep='\\t', comment='#', header=None, names=names) \n        file_df['WalkID'] = files.index(file)\n        deets = train_files[0].split(\"/\")\n        file_df['SiteID'] = deets[4]\n        file_df['Floor'] = find_floor(deets[5])\n        full_df = full_df.append(file_df)\n    full_df.replace(0, np.nan, inplace=True)\n    return full_df\n\nraw_train_df = read_files(train_files, train_names)\nraw_test_df = read_files(test_files, test_names)\nsample_submission = pd.read_csv(ss)\n\n\n# additional data:\n# start time\n# floor name\n# phone details\n# sensor details","metadata":{"execution":{"iopub.status.busy":"2021-11-20T15:13:43.891855Z","iopub.execute_input":"2021-11-20T15:13:43.892314Z","iopub.status.idle":"2021-11-20T15:14:04.099377Z","shell.execute_reply.started":"2021-11-20T15:13:43.892267Z","shell.execute_reply":"2021-11-20T15:14:04.098513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#file_df = pd.read_csv(train_files[0], sep='\\t', comment='#', header=None, names=train_names)\n#file_df[file_df['Type']=='TYPE_WAYPOINT']","metadata":{"execution":{"iopub.status.busy":"2021-11-20T15:14:04.100978Z","iopub.execute_input":"2021-11-20T15:14:04.101370Z","iopub.status.idle":"2021-11-20T15:14:04.105689Z","shell.execute_reply.started":"2021-11-20T15:14:04.101328Z","shell.execute_reply":"2021-11-20T15:14:04.104689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#with open(train_files[0], \"r\") as fh:\n#    for line in fh.readlines():\n#        print(line)\n#    fh.close()","metadata":{"execution":{"iopub.status.busy":"2021-11-20T15:14:04.107073Z","iopub.execute_input":"2021-11-20T15:14:04.107649Z","iopub.status.idle":"2021-11-20T15:14:04.120317Z","shell.execute_reply.started":"2021-11-20T15:14:04.107606Z","shell.execute_reply":"2021-11-20T15:14:04.119607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **EDA**\n#### exploring the data available as well as it's health and completeness","metadata":{}},{"cell_type":"code","source":"floor_images = glob.glob(os.path.join(meta_dir, \"**/*.png\"), recursive=True)\nfloor_info = glob.glob(os.path.join(meta_dir, \"**/floor_info.json\"), recursive=True)\nGeoMaps = glob.glob(os.path.join(meta_dir, \"**/geojson_map.json\"), recursive=True)\n                                      \nprint(\"Number of Floor Images in Meta Data: \", len(floor_images))\nprint(\"Number of Floor Info(in JSON) in Meta Data: \", len(floor_info))\nprint(\"Number of Geo Map (in JSON) in Meta Data: \", len(GeoMaps))","metadata":{"execution":{"iopub.status.busy":"2021-11-20T15:14:04.122413Z","iopub.execute_input":"2021-11-20T15:14:04.122848Z","iopub.status.idle":"2021-11-20T15:14:12.010841Z","shell.execute_reply.started":"2021-11-20T15:14:04.122806Z","shell.execute_reply":"2021-11-20T15:14:12.008741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for _ in range(5):\n    img = Image.open(floor_images[np.random.randint(0, len(floor_images))])\n    display(img)","metadata":{"execution":{"iopub.status.busy":"2021-11-20T15:14:12.013069Z","iopub.execute_input":"2021-11-20T15:14:12.013613Z","iopub.status.idle":"2021-11-20T15:14:12.533874Z","shell.execute_reply.started":"2021-11-20T15:14:12.013548Z","shell.execute_reply":"2021-11-20T15:14:12.532778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_txt(txt_path):\n    # ignore lines starting with # because they contain meta-data sort of thing\n    with open(txt_path, 'r') as fh:\n        unique_keys = []\n        for line in fh.readlines():\n            if line.startswith(\"#\"):\n                dummy = line.split(\"\\n\")[0].split(\"\\t\")\n                unique_keys.extend(list(map(lambda x: '' if x==\"#\" else x, dummy)))\n            else:\n                pass\n        fh.close()\n    return unique_keys\n    pass\n\nread_txt(train_files[0])","metadata":{"execution":{"iopub.status.busy":"2021-11-20T15:14:12.535384Z","iopub.execute_input":"2021-11-20T15:14:12.535797Z","iopub.status.idle":"2021-11-20T15:14:12.550062Z","shell.execute_reply.started":"2021-11-20T15:14:12.535754Z","shell.execute_reply":"2021-11-20T15:14:12.548859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_train_df.groupby('Type').agg({'Time': 'count'})","metadata":{"execution":{"iopub.status.busy":"2021-11-20T15:14:12.551645Z","iopub.execute_input":"2021-11-20T15:14:12.552305Z","iopub.status.idle":"2021-11-20T15:14:12.592867Z","shell.execute_reply.started":"2021-11-20T15:14:12.552260Z","shell.execute_reply":"2021-11-20T15:14:12.591660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"null_counts = raw_train_df.isnull().groupby([raw_train_df['Type']]).sum().astype(int)\nprint (null_counts)\n# should be no nulls for 'TYPE_MAGNETIC_FIELD_UNCALIBRATED' ?\n#raw_train_df[raw_train_df['Type']=='TYPE_MAGNETIC_FIELD_UNCALIBRATED']","metadata":{"execution":{"iopub.status.busy":"2021-11-20T15:14:12.594444Z","iopub.execute_input":"2021-11-20T15:14:12.594889Z","iopub.status.idle":"2021-11-20T15:14:12.722929Z","shell.execute_reply.started":"2021-11-20T15:14:12.594846Z","shell.execute_reply":"2021-11-20T15:14:12.722179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"null_counts = raw_test_df.isnull().groupby([raw_test_df['Type']]).sum().astype(int)\nprint (null_counts)","metadata":{"execution":{"iopub.status.busy":"2021-11-20T15:14:12.724146Z","iopub.execute_input":"2021-11-20T15:14:12.724726Z","iopub.status.idle":"2021-11-20T15:14:13.130666Z","shell.execute_reply.started":"2021-11-20T15:14:12.724683Z","shell.execute_reply":"2021-11-20T15:14:13.129684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_test_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-20T15:14:13.132400Z","iopub.execute_input":"2021-11-20T15:14:13.132993Z","iopub.status.idle":"2021-11-20T15:14:13.151703Z","shell.execute_reply.started":"2021-11-20T15:14:13.132950Z","shell.execute_reply":"2021-11-20T15:14:13.150680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-20T15:14:13.153037Z","iopub.execute_input":"2021-11-20T15:14:13.153566Z","iopub.status.idle":"2021-11-20T15:14:13.169704Z","shell.execute_reply.started":"2021-11-20T15:14:13.153520Z","shell.execute_reply":"2021-11-20T15:14:13.168991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Data Prep**\n#### wrangling the data into a format that we can use to build a model","metadata":{}},{"cell_type":"markdown","source":"| ID Variables | Target Variables | Prediction Variables |\n| --- | --- | --- |\n| siteID    | Floor       | TYPE_ACCELEROMETER_1 |\n| walkID    | x           | TYPE_ACCELEROMETER_UNCALIBRATED_1 | \n| timestamp | y           | TYPE_BEACON_1 |\n|           |             | TYPE_GYROSCOPE_1 |\n|           |             | TYPE_GYROSCOPE_UNCALIBRATED_1 |\n|           |             | TYPE_MAGNETIC_FIELD_1 |\n|           |             | TYPE_MAGNETIC_FIELD_UNCALIBRATED_1 |\n|           |             | TYPE_ROTATION_VECTOR_1 |\n|           |             | TYPE_WIFI_1 |\n|           |             | (projected_x) |\n|           |             | (projected_y) |","metadata":{}},{"cell_type":"markdown","source":"## Data Cleaning","metadata":{}},{"cell_type":"markdown","source":"The waypoints and readings are gathered at different times, so will split these apart as different data sets and interpolate the readings at each waypoint","metadata":{}},{"cell_type":"code","source":"# create clean dataset of paths\n\nwaypoint_df = raw_train_df[raw_train_df['Type'] == 'TYPE_WAYPOINT']\nwaypoint_df['x'] =  pd.to_numeric(waypoint_df['reading_1_'])\nwaypoint_df['y'] = pd.to_numeric(waypoint_df['reading_2_'])\nwaypoint_df = waypoint_df[['WalkID', 'SiteID', 'Time', 'Floor', 'x', 'y']]","metadata":{"execution":{"iopub.status.busy":"2021-11-20T15:14:13.170683Z","iopub.execute_input":"2021-11-20T15:14:13.171060Z","iopub.status.idle":"2021-11-20T15:14:13.224808Z","shell.execute_reply.started":"2021-11-20T15:14:13.171029Z","shell.execute_reply":"2021-11-20T15:14:13.223950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# display example path\n\nexample_id = 6\nexample_path = waypoint_df[waypoint_df['WalkID']==example_id][['x', 'y']].to_numpy()\n\nstart_x = example_path[0, 0]\nstart_y = example_path[0, 1]   \nend_x = example_path[len(example_path)-1, 0]\nend_y = example_path[len(example_path)-1, 1]\n\nfig = plt.figure()\nax = fig.add_subplot(111)\nplt.plot(start_x, start_y, 'go', end_x, end_y, 'ro', example_path[:, 0], example_path[:, 1])\nax.annotate('start', (example_path[0, 0]-0.2, example_path[0, 1]-0.2))\nax.annotate('end', (example_path[len(example_path)-1, 0]+0.2, example_path[len(example_path)-1, 1]+0.2))","metadata":{"execution":{"iopub.status.busy":"2021-11-20T15:14:13.226034Z","iopub.execute_input":"2021-11-20T15:14:13.226311Z","iopub.status.idle":"2021-11-20T15:14:13.410496Z","shell.execute_reply.started":"2021-11-20T15:14:13.226284Z","shell.execute_reply":"2021-11-20T15:14:13.409547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create clean dataset of readings\n\n# raw_train_df.dtypes\n# raw_train_df[raw_train_df['Type']=='TYPE_BEACON'].head(5)\n# train_df_clean.iloc[[738,739,740]]\n# many reading seem to contain SiteIDs or other non numeric values\n\ntrain_df_clean = raw_train_df[raw_train_df['Type'] != 'TYPE_WAYPOINT']\n\n# Convert sensor readings to decimal floats\n\n# TYPE_WIFI readings 1 and 2 are strings for some reason\ntrain_df_clean.loc[train_df_clean['Type']=='TYPE_WIFI', 'reading_1_'] = np.NaN\ntrain_df_clean.loc[train_df_clean['Type']=='TYPE_WIFI', 'reading_2_'] = np.NaN\n# TYPE_BEACON readings 1, 2, 3 and 7 are strings for some reason\ntrain_df_clean.loc[train_df_clean['Type']=='TYPE_BEACON', 'reading_1_'] = np.NaN\ntrain_df_clean.loc[train_df_clean['Type']=='TYPE_BEACON', 'reading_2_'] = np.NaN\ntrain_df_clean.loc[train_df_clean['Type']=='TYPE_BEACON', 'reading_3_'] = np.NaN\ntrain_df_clean.loc[train_df_clean['Type']=='TYPE_BEACON', 'reading_7_'] = np.NaN\ntrain_df_clean.loc[train_df_clean['Type']=='TYPE_BEACON', 'reading_8_'] = np.NaN\n\ntrain_df_clean = train_df_clean.astype({'reading_1_': 'float64', 'reading_2_': 'float64', 'reading_3_': 'float64', 'reading_4_': 'float64', 'reading_5_': 'float64', 'reading_6_': 'float64', 'reading_7_': 'float64', 'reading_8_': 'float64'})\ntrain_df_clean.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-11-20T15:14:13.411548Z","iopub.execute_input":"2021-11-20T15:14:13.411829Z","iopub.status.idle":"2021-11-20T15:14:13.740392Z","shell.execute_reply.started":"2021-11-20T15:14:13.411801Z","shell.execute_reply":"2021-11-20T15:14:13.739447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df_sparce = pd.pivot_table(train_df_clean, \n                          index=['WalkID', 'SiteID', 'Time', 'Floor'], \n                          columns=['Type'], \n                          values=['reading_1_', 'reading_2_', 'reading_3_', 'reading_4_', 'reading_5_', 'reading_6_', 'reading_7_', 'reading_8_'], \n                          aggfunc={'reading_1_': np.sum, 'reading_2_': np.sum, 'reading_3_': np.sum, 'reading_4_' : np.sum, 'reading_5_': np.sum, 'reading_6_': np.sum, 'reading_7_': np.sum, 'reading_8_': np.sum})\ntrain_df_sparce.reset_index(inplace=True)\ntrain_df_sparce.columns = [''.join(col).strip() for col in train_df_sparce.columns.values]","metadata":{"execution":{"iopub.status.busy":"2021-11-20T15:14:13.742939Z","iopub.execute_input":"2021-11-20T15:14:13.743207Z","iopub.status.idle":"2021-11-20T15:14:14.155762Z","shell.execute_reply.started":"2021-11-20T15:14:13.743182Z","shell.execute_reply":"2021-11-20T15:14:14.154914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"null_counts = train_df_sparce.isnull().sum().astype(int)\nprint(null_counts)\n#with pd.option_context('display.max_rows', None, 'display.max_columns',  None):  # more options can be specified also\n#    print(null_counts)","metadata":{"execution":{"iopub.status.busy":"2021-11-20T15:14:14.157529Z","iopub.execute_input":"2021-11-20T15:14:14.157934Z","iopub.status.idle":"2021-11-20T15:14:14.172119Z","shell.execute_reply.started":"2021-11-20T15:14:14.157892Z","shell.execute_reply":"2021-11-20T15:14:14.171062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#TYPE_WIFI and TYPE_BEACON have too many nulls for all readings (why?) - remove these columns\n\nnan_columns = ['reading_1_TYPE_BEACON', 'reading_1_TYPE_WIFI', 'reading_2_TYPE_BEACON', 'reading_2_TYPE_WIFI', 'reading_3_TYPE_BEACON', 'reading_3_TYPE_WIFI', 'reading_4_TYPE_BEACON', 'reading_4_TYPE_WIFI', 'reading_5_TYPE_BEACON', 'reading_5_TYPE_WIFI', 'reading_6_TYPE_BEACON', 'reading_6_TYPE_WIFI', 'reading_7_TYPE_BEACON', 'reading_7_TYPE_WIFI', 'reading_8_TYPE_BEACON', 'reading_8_TYPE_WIFI']\ntrain_df_sparce.drop(nan_columns, axis=1, inplace=True)\ntrain_df_sparce.rename(columns={\"Time\": \"Reading_Time\"}, inplace=True)\ntrain_df_sparce.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-20T15:14:14.173351Z","iopub.execute_input":"2021-11-20T15:14:14.173661Z","iopub.status.idle":"2021-11-20T15:14:14.216835Z","shell.execute_reply.started":"2021-11-20T15:14:14.173631Z","shell.execute_reply":"2021-11-20T15:14:14.215831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Readings Interpolation","metadata":{}},{"cell_type":"code","source":"interpolation_data = waypoint_df.merge(train_df_sparce, on=['WalkID', 'SiteID', 'Floor'], how='left')\n#interpolation_data['ID'] = interpolation_data['WalkID'].map(str) + '_' + interpolation_data['SiteID'].map(str)  + '_' + interpolation_data['Floor'].map(str) \n\ninterpolation_data_prev = interpolation_data[interpolation_data['Reading_Time'] <= interpolation_data['Time']]\ninterpolation_data_prev_agg = interpolation_data_prev[['WalkID', 'SiteID', 'Floor', 'Time', 'Reading_Time']].groupby(['WalkID', 'SiteID', 'Floor', 'Time']).agg('max')\ninterpolation_data_prev = interpolation_data_prev.merge(interpolation_data_prev_agg, on=['WalkID', 'SiteID', 'Floor', 'Time', 'Reading_Time'], how='inner')\n\ninterpolation_data_after = interpolation_data[interpolation_data['Reading_Time'] >= interpolation_data['Time']]\ninterpolation_data_after_agg = interpolation_data_after[['WalkID', 'SiteID', 'Floor', 'Time', 'Reading_Time']].groupby(['WalkID', 'SiteID', 'Floor', 'Time']).agg('min')\ninterpolation_data_after = interpolation_data_after.merge(interpolation_data_after_agg, on=['WalkID', 'SiteID', 'Floor', 'Time', 'Reading_Time'], how='inner')","metadata":{"execution":{"iopub.status.busy":"2021-11-20T15:14:14.218425Z","iopub.execute_input":"2021-11-20T15:14:14.218866Z","iopub.status.idle":"2021-11-20T15:14:14.434786Z","shell.execute_reply.started":"2021-11-20T15:14:14.218825Z","shell.execute_reply":"2021-11-20T15:14:14.433663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"interpolation_data = interpolation_data_prev.merge(interpolation_data_after, on=['WalkID', 'SiteID', 'Floor', 'Time', 'x', 'y'], how='outer')\ninterpolation_data['interpolation_fraction'] = [np.NaN if np.isnan(i) or np.isnan(m) or np.isnan(j) else 1 if j==i else (m-i)/(j-i) for i,m,j in zip(interpolation_data['Reading_Time_x'], interpolation_data['Time'], interpolation_data['Reading_Time_y'])]\nreading_type_names = ['TYPE_ACCELEROMETER', 'TYPE_ACCELEROMETER_UNCALIBRATED', 'TYPE_BEACON', 'TYPE_GYROSCOPE', 'TYPE_GYROSCOPE_UNCALIBRATED', 'TYPE_MAGNETIC_FIELD', 'TYPE_MAGNETIC_FIELD_UNCALIBRATED', 'TYPE_ROTATION_VECTOR', 'TYPE_WIFI']\nreading_numbers = ['reading_1_', 'reading_2_', 'reading_3_']#, 'reading_4_', 'reading_5_', 'reading_6_', 'reading_7_', 'reading_8_']\nreading_names = [i[0]+i[1] for i in list(itertools.product(reading_numbers, reading_type_names))]\nfor nan_column in nan_columns:\n    if nan_column in reading_names:\n        reading_names.remove(nan_column)\nfor reading_name in reading_names:\n    prev_readings = interpolation_data[reading_name+'_x']\n    after_readings = interpolation_data[reading_name+'_y'] \n    interpolation_data[reading_name] = [j if np.isnan(i) else i if np.isnan(j) is None else i+((j-i)*f) for i,j,f in zip(prev_readings, after_readings, interpolation_data['interpolation_fraction'])]\n","metadata":{"execution":{"iopub.status.busy":"2021-11-20T15:14:14.436307Z","iopub.execute_input":"2021-11-20T15:14:14.436736Z","iopub.status.idle":"2021-11-20T15:14:14.478823Z","shell.execute_reply.started":"2021-11-20T15:14:14.436693Z","shell.execute_reply":"2021-11-20T15:14:14.478002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = interpolation_data[['WalkID', 'SiteID', 'Time', 'Floor', 'x', 'y']+reading_names]\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-20T15:14:14.480220Z","iopub.execute_input":"2021-11-20T15:14:14.480864Z","iopub.status.idle":"2021-11-20T15:14:14.522988Z","shell.execute_reply.started":"2021-11-20T15:14:14.480809Z","shell.execute_reply":"2021-11-20T15:14:14.522150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Path Projection","metadata":{}},{"cell_type":"code","source":"waypoint_projection_df = waypoint_df\n\ndef projected_coordinate(prev_prev_point, prev_point, prev_prev_time, prev_time, new_time):\n    if prev_point == pd.NaN:\n        new_point = 0\n    elif prev_prev_point == pd.NaN:\n        new_point = prev_point\n    else:\n        speed = (prev_point-prev_prev_point) / (prev_time-prev_prev_time)\n        new_point = prev_point + speed*(new_time-prev_time)\n        \n#waypoint_df['projected_x'] = \n#waypoint_df['projected_y'] = ","metadata":{"execution":{"iopub.status.busy":"2021-11-20T15:19:58.407078Z","iopub.execute_input":"2021-11-20T15:19:58.407594Z","iopub.status.idle":"2021-11-20T15:19:58.412531Z","shell.execute_reply.started":"2021-11-20T15:19:58.407546Z","shell.execute_reply":"2021-11-20T15:19:58.411755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Additional Ideas:**\nproject path,\nlook at site_path as a whole,\nproject distance from sensor\n\nFor projecting path:\nprevious_x,\nprevious_y,\nprevious_direction,\nprevious_speed\n-->\nprojected_x,\nprojected_y","metadata":{}},{"cell_type":"markdown","source":"# **Data Mining**\n#### exploring the data correlations to inform the model build","metadata":{}},{"cell_type":"code","source":"from matplotlib.pyplot import figure\nfigure(figsize=(20, 5), dpi=80)\nboxplot = train_df.boxplot(column=reading_names)\nboxplot = plt.xticks(rotation=45, ha=\"right\", fontsize=8)","metadata":{"execution":{"iopub.status.busy":"2021-11-20T15:26:03.906480Z","iopub.execute_input":"2021-11-20T15:26:03.906975Z","iopub.status.idle":"2021-11-20T15:26:04.460814Z","shell.execute_reply.started":"2021-11-20T15:26:03.906945Z","shell.execute_reply":"2021-11-20T15:26:04.460013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# select the metrics and factors to correlate\nmetrics = ['Floor', 'x', 'y']\nfactors = reading_names\n#train_df.dtypes","metadata":{"execution":{"iopub.status.busy":"2021-11-20T15:27:21.651484Z","iopub.execute_input":"2021-11-20T15:27:21.651836Z","iopub.status.idle":"2021-11-20T15:27:21.655733Z","shell.execute_reply.started":"2021-11-20T15:27:21.651807Z","shell.execute_reply":"2021-11-20T15:27:21.654654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# calculate the correlation coefficient map \ncorr_matrix = train_df.corr(method ='spearman')\ncorr_matrix = corr_matrix[metrics].filter(factors, axis = 0)\ncm = sns.diverging_palette(20, 133, sep=20, as_cmap=True)\ncorr_matrix.style.background_gradient(cmap=cm)","metadata":{"execution":{"iopub.status.busy":"2021-11-20T15:27:33.891911Z","iopub.execute_input":"2021-11-20T15:27:33.892518Z","iopub.status.idle":"2021-11-20T15:27:33.962907Z","shell.execute_reply.started":"2021-11-20T15:27:33.892478Z","shell.execute_reply":"2021-11-20T15:27:33.961906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot the correlations\nfor factor in factors:\n    fig, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(20,3))\n    axs = [ax1, ax2, ax3]\n    x_data = train_df[factor]\n    for i in range(len(metrics)):\n        metric = metrics[i]\n        y_data = train_df[metric]\n        axs[i].set_title(metric+' vs '+factor)\n        axs[i].plot(x_data, y_data, 'o')","metadata":{"execution":{"iopub.status.busy":"2021-11-20T15:28:28.665360Z","iopub.execute_input":"2021-11-20T15:28:28.665722Z","iopub.status.idle":"2021-11-20T15:28:36.489572Z","shell.execute_reply.started":"2021-11-20T15:28:28.665693Z","shell.execute_reply":"2021-11-20T15:28:36.488656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prof = ProfileReport(train_df)\nprof.to_file(output_file='output.html')\ndisplay(prof)","metadata":{"execution":{"iopub.status.busy":"2021-11-20T15:33:21.403042Z","iopub.execute_input":"2021-11-20T15:33:21.403530Z","iopub.status.idle":"2021-11-20T15:33:22.499886Z","shell.execute_reply.started":"2021-11-20T15:33:21.403499Z","shell.execute_reply":"2021-11-20T15:33:22.498935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Model Builds**\n#### building the predictive model(s)","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Model Implimentation**\n#### applying the predictive model(s) to the test data","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}