{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Overview\n\n[Dataset is linked here](https://www.kaggle.com/ravishah1/indoor-location-navigation-sensor-data) - this notebook is the code to how I made the dataset\n\nI have a discussion post up about this dataset with a bunch of info explaining this dataset that I suggest reading. I use multiprocessing Pool in order to speed up the process of creating this dataset. I learned a bunch about feature generation from [this notebook](https://www.kaggle.com/higepon/generate-wifi-features-5-times-faster). \n\nThings to note: this dataset is intended to not include WiFi Features but instead other types of data calculated from sensors such as acce and ahrs using the GitHub - [see my tutorial here](https://www.kaggle.com/ravishah1/understanding-the-indoor-loc-github-data-eda) about using the GitHub for this competition. As a result, this is not a good starting dataset as something with wifi is better; however, it may still be useful for fine tuning you results. Also, you can probably get better results by using postprocessing rather than this dataset, I myself am still deciding if I want to use these features. I created and published this dataset just to help anyone who can find a good way to use it. \n\nGood Luck and if you end up using this dataset, comment your results.","metadata":{}},{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport glob\nimport json\nimport os\nimport sys\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom dataclasses import dataclass\nimport scipy.signal as signal\n\nimport multiprocessing\nfrom multiprocessing import Pool\nfrom pathlib import Path\n\nimport time\nimport psutil\nimport math\nfrom tqdm import tqdm\nfrom dataclasses import dataclass\nfrom contextlib import contextmanager\nfrom math import floor, ceil\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!git clone --depth 1 https://github.com/location-competition/indoor-location-competition-20 indoor_location_competition_20","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from indoor_location_competition_20.io_f import read_data_file\n\nfrom indoor_location_competition_20.compute_f import split_ts_seq\nfrom indoor_location_competition_20.compute_f import correct_trajectory\nfrom indoor_location_competition_20.compute_f import correct_positions\nfrom indoor_location_competition_20.compute_f import init_parameters_filter\nfrom indoor_location_competition_20.compute_f import get_rotation_matrix_from_vector\nfrom indoor_location_competition_20.compute_f import get_orientation\nfrom indoor_location_competition_20.compute_f import compute_steps\nfrom indoor_location_competition_20.compute_f import compute_stride_length\nfrom indoor_location_competition_20.compute_f import compute_headings\nfrom indoor_location_competition_20.compute_f import compute_step_heading\nfrom indoor_location_competition_20.compute_f import compute_rel_positions\nfrom indoor_location_competition_20.compute_f import compute_step_positions","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generate Dataset","metadata":{}},{"cell_type":"markdown","source":"### Helpers","metadata":{}},{"cell_type":"code","source":"\"\"\"Functions\"\"\"\n@contextmanager\ndef timer(name: str):\n    t0 = time.time()\n    p = psutil.Process(os.getpid())\n    m0 = p.memory_info()[0] / 2. ** 30\n    try:\n        yield\n    finally:\n        m1 = p.memory_info()[0] / 2. ** 30\n        delta = m1 - m0\n        sign = '+' if delta >= 0 else '-'\n        delta = math.fabs(delta)\n        print(f\"[{m1:.1f}GB({sign}{delta:.1f}GB): {time.time() - t0:.3f}sec] {name}\", file=sys.stderr)\n        \ndef time_float_to_str(time: float):\n    return str(int(time))\n        \n\"\"\"Variables\"\"\"\n# Sample Submission Important Info\nss = pd.read_csv(\"../input/indoor-location-navigation/sample_submission.csv\")\n\nsub_df = ss[\"site_path_timestamp\"].apply(\n    lambda x: pd.Series(x.split(\"_\")))\nsub_df.columns = ['site', 'path', 'timestamp']\n\nsites = sub_df.site.unique()\nbuilding_dfs = [building_df for _, building_df in sub_df.groupby('site')]\n\n# Paths\ndata_path = \"../input/indoor-location-navigation\"\ntrain_path = \"../input/indoor-location-navigation/train\"\ntest_path = \"../input/indoor-location-navigation/test\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(building_dfs[0].path.unique())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Database Generation Using GitHub read_data_file","metadata":{}},{"cell_type":"code","source":"def generate_db(file_path: str):\n    \"\"\" This function calls the read_data_file in the GitHub to create a database to work from. \"\"\"\n    db = read_data_file(file_path)\n    \n    acce = db.acce\n    ahrs = db.ahrs\n    gyro = db.gyro\n    magn = db.magn\n    posi = db.waypoint\n    return acce, ahrs, gyro, magn, posi","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Sensor DataFrame Generation","metadata":{}},{"cell_type":"code","source":"def generate_sensor_df(file_path: str, floor: str, path: str, train: bool):\n    \"\"\" \n    This function generates a dataframe of sensor information computed with help from the GitHub. \n    It also returns the waypoint dataframe\n    \"\"\"\n    \n    acce, ahrs, gyro, magn, posi = generate_db(file_path)\n    \n    # Waypoint DF \n    if train:\n        posi_df = pd.DataFrame(posi, columns=['timestamp','x','y'])\n        posi_df[\"timestamp\"] = posi_df[\"timestamp\"].apply(time_float_to_str)\n        posi_df[\"path\"] = path\n    \n    # Sensor DF\n    step_timestamps, step_indexs, step_acce_max_mins = compute_steps(acce)    \n    sensor_df = pd.DataFrame(step_acce_max_mins, index=step_indexs)\n    sensor_df.columns = [\"timestamp\", \"acce_max\", \"acce_min\", \"acce_std\"]\n    \n    stride_lengths = compute_stride_length(step_acce_max_mins)\n    sensor_df[\"stride_length\"] = stride_lengths[:, 1]\n    \n    headings = compute_headings(ahrs)\n    step_headings = compute_step_heading(step_timestamps, headings)\n    sensor_df[\"step_heading\"] = step_headings[:, 1]\n\n    rel_positions = compute_rel_positions(stride_lengths, step_headings)\n    sensor_df[\"rel_pos_x\"] = rel_positions[:, 1]\n    sensor_df[\"rel_pos_y\"] = rel_positions[:, 2]\n    sensor_df[\"timestamp\"] = sensor_df[\"timestamp\"].apply(time_float_to_str)\n    \n    sensor_df[\"floor\"] = floor\n    sensor_df[\"path\"] = path\n    \n    if train:\n        return sensor_df, posi_df\n    return sensor_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Generate Train Data","metadata":{}},{"cell_type":"code","source":"def generate_one_train(site: str):\n    \"\"\" This function creates a sensor train and waypoint train csv file for a single site \"\"\"\n    file_path = f\"{train_path}/{site}\"\n    floor_paths = glob.glob(os.path.join(file_path+'/*'))\n    sensor_data = pd.DataFrame()\n    wayoints = pd.DataFrame()\n    \n    for floor in floor_paths:\n        files = glob.glob(os.path.join(floor+'/*'))\n        for file in files:\n            floor_name = file.split(\"/\")[-2]\n            path = file.split(\"/\")[-1].split(\".\")[0]\n\n            sensor_df, posi_df = generate_sensor_df(file_path=file, floor=floor_name, path=path, train=True)\n            sensor_data = pd.concat([sensor_data, sensor_df])\n            wayoints = pd.concat([wayoints, posi_df]) \n            \n    sensor_data.to_csv(f\"./train/{site}_sensor_train.csv\", index=False)\n    wayoints.to_csv(f\"./train/{site}_waypoint_train.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_train():\n    \"\"\" This function uses multiprocessing Pool in order to call generate_one_train for each site to create all train files \"\"\"\n    try:\n        os.mkdir(\"./train\")\n    except FileExistsError:\n        pass\n    \n    num_cores = multiprocessing.cpu_count()\n    print(f\"num_cores={num_cores}\")\n    pool = Pool(num_cores)\n    pool.map(generate_one_train, sites)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with timer(\"Generating Train Dataset\"):\n    generate_train()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Generate Test Data","metadata":{}},{"cell_type":"code","source":"def generate_one_test(building_df: pd.DataFrame):\n    \"\"\" This function creates a sensor test csv for a single site \"\"\"\n    site = str((building_df.site.unique()[0]))\n    paths = building_df.path.unique()\n    sensor_data = pd.DataFrame()\n    \n    for path in paths:\n        file_path = f\"{test_path}/{path}.txt\"\n        sensor_df = generate_sensor_df(file_path=file_path, floor=\"TBD\", path=path, train=False)\n        sensor_data = pd.concat([sensor_data, sensor_df])\n        \n    sensor_data.to_csv(f\"./test/{site}_sensor_test.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_test():\n    \"\"\" This function uses multiprocessing Pool in order to call generate_one_test for each site to create all test files \"\"\"\n    try:\n        os.mkdir(\"./test\")\n    except FileExistsError:\n        pass\n    \n    num_cores = multiprocessing.cpu_count()\n    print(f\"num_cores={num_cores}\")\n    pool = Pool(num_cores)\n    pool.map(generate_one_test, building_dfs)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with timer(\"Generate Test Dataset\"):\n    generate_test()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}