{"cells":[{"metadata":{},"cell_type":"markdown","source":"# We are used to pandas, aren't we?\n\n(please upvote if it helps, it motivates me;)\n\nThe official API to access the data uses original data-class.\nI am not familiar with that data format, so I just made a reading api for pandas dataframe.\n\nThis is WIP notebook, as just one data is treated.\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def floorname2num(floorname):\n    if floorname == \"F11\" or floorname == \"11F\" or floorname == \"L11\" or floorname == \"11L\":\n        return 10\n    if floorname == \"F10\" or floorname == \"10F\" or floorname == \"L10\" or floorname == \"10L\":\n        return 9\n    if floorname == \"F9\" or floorname == \"9F\" or floorname == \"L9\" or floorname == \"9L\":\n        return 8\n    if floorname == \"F8\" or floorname == \"8F\" or floorname == \"L8\" or floorname == \"8L\":\n        return 7\n    if floorname == \"F7\" or floorname == \"7F\" or floorname == \"L7\" or floorname == \"7L\":\n        return 6\n    if floorname == \"F6\" or floorname == \"6F\" or floorname == \"L6\" or floorname == \"6L\":\n        return 5\n    if floorname == \"F5\" or floorname == \"5F\" or floorname == \"L5\" or floorname == \"5L\":\n        return 4\n    if floorname == \"F4\" or floorname == \"4F\" or floorname == \"L4\" or floorname == \"4L\":\n        return 3\n    if floorname == \"F3\" or floorname == \"3F\" or floorname == \"L3\" or floorname == \"3L\":\n        return 2\n    if floorname == \"F2\" or floorname == \"2F\" or floorname == \"L2\" or floorname == \"2L\":\n        return 1\n    if floorname == \"F1\" or floorname == \"1F\" or floorname == \"L1\" or floorname == \"1L\":\n        return 0\n    if floorname == \"B1\" or floorname == \"1B\" or floorname == \"BF\" or floorname == \"B\":\n        return -1\n    if floorname == \"B2\" or floorname == \"2B\":\n        return -2\n    if floorname == \"B3\" or floorname == \"3B\":\n        return -3\n        \n    return None\n    # ignoring following floor names\n    # M at 玫瑰坊 and ../input/indoor-location-navigation/train/5cdac620e403deddaf467ff9 (亚新生活广场)\n    # ../input/indoor-location-navigation/train/5cd969bd39e2fc0b4afe727d (绿地缤纷城(徐汇店))\n    # BM at ../input/indoor-location-navigation/train/5d2709dd03f801723c32cfb6 (印象城(西溪店))\n    # LG1, LG2, P1, P2 at ../input/indoor-location-navigation/train/5cd969ba39e2fc0b4afe6fae (兴业太古汇)\n    # LG1, LG2 at ../input/indoor-location-navigation/train/5cd56bd9e2acfd2d33b662df (晶品)\n    #          and ../input/indoor-location-navigation/train/5cd56bd6e2acfd2d33b65dca (尚嘉中心)\n    # G, LM at ../input/indoor-location-navigation/train/5cd56bc1e2acfd2d33b6404d (世博源) ... this also has B and L1 \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# copy and modified from https://github.com/location-competition/indoor-location-competition-20/blob/master/io_f.py\n\ndef read_one_data_file(data_filename, return_metadata=True):\n    \n    with open(data_filename, 'r', encoding='utf-8') as file:\n        lines = file.readlines()\n        \n    metadata = {}\n    data_table = []\n\n    for line_data in lines:\n        \n        line_data = line_data.strip()\n        if not line_data :\n            continue\n        if line_data[0] == '#':\n            # read metadata that looks useful\n            line_data = line_data.split('\\t')\n            if \"SiteID\" in line_data[1] :\n                metadata[\"site_id\"] = line_data[1].split(\":\")[-1]\n                continue\n                \n            if \"Brand\" in line_data[1] :\n                metadata[\"brand\"] = line_data[1].split(\":\")[-1]\n                metadata[\"model\"] = line_data[2].split(\":\")[-1]\n                continue\n                \n            if \"type:\" in line_data[1] :\n                if \"Accelerometer\" in line_data[2] and \"Uncalibrated\" not in line_data[2] :\n                    metadata[\"accelerometer_name\"] = line_data[2].split(\":\")[-1].split(\" \")[0]\n                \n                if \"Gyroscope\" in line_data[2] and \"Uncalibrated\" not in line_data[2] :\n                    metadata[\"gyroscope_name\"] = line_data[2].split(\":\")[-1].split(\" \")[0]\n                \n                if \"Magnetometer\" in line_data[2] and \"Uncalibrated\" not in line_data[2] :\n                    metadata[\"magnetometer_name\"] = line_data[2].split(\":\")[-1].split(\" \")[0]\n                \n                continue\n            continue\n\n        line_data = line_data.split('\\t')\n        \n        if line_data[1] == 'TYPE_ACCELEROMETER':\n            data_table.append([int(line_data[0]), # Timestamp\n                               float(line_data[2]), float(line_data[3]), float(line_data[4]), # Accelerometer \n                               None, None, None, # Accelerometer uncalibrated\n                               None, None, None, # Gyro\n                               None, None, None, # Gyro uncalibrated\n                               None, None, None, # Magnetometer\n                               None, None, None, # Magnetometer uncalibrated\n                               None, None, None, # AHRS\n                               None, None, None, None, # WiFi\n                               None, None, None, None, None, # iBeacon\n                               None, None # Waypoint\n                              ])\n            continue\n\n        if line_data[1] == 'TYPE_ACCELEROMETER_UNCALIBRATED':\n            data_table.append([int(line_data[0]), # Timestamp\n                               None, None, None, # Accelerometer \n                               float(line_data[2]), float(line_data[3]), float(line_data[4]),# Accelerometer uncalibrated\n                               None, None, None, # Gyro\n                               None, None, None, # Gyro uncalibrated\n                               None, None, None, # Magnetometer\n                               None, None, None, # Magnetometer uncalibrated\n                               None, None, None, # AHRS\n                               None, None, None, None, # WiFi\n                               None, None, None, None, None, # iBeacon\n                               None, None # Waypoint\n                              ])\n            continue\n\n        if line_data[1] == 'TYPE_GYROSCOPE':\n            data_table.append([int(line_data[0]), # Timestamp\n                               None, None, None, # Accelerometer \n                               None, None, None, # Accelerometer uncalibrated\n                               float(line_data[2]), float(line_data[3]), float(line_data[4]), # Gyro\n                               None, None, None, # Gyro uncalibrated\n                               None, None, None, # Magnetometer\n                               None, None, None, # Magnetometer uncalibrated\n                               None, None, None, # AHRS\n                               None, None, None, None, # WiFi\n                               None, None, None, None, None, # iBeacon\n                               None, None # Waypoint\n                              ])\n            continue\n\n        if line_data[1] == 'TYPE_GYROSCOPE_UNCALIBRATED':\n            data_table.append([int(line_data[0]), # Timestamp\n                               None, None, None, # Accelerometer \n                               None, None, None, # Accelerometer uncalibrated\n                               None, None, None, # Gyro\n                               float(line_data[2]), float(line_data[3]), float(line_data[4]), # Gyro uncalibrated\n                               None, None, None, # Magnetometer\n                               None, None, None, # Magnetometer uncalibrated\n                               None, None, None, # AHRS\n                               None, None, None, None, # WiFi\n                               None, None, None, None, None, # iBeacon\n                               None, None # Waypoint\n                              ])\n            continue\n\n        if line_data[1] == 'TYPE_MAGNETIC_FIELD':\n            data_table.append([int(line_data[0]), # Timestamp\n                               None, None, None, # Accelerometer \n                               None, None, None, # Accelerometer uncalibrated\n                               None, None, None, # Gyro\n                               None, None, None, # Gyro uncalibrated\n                               float(line_data[2]), float(line_data[3]), float(line_data[4]), # Magnetometer\n                               None, None, None, # Magnetometer uncalibrated\n                               None, None, None, # AHRS\n                               None, None, None, None, # WiFi\n                               None, None, None, None, None, # iBeacon\n                               None, None # Waypoint\n                              ])\n            continue\n\n        if line_data[1] == 'TYPE_MAGNETIC_FIELD_UNCALIBRATED':\n            data_table.append([int(line_data[0]), # Timestamp\n                               None, None, None, # Accelerometer \n                               None, None, None, # Accelerometer uncalibrated\n                               None, None, None, # Gyro\n                               None, None, None, # Gyro uncalibrated\n                               None, None, None, # Magnetometer\n                               float(line_data[2]), float(line_data[3]), float(line_data[4]), # Magnetometer uncalibrated\n                               None, None, None, # AHRS\n                               None, None, None, None, # WiFi\n                               None, None, None, None, None, # iBeacon\n                               None, None # Waypoint\n                              ])\n            continue\n\n        if line_data[1] == 'TYPE_ROTATION_VECTOR':\n            data_table.append([int(line_data[0]), # Timestamp\n                               None, None, None, # Accelerometer \n                               None, None, None, # Accelerometer uncalibrated\n                               None, None, None, # Gyro\n                               None, None, None, # Gyro uncalibrated\n                               None, None, None, # Magnetometer\n                               None, None, None, # Magnetometer uncalibrated\n                               float(line_data[2]), float(line_data[3]), float(line_data[4]), # AHRS\n                               None, None, None, None, # WiFi\n                               None, None, None, None, None, # iBeacon\n                               None, None # Waypoint\n                              ])\n            continue\n\n        if line_data[1] == 'TYPE_WIFI':\n            data_table.append([int(line_data[0]), # Timestamp\n                               None, None, None, # Accelerometer \n                               None, None, None, # Accelerometer uncalibrated\n                               None, None, None, # Gyro\n                               None, None, None, # Gyro uncalibrated\n                               None, None, None, # Magnetometer\n                               None, None, None, # Magnetometer uncalibrated\n                               None, None, None, # AHRS\n                               line_data[2], line_data[3], float(line_data[4]), line_data[6], # WiFi\n                               None, None, None, None, None, # iBeacon\n                               None, None # Waypoint\n                              ])\n            continue\n\n        if line_data[1] == 'TYPE_BEACON':\n            data_table.append([int(line_data[0]), # Timestamp\n                               None, None, None, # Accelerometer \n                               None, None, None, # Accelerometer uncalibrated\n                               None, None, None, # Gyro\n                               None, None, None, # Gyro uncalibrated\n                               None, None, None, # Magnetometer\n                               None, None, None, # Magnetometer uncalibrated\n                               None, None, None, # AHRS\n                               None, None, None, None, # WiFi\n                               line_data[2], line_data[3], line_data[4], '_'.join([line_data[2], line_data[3], line_data[4]]), float(line_data[6]), # iBeacon\n                               None, None # Waypoint\n                              ])\n            continue\n\n        if line_data[1] == 'TYPE_WAYPOINT':\n            data_table.append([int(line_data[0]), # Timestamp\n                               None, None, None, # Accelerometer \n                               None, None, None, # Accelerometer uncalibrated\n                               None, None, None, # Gyro\n                               None, None, None, # Gyro uncalibrated\n                               None, None, None, # Magnetometer\n                               None, None, None, # Magnetometer uncalibrated\n                               None, None, None, # AHRS\n                               None, None, None, None, # WiFi\n                               None, None, None, None, None, # iBeacon\n                               float(line_data[2]), float(line_data[3]) # Waypoint\n                              ])\n            continue\n\n    \n    df = pd.DataFrame(data_table)\n    df.columns = [\"timestamp\", \n                  \"acc_x\", \"acc_y\", \"acc_z\", \n                  \"acc_uncali_x\", \"acc_uncali_y\", \"acc_uncali_z\",\n                  \"gyro_x\", \"gyro_y\", \"gyro_z\",\n                  \"gyro_uncali_x\", \"gyro_uncali_y\", \"gyro_uncali_z\",\n                  \"magn_x\", \"magn_y\", \"magn_z\",\n                  \"magn_uncali_x\", \"magn_uncali_y\", \"magn_uncali_z\",\n                  \"rotvec_x\", \"rotvec_y\", \"rotvec_z\",\n                  \"wifi_ssid\", \"wifi_bssid\", \"wifi_rssi\", \"wifi_lastseen_ts\",\n                  \"ib_uuid\", \"ib_major\", \"ib_minor\", \"ib_uuid_major_minor\", \"ib_rssi\",\n                  \"waypoint_x\", \"waypoint_y\"\n                 ]\n    \n    if return_metadata:\n        return df, metadata\n    else:\n        return df\n    \n    \ndef read_data_file(filepath):\n    filepath_sp = filepath.split(\"/\")\n    floor = filepath_sp[-2]\n    filename = filepath_sp[-1]\n\n    train_only_metadata_keys = [\"brand\", \"model\", # only for train\n                                \"accelerometer_name\",# only for train\n                                \"gyroscope_name\",# only for train\n                                \"magnetometer_name\"]# only for train\n    \n    df, metadata = read_one_data_file(filepath)\n    df[\"site_id\"] = metadata[\"site_id\"] \n    df[\"floor\"] = floorname2num(floor) # if test file, this returns None\n    for key in train_only_metadata_keys:\n        if key in metadata.keys():\n            df[key] = metadata[key]\n        else:\n            df[key] = None\n    df[\"filename\"] = filename\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Lets read a data for example."},{"metadata":{},"cell_type":"markdown","source":"Train data "},{"metadata":{"trusted":true},"cell_type":"code","source":"# manualy set file names for now.\nfilepath = \"../input/indoor-location-navigation/train/5a0546857ecc773753327266/B1/5e15730aa280850006f3d005.txt\"\n\ndf = read_data_file(filepath)\ndf","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Test data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# manualy set file names for now.\nfilepath = \"../input/indoor-location-navigation/test/01c41f1aeba5c48c2c4dd568.txt\"\n\ndf = read_data_file(filepath)\ndf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}