{"cells":[{"metadata":{},"cell_type":"markdown","source":"Given is the code to convert all given information to CSV files, for further use.\n\nI have only converted the test paths and the associated 24 sites, but you may choose to convert all of them if you so choose, but keep in mind some sites are different/poorly formatted, so the below code will not work on those. But it works for the sites that we are supposed to predict, so here goes: "},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nfrom IPython.display import clear_output, display\nfrom dataclasses import dataclass\n@dataclass\nclass ReadData:\n    acce: np.ndarray        # TS, X, Y, Z, Accuracy\n    acce_uncali: np.ndarray # TS, X, Y, Z, X_uncali,Y_uncali, Z_uncali, Accuracy \n    gyro: np.ndarray        # TS, X, Y, Z, Accuracy\n    gyro_uncali: np.ndarray # TS, X, Y, Z, X_uncali, Y_uncali, Z_uncali, Accuracy\n    magn: np.ndarray        # TS, X, Y, Z, Accuracy\n    magn_uncali: np.ndarray # TS, X, Y, Z, X_uncali, Y_uncali, Z_uncali, Accuracy\n    ahrs: np.ndarray        # TS, X, Y, Z, Accuracy\n    wifi: np.ndarray        # TS, ssid, bssid, rssi, frequency, lastseen_ts\n    ibeacon: np.ndarray     # TS, uuid, majorid, minorid, txpower, rssi, distance, mac, unix_time\n    waypoint: np.ndarray    # TS, X, Y\n\n\ndef read_data_file(data_filename):\n    acce = [[0, 0, 0, 0, 0]]\n    acce_uncali = [[0, 0, 0, 0, 0, 0, 0, 0]]\n    gyro = [[0, 0, 0, 0, 0]]\n    gyro_uncali = [[0, 0, 0, 0, 0, 0, 0, 0]]\n    magn = [[0, 0, 0, 0, 0]]\n    magn_uncali = [[0, 0, 0, 0, 0, 0, 0, 0]]\n    ahrs = [[0, 0, 0, 0, 0]]\n    wifi = [[0, 0, 0, 0, 0, 0]]\n    ibeacon = [[0, 0, 0, 0, 0, 0, 0, 0, 0]]\n    waypoint = [[0, 0, 0]]\n\n    with open(data_filename, 'r', encoding='utf-8') as file:\n        lines = file.readlines()\n\n    for line_data in lines:\n        line_data = line_data.strip()\n        if not line_data or line_data[0] == '#':\n            continue\n\n        line_data = line_data.split('\\t')\n        try:\n            if line_data[1] == 'TYPE_ACCELEROMETER':\n                ts = int(line_data[0])\n                x, y, z = float(line_data[2]), float(line_data[3]), float(line_data[4])\n                accuracy = float(line_data[5])\n                acce.append([ts, x, y, z, accuracy])\n                continue\n\n            if line_data[1] == 'TYPE_ACCELEROMETER_UNCALIBRATED':\n                ts = int(line_data[0])\n                x, y, z = float(line_data[2]), float(line_data[3]), float(line_data[4])\n                x_uncali, y_uncali, z_uncali = float(line_data[5]), float(line_data[6]), float(line_data[7])\n                accuracy = float(line_data[8])\n                acce_uncali.append([ts, x, y, z, x_uncali, y_uncali, z_uncali, accuracy])\n                continue\n\n            if line_data[1] == 'TYPE_GYROSCOPE':\n                ts = int(line_data[0])\n                x, y, z = float(line_data[2]), float(line_data[3]), float(line_data[4])\n                accuracy = float(line_data[5])\n                gyro.append([ts, x, y, z, accuracy])\n                continue\n\n            if line_data[1] == 'TYPE_GYROSCOPE_UNCALIBRATED':\n                ts = int(line_data[0])\n                x, y, z = float(line_data[2]), float(line_data[3]), float(line_data[4])\n                x_uncali, y_uncali, z_uncali = float(line_data[5]), float(line_data[6]), float(line_data[7])\n                accuracy = float(line_data[8])\n                gyro_uncali.append([ts, x, y, z, x_uncali, y_uncali, z_uncali, accuracy])\n                continue\n\n            if line_data[1] == 'TYPE_MAGNETIC_FIELD':\n                ts = int(line_data[0])\n                x, y, z = float(line_data[2]), float(line_data[3]), float(line_data[4])\n                accuracy = float(line_data[5])\n                magn.append([ts, x, y, z, accuracy])\n                continue\n\n            if line_data[1] == 'TYPE_MAGNETIC_FIELD_UNCALIBRATED':\n                ts = int(line_data[0])\n                x, y, z = float(line_data[2]), float(line_data[3]), float(line_data[4])\n                x_uncali, y_uncali, z_uncali = float(line_data[5]), float(line_data[6]), float(line_data[7])\n                accuracy = float(line_data[8])\n                magn_uncali.append([ts, x, y, z, x_uncali, y_uncali, z_uncali, accuracy])\n                continue\n\n            if line_data[1] == 'TYPE_ROTATION_VECTOR':\n                ts = int(line_data[0])\n                x, y, z = float(line_data[2]), float(line_data[3]), float(line_data[4])\n                accuracy = float(line_data[5])\n                ahrs.append([ts, x, y, z, accuracy])\n                continue\n\n            if line_data[1] == 'TYPE_WIFI':\n                ts = int(line_data[0])\n                ssid = line_data[2]\n                bssid = line_data[3]\n                rssi = line_data[4]\n                freq = line_data[5]\n                lastseen_ts = line_data[6]\n                wifi_data = [ts, ssid, bssid, rssi, freq, lastseen_ts]\n                wifi.append(wifi_data)\n                continue\n\n            if line_data[1] == 'TYPE_BEACON':\n                ts = int(line_data[0])\n                uuid = line_data[2]\n                major = line_data[3]\n                minor = line_data[4]\n                txpower = int(line_data[5])\n                rssi = int(line_data[6])\n                distance = float(line_data[7])\n                macaddress = line_data[8]\n                unixtime = int(line_data[9])\n                ibeacon_data = [ts, uuid, major, minor, txpower, rssi, distance, macaddress, unixtime]\n                ibeacon.append(ibeacon_data)\n                continue\n\n            if line_data[1] == 'TYPE_WAYPOINT':\n                waypoint.append([int(line_data[0]), float(line_data[2]), float(line_data[3])])\n        except:\n            continue\n            \n    acce = np.array(acce)\n    acce_uncali = np.array(acce_uncali)\n    gyro = np.array(gyro)\n    gyro_uncali = np.array(gyro_uncali)\n    magn = np.array(magn)\n    magn_uncali = np.array(magn_uncali)\n    ahrs = np.array(ahrs)\n    wifi = np.array(wifi)\n    ibeacon = np.array(ibeacon)\n    waypoint = np.array(waypoint)\n\n    return ReadData(acce, acce_uncali, gyro, gyro_uncali, magn, magn_uncali, ahrs, wifi, ibeacon, waypoint)\n\n\ndef get_site_from_test(tfile):\n    with open(tfile, 'r', encoding='utf-8') as file:\n        lines = file.readlines()\n    for line in lines:\n        line_data = line.split('\\t')\n        if (not line_data) or line_data[0] != '#':\n            continue\n        for word in line_data:\n            if word[:6].lower() == 'siteid':\n                crap, sid = word.split(':')\n                return sid\n\ndef get_unique_ts(arr):\n    ts = set()\n    for item in arr:\n        for k in item:\n            ts.add(int(k))\n    return sorted(list(ts))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"basedir = '../input/indoor-location-navigation/test'\npaths = os.listdir(basedir)\ntest_sites = set()\n\ntest_dir = '/kaggle/working/test'\nos.makedirs(test_dir, exist_ok=True)\n\nTESTCOLS = ['acce_x', 'acce_y', 'acce_z', 'acce_accuracy',\n        'acce_uncali_x', 'acce_uncali_y', 'acce_uncali_z', 'acce_bias_x', 'acce_bias_y', 'acce_bias_z', 'acce_uncali_accuracy',\n        'magn_x', 'magn_y', 'magn_z', 'magn_accuracy',\n        'magn_uncali_x', 'magn_uncali_y', 'magn_uncali_z', 'magn_bias_x', 'magn_bias_y', 'magn_bias_z', 'magn_uncali_accuracy',\n        'gyro_x', 'gyro_y', 'gyro_z', 'gyro_accuracy',\n        'gyro_uncali_x', 'gyro_uncali_y', 'gyro_uncali_z', 'gyro_bias_x', 'gyro_bias_y', 'gyro_bias_z', 'gyro_uncali_accuracy',\n        'ahrs_x', 'ahrs_y', 'ahrs_z', 'ahrs_accuracy',\n        'wifi_ssid', 'wifi_bssid', 'wifi_rssi', 'wifi_freq', 'wifi_ls_ts',\n        'uuid', 'majorid', 'minorid', 'txpow', 'beacon_rssi', 'distance', 'macaddr', 'unix_time'\n       ]\n\n\nfor path in paths:\n    clear_output(wait=True)\n    display(f'On Path: {paths.index(path)}/{len(paths)}')\n    \n    siteId = get_site_from_test(os.path.join(basedir, path))\n    test_sites.add(siteId)\n    \n    try:\n        data = read_data_file(os.path.join(basedir, path))\n        \n        all_ts = get_unique_ts([data.acce[:, 0], data.magn[:, 0], data.gyro[:, 0], data.ahrs[:, 0],\n                                data.acce_uncali[:, 0], data.magn_uncali[:, 0], data.gyro_uncali[:, 0],\n                               data.wifi[:, 0], data.ibeacon[:, 0]])\n        all_ts = [int(s) for s in all_ts]\n        df = pd.DataFrame(np.nan, index=all_ts, columns=TESTCOLS)\n        \n        df.loc[data.acce[:,0].astype(int), ['acce_x', 'acce_y', 'acce_z', 'acce_accuracy']] = data.acce[:, 1:]\n        df.loc[data.magn[:,0].astype(int), ['magn_x', 'magn_y', 'magn_z', 'magn_accuracy']] = data.magn[:, 1:]\n        df.loc[data.gyro[:,0].astype(int), ['gyro_x', 'gyro_y', 'gyro_z', 'gyro_accuracy']] = data.gyro[:, 1:]\n        df.loc[data.ahrs[:,0].astype(int), ['ahrs_x', 'ahrs_y', 'ahrs_z', 'ahrs_accuracy']] = data.ahrs[:, 1:]\n        df.loc[data.wifi[:,0].astype(int), ['wifi_ssid', 'wifi_bssid', 'wifi_rssi', 'wifi_freq', 'wifi_ls_ts']] = data.wifi[:, 1:]\n        df.loc[data.ibeacon[:,0].astype(int), ['uuid', 'majorid', 'minorid', 'txpow', 'beacon_rssi', 'distance', 'macaddr', 'unix_time']] = data.ibeacon[:, 1:]\n        df.loc[data.acce_uncali[:,0].astype(int), ['acce_uncali_x', 'acce_uncali_y', 'acce_uncali_z', 'acce_bias_x', 'acce_bias_y', 'acce_bias_z', 'acce_uncali_accuracy']] = data.acce_uncali[:, 1:]\n        df.loc[data.magn_uncali[:,0].astype(int), ['magn_uncali_x', 'magn_uncali_y', 'magn_uncali_z', 'magn_bias_x', 'magn_bias_y', 'magn_bias_z', 'magn_uncali_accuracy']] = data.magn_uncali[:, 1:]\n        df.loc[data.gyro_uncali[:,0].astype(int), ['gyro_uncali_x', 'gyro_uncali_y', 'gyro_uncali_z', 'gyro_bias_x', 'gyro_bias_y', 'gyro_bias_z', 'gyro_uncali_accuracy']] = data.gyro_uncali[:, 1:]\n        \n            \n        df = df.loc[1:, :]\n        df.to_csv(os.path.join(test_dir, f'{siteId}_{path[:-4]}.csv'))\n        \n    except:\n        print(err)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dir = '/kaggle/working/train'\ninput_dir = '../input/indoor-location-navigation/train'\nos.makedirs(train_dir, exist_ok=True)\ntest_sites2 = list(test_sites)\nTRAINCOLS = ['acce_x', 'acce_y', 'acce_z', 'acce_accuracy',\n        'acce_uncali_x', 'acce_uncali_y', 'acce_uncali_z', 'acce_bias_x', 'acce_bias_y', 'acce_bias_z', 'acce_uncali_accuracy',\n        'magn_x', 'magn_y', 'magn_z', 'magn_accuracy',\n        'magn_uncali_x', 'magn_uncali_y', 'magn_uncali_z', 'magn_bias_x', 'magn_bias_y', 'magn_bias_z', 'magn_uncali_accuracy',\n        'gyro_x', 'gyro_y', 'gyro_z', 'gyro_accuracy',\n        'gyro_uncali_x', 'gyro_uncali_y', 'gyro_uncali_z', 'gyro_bias_x', 'gyro_bias_y', 'gyro_bias_z', 'gyro_uncali_accuracy',\n        'ahrs_x', 'ahrs_y', 'ahrs_z', 'ahrs_accuracy',\n        'wifi_ssid', 'wifi_bssid', 'wifi_rssi', 'wifi_freq', 'wifi_ls_ts',\n        'uuid', 'majorid', 'minorid', 'txpow', 'beacon_rssi', 'distance', 'macaddr', 'unix_time',\n        'way_x', 'way_y'\n       ]\n\nfor site in test_sites2:\n    \n    floors = os.listdir(os.path.join(input_dir, site))\n    \n    for floor in floors:\n            \n        paths = os.listdir(os.path.join(input_dir, site, floor))\n        \n        for path in paths:\n            clear_output(wait=True)\n            print(f'Site: {test_sites2.index(site)}/{len(test_sites2)}, Floor: {floors.index(floor)}/{len(floors)}, Path: {paths.index(path)}/{len(paths)}')\n            \n            try:\n                data = read_data_file(os.path.join(input_dir,site,floor, path))\n        \n                all_ts = get_unique_ts([data.acce[:, 0], data.magn[:, 0], data.gyro[:, 0], data.ahrs[:, 0],\n                                        data.acce_uncali[:, 0], data.magn_uncali[:, 0], data.gyro_uncali[:, 0],\n                                       data.wifi[:, 0], data.ibeacon[:, 0], data.waypoint[:, 0]])\n                all_ts = [int(s) for s in all_ts]\n                df = pd.DataFrame(np.nan, index=all_ts, columns=TRAINCOLS)\n\n                df.loc[data.acce[:,0].astype(int), ['acce_x', 'acce_y', 'acce_z', 'acce_accuracy']] = data.acce[:, 1:]\n                df.loc[data.magn[:,0].astype(int), ['magn_x', 'magn_y', 'magn_z', 'magn_accuracy']] = data.magn[:, 1:]\n                df.loc[data.gyro[:,0].astype(int), ['gyro_x', 'gyro_y', 'gyro_z', 'gyro_accuracy']] = data.gyro[:, 1:]\n                df.loc[data.ahrs[:,0].astype(int), ['ahrs_x', 'ahrs_y', 'ahrs_z', 'ahrs_accuracy']] = data.ahrs[:, 1:]\n                df.loc[data.wifi[:,0].astype(int), ['wifi_ssid', 'wifi_bssid', 'wifi_rssi', 'wifi_freq', 'wifi_ls_ts']] = data.wifi[:, 1:]\n                df.loc[data.ibeacon[:,0].astype(int), ['uuid', 'majorid', 'minorid', 'txpow', 'beacon_rssi', 'distance', 'macaddr', 'unix_time']] = data.ibeacon[:, 1:]\n                df.loc[data.acce_uncali[:,0].astype(int), ['acce_uncali_x', 'acce_uncali_y', 'acce_uncali_z', 'acce_bias_x', 'acce_bias_y', 'acce_bias_z', 'acce_uncali_accuracy']] = data.acce_uncali[:, 1:]\n                df.loc[data.magn_uncali[:,0].astype(int), ['magn_uncali_x', 'magn_uncali_y', 'magn_uncali_z', 'magn_bias_x', 'magn_bias_y', 'magn_bias_z', 'magn_uncali_accuracy']] = data.magn_uncali[:, 1:]\n                df.loc[data.gyro_uncali[:,0].astype(int), ['gyro_uncali_x', 'gyro_uncali_y', 'gyro_uncali_z', 'gyro_bias_x', 'gyro_bias_y', 'gyro_bias_z', 'gyro_uncali_accuracy']] = data.gyro_uncali[:, 1:]\n                df.loc[data.waypoint[:,0].astype(int), ['way_x', 'way_y']] = data.waypoint[:, 1:]\n                \n                df = df.loc[1:, :]\n                df.to_csv(os.path.join(train_dir, f'{site}_{floor}_{path[:-4]}.csv'))\n            except:\n                print(err)\n            ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!zip -r './train.zip' './train/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!zip -r './test.zip' './test/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm -r './train'\n!rm -r './test'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Hope it helps, I will be saving the output to a public dataset, so you can directly get the CSVs from there if you want. Thank you, Have a nice day!\n"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}