{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import glob\nimport pandas as pd\nimport numpy as np\nfrom pathlib import Path\nimport re\nimport os\nimport gc","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import functions provided by Competition","metadata":{}},{"cell_type":"code","source":"#!git clone https://github.com/location-competition/indoor-location-competition-20.git","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%cd /kaggle/working/indoor-location-competition-20/","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#from io_f import read_data_file","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inpath = '/kaggle/input/indoor-location-navigation/'\nmetapath = inpath + 'metadata/'\ntrainpath = inpath + 'train/'\ntestpath = inpath + 'test/'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_buildings = os.listdir('/kaggle/input/indoor-location-navigation/train')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# os.system(f'grep SiteID {testpath}/* > test_buildings.txt' )\n# test_buildings = pd.read_csv('test_buildings.txt',sep='\\t',header=None,names=['file','building','site'])\n# test_buildings['building'] = test_buildings['building'].apply(lambda x: x[7:])\n# buildings = np.unique(test_buildings['building'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# floors = []\n# for b in buil:\n#     floors.extend(os.listdir(os.path.join('/kaggle/input/indoor-location-navigation/train', b)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# np.unique(floors)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"floors = {'1F':0, '2F':1, '3F':2, '4F':3, '5F':4, '6F':5, '7F':6, '8F':7, '9F':8, \n          'B':-1, 'B1':-1,'B2':-2, 'B3':-3, 'BF':-1, 'BM':-1, \n          'F1':0, 'F2':1, 'F3':2, 'F4':3, 'F5':4, 'F6':5,'F7':6, 'F8':7, 'F9':8, 'F10':9,\n          'G':0, 'LG1':-1, 'LG2':-2, \n          \"L1\":0,\"L2\":1,\"L3\":2,\"L4\":3,\"L5\":4,\"L6\":5,\"L7\":6,\"L8\":7,\"L9\":8,\"L10\":9,'L11':10, \n          'LM':0, 'M':0, 'P1':-1, 'P2':-2}\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs('./cleaned/')\n# os.makedirs('./cleaned/imu/')\n# os.makedirs('./cleaned/waypoint/')\n# os.makedirs('./cleaned/beacon/')\n# os.makedirs('./cleaned/wifi/')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wifi_test_dfs = pd.DataFrame()\nimu_test_dfs = pd.DataFrame()\nbeacon_test_dfs = pd.DataFrame()\n\nfor filename in os.listdir(testpath):\n        print('----'+filename)\n        names = ['Time', 'Type'] + ['col'+str(x) for x in range(1,9)]\n        df = pd.read_csv(testpath+filename, sep='\\t', comment='#', header=None, names=names, dtype=object)\n        df['path_id'] = filename.split('/')[-1][:-4]\n        df['building_id'] = open(testpath+filename).readlines()[1].split('\\t')[1][7:]        \n        \n#         l = []\n#         f = pd.read_csv(testpath+filename, sep='\\t', comment='#', header=None, names=names, dtype=object)\n#         f['path_id'] = filename.split('/')[-1][:-4]\n#         f['building_id'] = open(testpath+filename).readlines()[1].split('\\t')[1][7:]\n#         l.append(f)\n#         df = pd.concat(l, axis=0)\n    \n        # wifi\n        wifi_df = df[df['Type'].isin(['TYPE_WIFI'])][\n            ['building_id','path_id','Time','Type','col1','col2','col3','col4','col5']].rename(\n            columns={'col1':'ssid','col2':'bssid','col3':'RSSI','col4':'frequency','col5':'lastSeenTimestamp'})\n        wifi_test_dfs = wifi_test_dfs.append(wifi_df)\n        \n        # beacon\n        beacon_df = df[df['Type'].isin(['TYPE_BEACON'])][\n            ['building_id','path_id','Time','Type','col1','col2','col3','col4','col5','col6','col7','col8']].rename(\n            columns={'col1':'UUID','col2':'MajorID','col3':'MinorID',\n                     'col4':'TxPower','col5':'RSSI','col6':'Distance','col7':'MACAddress','col8':'UnixTime'})\n        beacon_test_dfs = beacon_test_dfs.append(beacon_df)\n        \n        # imu\n        imu_df = df[df['Type'].isin(['TYPE_ACCELEROMETER','TYPE_GYROSCOPE','TYPE_MAGNETIC_FIELD','TYPE_ROTATION_VECTOR'])][\n            ['building_id','path_id','Time','Type','col1','col2','col3','col4']].rename(\n            columns={'col1':'x','col2':'y','col3':'z','col4':'accuracy'})\n        imu_test_dfs = imu_test_dfs.append(imu_df)\n        \n        del df, wifi_df, beacon_df, imu_df\n        gc.collect()","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wifi_test_dfs.to_csv(f'./cleaned/wifi_test.csv')\nbeacon_test_dfs.to_csv(f'./cleaned/beacon_test.csv')\nimu_test_dfs.to_csv(f'./cleaned/imu_test.csv')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"buildings = wifi_test_dfs['building_id'].unique().tolist()#np.unique(train_buildings)\nprint('# of buildings:',len(buildings))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del wifi_test_dfs, beacon_test_dfs, imu_test_dfs\ngc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wypt_dfs = pd.DataFrame()\nwifi_dfs = pd.DataFrame()\nimu_dfs = pd.DataFrame()\nbeacon_dfs = pd.DataFrame()\n\nfor building in buildings:\n    print('Building ID:',building, buildings.index(building))\n    \n    for floor in os.listdir(trainpath+building):\n        print('----',floor)\n        \n        names = ['Time', 'Type'] + ['col'+str(x) for x in range(1,9)]\n        l = []\n        for filename in glob.glob(f'{trainpath}/{building}/{floor}/*.txt'):\n            f = pd.read_csv(filename, sep='\\t', comment='#', header=None, names=names, dtype=object)\n            f['path_id'] = filename.split('/')[-1][:-4]\n            f['floor'] = floors[str(floor)]\n            f['building_id'] = str(building)\n            l.append(f)\n        df = pd.concat(l, axis=0)\n    \n        # waypoint\n        wypt_df = df[df['Type']=='TYPE_WAYPOINT'][\n            ['building_id','floor','path_id','Time','Type','col1','col2']].rename(columns={'col1':'x','col2':'y'})\n        wypt_dfs = wypt_dfs.append(wypt_df)\n        \n        # wifi\n        wifi_df = df[df['Type'].isin(['TYPE_WIFI'])][\n            ['building_id','floor','path_id','Time','Type','col1','col2','col3','col4','col5']].rename(\n            columns={'col1':'ssid','col2':'bssid','col3':'RSSI','col4':'frequency','col5':'lastSeenTimestamp'})\n        wifi_dfs = wifi_dfs.append(wifi_df)\n        \n        # beacon\n        beacon_df = df[df['Type'].isin(['TYPE_BEACON'])][\n            ['building_id','floor','path_id','Time','Type','col1','col2','col3','col4','col5','col6','col7','col8']].rename(\n            columns={'col1':'UUID','col2':'MajorID','col3':'MinorID',\n                     'col4':'TxPower','col5':'RSSI','col6':'Distance','col7':'MACAddress','col8':'UnixTime'})\n        beacon_dfs = beacon_dfs.append(beacon_df)\n        \n#         # imu\n#         imu_df = df[df['Type'].isin(['TYPE_ACCELEROMETER','TYPE_GYROSCOPE','TYPE_MAGNETIC_FIELD','TYPE_ROTATION_VECTOR'])][\n#             ['building_id','floor','path_id','Time','Type','col1','col2','col3','col4']].rename(\n#             columns={'col1':'x','col2':'y','col3':'z','col4':'accuracy'})\n#         imu_dfs = imu_dfs.append(imu_df)\n\n        del df, wypt_df, wifi_df, beacon_df#, imu_df\n        gc.collect()","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wypt_dfs.to_csv(f'./cleaned/waypoint_train.csv')\nwifi_dfs.to_csv(f'./cleaned/wifi_train.csv')\nbeacon_dfs.to_csv(f'./cleaned/beacon_train.csv')\n# imu_dfs.to_csv(f'./cleaned/imu_train.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# del wypt_dfs, wifi_dfs, beacon_dfs, imu_dfs\n# gc.collect()","metadata":{},"execution_count":null,"outputs":[]}]}