{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This is the notebook for generating the wifi-features for all 204 buildings. [Dataset](https://www.kaggle.com/samratthapa/training-wifi-features-for-all-204-buildings)\n\nYou can use it to pretrain your unified model.Each row includes the 100 bssids with strongest rssi, the xy coordinates, floor label. It also includes the difference between the timestamp of the wifi and the nearest waypoint in the 'td' column. \n\nI adopted code from this notebook.https://www.kaggle.com/devinanzelmo/wifi-features","metadata":{}},{"cell_type":"code","source":"packages = [\n    '../input/indoor-locationnavigation-2021/indoor-location-competition-20-master/indoor-location-competition-20-master'\n]\nimport sys\nfor pth in packages:\n    sys.path.append(pth)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from io_f import read_data_file\nfrom compute_f import *","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport glob\nimport os\nimport gc\nimport json ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_path = '../input/indoor-location-navigation/'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ssubm = pd.read_csv('../input/indoor-location-navigation/sample_submission.csv')\n\n\nssubm_df = ssubm[\"site_path_timestamp\"].apply(lambda x: pd.Series(x.split(\"_\")))\ntest_buildings = sorted(ssubm_df[0].value_counts().index.tolist())\nused_buildings = sorted(os.listdir('../input/indoor-location-navigation/train'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(test_buildings),len(used_buildings)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"column_reor = [x for x in range(0,200,2)]\ncolumn_reor.extend([x for x in range(1,200,2)])\ncolumn_reor.extend([200,201,202,203,204,205])\n\ncolumn_names = [f'bssid{x}' for x in range(100)]\ncolumn_names.extend([f'rssi{x}' for x in range(100)])\ncolumn_names.extend(['x','y','f','path','td','timestamps'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get only the wifi bssid that occur over 1000 times(this number can be experimented with)\n# these will be the only ones used when constructing features\nuniques = np.array([\"none\"])\nfor building in test_buildings:\n    folders = sorted(glob.glob(os.path.join(base_path,'train/'+building+'/*')))\n    print(f'{building} :')\n    dfs = list()\n    for folder in folders:\n        floor = folder.split('/')[-1]\n        files = glob.glob(os.path.join(folder, \"*.txt\"))\n        print(f'\\t{floor}')\n        for file in files:\n            path_name = file.split('/')[-1].split('.')[0]\n            temp_data = read_data_file(file)\n            if len(temp_data.wifi)<50 or len(temp_data.waypoint)<2:\n                continue\n            wifi = temp_data.wifi\n            waypoint = temp_data.waypoint\n            \n            df_wifi = pd.DataFrame(wifi)\n            ext = []\n            tar = []\n            for gid, g in df_wifi.groupby(0):\n                dists = list()\n                for e, k in enumerate(waypoint):\n                    dist = abs(int(gid) - int(k[0]))\n                    dists.append(dist)\n                nearest_wp_index = np.argmin(dists)\n                \n                g = g.drop_duplicates()\n#                 Extracting the top 100 bssids with strongest signals.\n                feat = g.sort_values(3).reset_index().iloc[:100,3:5].reindex(np.arange(100)).values.flatten()\n                ext.append(feat)\n                tar.append(np.array([float(waypoint[nearest_wp_index][1]), \n                                     float(waypoint[nearest_wp_index][2]),floor,path_name,np.min(dists),int(gid)]))\n            data =  pd.DataFrame(np.hstack([ext,tar]))[column_reor]\n            data.columns = column_names\n            data.timestamps = data.timestamps.astype('int')\n            data = data.sort_values(['timestamps'])\n            data['t_diff'] = data['timestamps']-np.hstack([data['timestamps'][0],data['timestamps'][:-1].to_numpy()])\n            dfs.append(data)\n        gc.collect()\n    dfs = pd.concat(dfs,0) \n    dfs.iloc[:,:100] = dfs.iloc[:,:100].fillna('none')\n    dfs.iloc[:,100:200] = dfs.iloc[:,100:200].fillna(-999).astype('int32')\n    uniques = np.concatenate([dfs.iloc[:,:100].stack().unique(),uniques])\n    dfs.to_parquet(os.path.join(f'{building}_train.parquet'))\n    print(\"Current unique bssid length: \",len(uniques))\n    print('Saved File.........',Path(f'{building}_train.parquet').stat().st_size/1024/1024,\"MB\\n-------------------------------\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files = glob.glob(os.path.join('', \"*.parquet\"))\nli =[]\nfor filename in train_files:\n    df = pd.read_parquet(filename)\n    li.append(df)\n\ntrain_all = pd.concat(li, axis=0, ignore_index=True)\ntrain_all.to_parquet(\"train_all.parq\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"column_reor = [x for x in range(0,200,2)]\ncolumn_reor.extend([x for x in range(1,200,2)])\ncolumn_reor.extend([200,201])\n\ncolumn_names = [f'bssid{x}' for x in range(100)]\ncolumn_names.extend([f'rssi{x}' for x in range(100)])\ncolumn_names.extend(['path','timestamps'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_test = np.array([\"none\"])\ndfs = list()\nfor path_name,g1 in ssubm_df.groupby(1):\n    temp_data = read_data_file(os.path.join(base_path,'test/'+f'/{path_name}.txt'))\n    wifi = temp_data.wifi\n    \n    df_wifi = pd.DataFrame(wifi)\n    ext = []\n    tar = []\n    \n    for gid, g in df_wifi.groupby(0):\n\n        g = g.drop_duplicates()\n        \n        feat = g.sort_values(3).reset_index().iloc[:100,3:5].reindex(np.arange(100)).values.flatten()\n        ext.append(feat)\n        tar.append(np.array([path_name,int(gid)]))\n    data =  pd.DataFrame(np.hstack([ext,tar]))[column_reor]\n    data.columns = column_names\n    data.timestamps = data.timestamps.astype('int')\n    data = data.sort_values(['timestamps'])\n    \n    for timepoint in g1.iloc[:,2].tolist():\n            deltas = np.abs(data['timestamps'].values - int(timepoint))\n            min_delta_idx = deltas.argmin()\n            data.loc[min_delta_idx,\"timestamps\"] = int(timepoint)\n    data['t_diff'] = data['timestamps']-np.hstack([data['timestamps'][0],data['timestamps'][:-1].to_numpy()])\n    \n    dfs.append(data)\ngc.collect()\ndfs = pd.concat(dfs,0) \ndfs.iloc[:,:100] = dfs.iloc[:,:100].fillna('none')\ndfs.iloc[:,100:200] = dfs.iloc[:,100:200].fillna(-999).astype('int32')\nunique_test = np.concatenate([dfs.iloc[:,:100].stack().unique(),unique_test])\ndfs.to_parquet('test_all.parq')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# uniques includes the 'none' tag from each building \nunique_bssids = pd.DataFrame(np.unique(unique_test))\nunique_bssids.to_csv('unique_bssids_test.csv',index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# uniques includes the 'none' tag from each building \nunique_bssids = pd.DataFrame(np.unique(uniques))\nunique_bssids.to_csv('unique_bssids_train.csv',index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}