{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Wifi features\n\nThis this is the code to generate the wifi features available in [this dataset](https://www.kaggle.com/devinanzelmo/indoor-navigation-and-location-wifi-features). Using these features can get a score below 14. For an example notebook using them see [this notebook](https://www.kaggle.com/devinanzelmo/wifi-features-lightgbm-starter). They only uses waypoints, wifi and timestamp data to generate solution. See this [forum post](https://www.kaggle.com/c/indoor-location-navigation/discussion/215445) for an outline of this solution method, and methods of improvement.\n\n**Set to only output data for 5da138364db8ce0c98bc00f1 and 5a0546857ecc773753327266 to allow ability to contrast\n\nThere are `break`'s inserted into loops (these are now ehanced with counts to control the number of outputs) which need to be removed to get this to run. Right now data is written to current working directory. This takes 2-4 hours to run depending on hard drive etc. There is a lot of room for improvement speeding up feature generation. \n\n**Update:** I added one line that creates a column for the path filename, this allows for a groupkfold crossvalidation.\n\n### Final output:\n - each column is a BSSID\n - last 4 columns are:\n  1. x position\n  2. y position\n  3. floor\n  4. path id\n - each row is a recorded point of the path\n - BSSID columns are rssi in regards to BSSID\n - x, y, f are position values in regards to the BSSID with the highest rssi \n","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport glob\nimport os\nimport gc\nimport json ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_path = '../input/indoor-location-navigation/'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# This has been modified to only use a single building for testing","metadata":{}},{"cell_type":"code","source":"# pull out all the buildings actually used in the test set, given current method we don't need the other ones\nssubm = pd.read_csv('../input/indoor-location-navigation/sample_submission.csv')\n\n# only 24 of the total buildings are used in the test set, \n# this allows us to greatly reduce the intial size of the dataset\n\nssubm_df = ssubm[\"site_path_timestamp\"].apply(lambda x: pd.Series(x.split(\"_\")))\nused_buildings = sorted(ssubm_df[0].value_counts().index.tolist())\n#../input/indoor-location-navigation/train/5da1383b4db8ce0c98bc11ab\n#ssubm_df = ssubm_df[(ssubm_df[0]=='5da138364db8ce0c98bc00f1') | (ssubm_df[0]=='5a0546857ecc773753327266')]# or ssubm_df[0]=='5a0546857ecc773753327266']\n#used_buildings = ['5da138364db8ce0c98bc00f1', '5a0546857ecc773753327266']\n\n# dictionary used to map the floor codes to the values used in the submission file. \nfloor_map = {\"B2\":-2, \"B1\":-1, \"F1\":0, \"F2\": 1, \"F3\":2, \"F4\":3, \"F5\":4, \"F6\":5, \"F7\":6,\"F8\":7, \"F9\":8,\n             \"1F\":0, \"2F\":1, \"3F\":2, \"4F\":3, \"5F\":4, \"6F\":5, \"7F\":6, \"8F\": 7, \"9F\":8}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#used_buildings\n#ssubm_df.head()\nssubm","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get only the wifi bssid that occur over 1000 times(this number can be experimented with)\n# these will be the only ones used when constructing features\nbssid = dict()\ncount = 0\nfor building in used_buildings:\n#    if count > 1:\n#        break\n    folders = sorted(glob.glob(os.path.join(base_path,'train/'+building+'/*')))    \n    #../input/indoor-location-navigation/metadata/5da138364db8ce0c98bc00f1\n    #if building == '5da138364db8ce0c98bc00f1' or building == '5a0546857ecc773753327266':\n    print(building)    \n    for folder in folders:\n        wifi = list()\n        floor = floor_map[folder.split('/')[-1]]\n        files = glob.glob(os.path.join(folder, \"*.txt\"))\n        for file in files:\n            with open(file) as f:\n                txt = f.readlines()\n                for e, line in enumerate(txt):\n                    tmp = line.strip().split()\n                    if tmp[1] == \"TYPE_WIFI\":\n                        wifi.append(tmp)\n        df = pd.DataFrame(wifi)\n        #top_bssid = df[3].value_counts().iloc[:500].index.tolist()\n        value_counts = df[3].value_counts()\n        top_bssid = value_counts[value_counts > 1000].index.tolist()\n        print(len(top_bssid))\n        bssid[building] = top_bssid\n        del df\n        del wifi\n        gc.collect()\n        count += 1\nprint(\"second value is the total number of BSSID\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#'5a0546857ecc773753327266' in bssid\nbssid","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(\"bssid_1000.json\", \"w\") as f:\n    json.dump(bssid, f)\n\nwith open(\"bssid_1000.json\") as f:\n    bssid = json.load(f)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# generate all the training data \nbuilding_dfs = dict()\ncount = 0\nfor building in used_buildings:\n#    if count > 1:\n#        break\n    folders = sorted(glob.glob(os.path.join(base_path,'train', building +'/*')))\n    dfs = list()\n    index = sorted(bssid[building])\n    print(building)\n    for folder in folders:\n        floor = floor_map[folder.split('/')[-1]]\n        files = glob.glob(os.path.join(folder, \"*.txt\"))\n        print(floor)\n        for file in files:\n            wifi = list()\n            waypoint = list()\n            with open(file) as f:\n                txt = f.readlines()\n            for line in txt:\n                line = line.strip().split()\n                if line[1] == \"TYPE_WAYPOINT\":\n                    waypoint.append(line)\n                if line[1] == \"TYPE_WIFI\":\n                    wifi.append(line)\n\n            df = pd.DataFrame(np.array(wifi))    \n\n            # generate a feature, and label for each wifi block\n            for gid, g in df.groupby(0):\n                dists = list()\n                for e, k in enumerate(waypoint):\n                    dist = abs(int(gid) - int(k[0]))\n                    dists.append(dist)\n                nearest_wp_index = np.argmin(dists)\n                \n                g = g.drop_duplicates(subset=3)\n                tmp = g.iloc[:,3:5]\n                feat = tmp.set_index(3).reindex(index).replace(np.nan, -999).T\n                feat[\"x\"] = float(waypoint[nearest_wp_index][2])\n                feat[\"y\"] = float(waypoint[nearest_wp_index][3])\n                feat[\"f\"] = floor\n                feat[\"path\"] = file.split('/')[-1].split('.')[0] # useful for crossvalidation\n                dfs.append(feat)\n                \n    building_df = pd.concat(dfs)\n    building_dfs[building] = df\n    building_df.to_csv(building+\"_1000_train.csv\")\n    count+=1\nprint('Following numbers are floors identified')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#building_dfs\n#building_df.columns\nprint(\"columns: buildingID, BSSID, Timestamp\")\nssubm_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Generate the features for the test set\n\nssubm_building_g = ssubm_df.groupby(0)\nfeature_dict = dict()\n\ncount = 0\nfor gid0, g0 in ssubm_building_g:\n#    if count > 1:\n#        break\n    index = sorted(bssid[g0.iloc[0,0]])\n    feats = list()\n    print(gid0)\n    for gid,g in g0.groupby(1):\n\n        # get all wifi time locations, \n        with open(os.path.join(base_path, 'test/' + g.iloc[0,1] + '.txt')) as f:\n            txt = f.readlines()\n\n        wifi = list()\n\n        for line in txt:\n            line = line.strip().split()\n            if line[1] == \"TYPE_WIFI\":\n                wifi.append(line)\n\n        wifi_df = pd.DataFrame(wifi)\n        wifi_points = pd.DataFrame(wifi_df.groupby(0).count().index.tolist())\n        \n        for timepoint in g.iloc[:,2].tolist():\n\n            deltas = (wifi_points.astype(int) - int(timepoint)).abs()\n            min_delta_idx = deltas.values.argmin()\n            wifi_block_timestamp = wifi_points.iloc[min_delta_idx].values[0]\n            \n            wifi_block = wifi_df[wifi_df[0] == wifi_block_timestamp].drop_duplicates(subset=3)\n            feat = wifi_block.set_index(3)[4].reindex(index).fillna(-999)\n\n            feat['site_path_timestamp'] = g.iloc[0,0] + \"_\" + g.iloc[0,1] + \"_\" + timepoint\n            feats.append(feat)\n    feature_df = pd.concat(feats, axis=1).T\n    feature_df.to_csv(gid0+\"_1000_test.csv\")\n    feature_dict[gid0] = feature_df\n    count += 1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}