{"cells":[{"metadata":{},"cell_type":"markdown","source":"### Wifi features\n\nThis this is the code to generate the wifi features available in [this dataset](https://www.kaggle.com/devinanzelmo/indoor-navigation-and-location-wifi-features). Using these features can get a score below 14. For an example notebook using them see [this notebook](https://www.kaggle.com/devinanzelmo/wifi-features-lightgbm-starter). They only uses waypoints, wifi and timestamp data to generate solution. See this [forum post](https://www.kaggle.com/c/indoor-location-navigation/discussion/215445) for an outline of this solution method, and methods of improvement.\n\nThere are `break`'s inserted into loops which need to be removed to get this to run. Right now data is written to current working directory. This takes 2-4 hours to run depending on hard drive etc. There is a lot of room for improvement speeding up feature generation. \n\n**Update:** I added one line that creates a column for the path filename, this allows for a groupkfold crossvalidation. \n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport glob\nimport os\nimport gc\nimport json ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base_path = '../input/indoor-location-navigation/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# pull out all the buildings actually used in the test set, given current method we don't need the other ones\nssubm = pd.read_csv('../input/indoor-location-navigation/sample_submission.csv')\n\n# only 24 of the total buildings are used in the test set, \n# this allows us to greatly reduce the intial size of the dataset\n\nssubm_df = ssubm[\"site_path_timestamp\"].apply(lambda x: pd.Series(x.split(\"_\")))\nused_buildings = sorted(ssubm_df[0].value_counts().index.tolist())\n\n# dictionary used to map the floor codes to the values used in the submission file. \nfloor_map = {\"B2\":-2, \"B1\":-1, \"F1\":0, \"F2\": 1, \"F3\":2, \"F4\":3, \"F5\":4, \"F6\":5, \"F7\":6,\"F8\":7, \"F9\":8,\n             \"1F\":0, \"2F\":1, \"3F\":2, \"4F\":3, \"5F\":4, \"6F\":5, \"7F\":6, \"8F\": 7, \"9F\":8}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get only the wifi bssid that occur over 1000 times(this number can be experimented with)\n# these will be the only ones used when constructing features\nbssid = dict()\n\nfor building in used_buildings:\n    break\n    folders = sorted(glob.glob(os.path.join(base_path,'train/'+building+'/*')))\n    print(building)\n    wifi = list()\n    for folder in folders:\n        floor = floor_map[folder.split('/')[-1]]\n        files = glob.glob(os.path.join(folder, \"*.txt\"))\n        for file in files:\n            with open(file) as f:\n                txt = f.readlines()\n                for e, line in enumerate(txt):\n                    tmp = line.strip().split()\n                    if tmp[1] == \"TYPE_WIFI\":\n                        wifi.append(tmp)\n    df = pd.DataFrame(wifi)\n    #top_bssid = df[3].value_counts().iloc[:500].index.tolist()\n    value_counts = df[3].value_counts()\n    top_bssid = value_counts[value_counts > 1000].index.tolist()\n    print(len(top_bssid))\n    bssid[building] = top_bssid\n    del df\n    del wifi\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(\"bssid_1000.json\", \"w\") as f:\n    json.dump(bssid, f)\n\nwith open(\"bssid_1000.json\") as f:\n    bssid = json.load(f)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# generate all the training data \nbuilding_dfs = dict()\n\nfor building in used_buildings:\n    break\n    folders = sorted(glob.glob(os.path.join(base_path,'train', building +'/*')))\n    dfs = list()\n    index = sorted(bssid[building])\n    print(building)\n    for folder in folders:\n        floor = floor_map[folder.split('/')[-1]]\n        files = glob.glob(os.path.join(folder, \"*.txt\"))\n        print(floor)\n        for file in files:\n            wifi = list()\n            waypoint = list()\n            with open(file) as f:\n                txt = f.readlines()\n            for line in txt:\n                line = line.strip().split()\n                if line[1] == \"TYPE_WAYPOINT\":\n                    waypoint.append(line)\n                if line[1] == \"TYPE_WIFI\":\n                    wifi.append(line)\n\n            df = pd.DataFrame(np.array(wifi))    \n\n            # generate a feature, and label for each wifi block\n            for gid, g in df.groupby(0):\n                dists = list()\n                for e, k in enumerate(waypoint):\n                    dist = abs(int(gid) - int(k[0]))\n                    dists.append(dist)\n                nearest_wp_index = np.argmin(dists)\n                \n                g = g.drop_duplicates(subset=3)\n                tmp = g.iloc[:,3:5]\n                feat = tmp.set_index(3).reindex(index).replace(np.nan, -999).T\n                feat[\"x\"] = float(waypoint[nearest_wp_index][2])\n                feat[\"y\"] = float(waypoint[nearest_wp_index][3])\n                feat[\"f\"] = floor\n                feat[\"path\"] = file.split('/')[-1].split('.')[0] # useful for crossvalidation\n                dfs.append(feat)\n                \n    building_df = pd.concat(dfs)\n    building_dfs[building] = df\n    building_df.to_csv(building+\"_1000_train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Generate the features for the test set\n\nssubm_building_g = ssubm_df.groupby(0)\nfeature_dict = dict()\n\nfor gid0, g0 in ssubm_building_g:\n    break\n    index = sorted(bssid[g0.iloc[0,0]])\n    feats = list()\n    print(gid0)\n    for gid,g in g0.groupby(1):\n\n        # get all wifi time locations, \n        with open(os.path.join(base_path, 'test/' + g.iloc[0,1] + '.txt')) as f:\n            txt = f.readlines()\n\n        wifi = list()\n\n        for line in txt:\n            line = line.strip().split()\n            if line[1] == \"TYPE_WIFI\":\n                wifi.append(line)\n\n        wifi_df = pd.DataFrame(wifi)\n        wifi_points = pd.DataFrame(wifi_df.groupby(0).count().index.tolist())\n        \n        for timepoint in g.iloc[:,2].tolist():\n\n            deltas = (wifi_points.astype(int) - int(timepoint)).abs()\n            min_delta_idx = deltas.values.argmin()\n            wifi_block_timestamp = wifi_points.iloc[min_delta_idx].values[0]\n            \n            wifi_block = wifi_df[wifi_df[0] == wifi_block_timestamp].drop_duplicates(subset=3)\n            feat = wifi_block.set_index(3)[4].reindex(index).fillna(-999)\n\n            feat['site_path_timestamp'] = g.iloc[0,0] + \"_\" + g.iloc[0,1] + \"_\" + timepoint\n            feats.append(feat)\n    feature_df = pd.concat(feats, axis=1).T\n    feature_df.to_csv(gid0+\"_1000_test.csv\")\n    feature_dict[gid0] = feature_df","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}