{"cells":[{"metadata":{},"cell_type":"markdown","source":"### Wifi features\n\nThis this is the code to generate the wifi features available in [this dataset](https://www.kaggle.com/devinanzelmo/indoor-navigation-and-location-wifi-features). Using these features can get a score below 14. For an example notebook using them see [this notebook](https://www.kaggle.com/devinanzelmo/wifi-features-lightgbm-starter). They only uses waypoints, wifi and timestamp data to generate solution. See this [forum post](https://www.kaggle.com/c/indoor-location-navigation/discussion/215445) for an outline of this solution method, and methods of improvement.\n\nThere are `break`'s inserted into loops which need to be removed to get this to run. Right now data is written to current working directory. This takes 2-4 hours to run depending on hard drive etc. There is a lot of room for improvement speeding up feature generation. \n\n**Update:** I added one line that creates a column for the path filename, this allows for a groupkfold crossvalidation. \n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport glob\nimport os\nimport gc\nimport json ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base_path = '../input/indoor-location-navigation/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# pull out all the buildings actually used in the test set, given current method we don't need the other ones\nssubm = pd.read_csv('../input/indoor-location-navigation/sample_submission.csv')\n\n# only 24 of the total buildings are used in the test set, \n# this allows us to greatly reduce the intial size of the dataset\n\nssubm_df = ssubm[\"site_path_timestamp\"].apply(lambda x: pd.Series(x.split(\"_\")))\nused_buildings = sorted(ssubm_df[0].value_counts().index.tolist())\n\n# dictionary used to map the floor codes to the values used in the submission file. \nfloor_map = {\"B2\":-2, \"B1\":-1, \"F1\":0, \"F2\": 1, \"F3\":2, \"F4\":3, \"F5\":4, \"F6\":5, \"F7\":6,\"F8\":7, \"F9\":8,\n             \"1F\":0, \"2F\":1, \"3F\":2, \"4F\":3, \"5F\":4, \"6F\":5, \"7F\":6, \"8F\": 7, \"9F\":8}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get only the wifi bssid that occur over 1000 times(this number can be experimented with)\n# these will be the only ones used when constructing features\nbssid = dict()\n\nfor building in used_buildings:\n    folders = sorted(glob.glob(os.path.join(base_path,'train/'+building+'/*')))\n    break\n    print(building)\n    wifi = list()\n    print(f\"The number of folders are: {len(folders)}\")\n    for folder in folders:\n        floor = floor_map[folder.split('/')[-1]]\n        files = glob.glob(os.path.join(folder, \"*.txt\"))\n        print(f\"The number of files in folder is: {len(files)}\")\n        for file in files:\n            with open(file) as f:\n                txt = f.readlines()\n                for e, line in enumerate(txt):\n                    tmp = line.strip().split()\n                    if tmp[1] == \"TYPE_WIFI\":\n                        wifi.append(tmp)\n    df = pd.DataFrame(wifi)\n    print(\"The dataframe is: \")\n    print(df.head())\n    value_counts = df[3].value_counts()\n    top_bssid = value_counts[value_counts > 1000].index.tolist()\n    print(f\"The len of the top bssids are: {len(top_bssid)}\")\n    # Take the top for each building max 1000 but we should take 90 % or something similar since some buildings might have a lot and some less\n    bssid[building] = top_bssid\n    del df\n    del wifi\n    gc.collect()\n    break ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(\"bssid_1000.json\", \"w\") as f:\n    json.dump(bssid, f)\n\nwith open(\"bssid_1000.json\") as f:\n    bssid = json.load(f)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# generate all the training data \nbuilding_dfs = dict()\n\nfor building in used_buildings:\n    folders = sorted(glob.glob(os.path.join(base_path,'train', building +'/*')))\n    dfs = list()\n    index = sorted(bssid[building])\n    print(f\"The building is:{building}\")\n    for folder in folders:\n        floor = floor_map[folder.split('/')[-1]]\n        files = glob.glob(os.path.join(folder, \"*.txt\"))\n        print(f\"The floor is {floor}\")\n        break\n        for file in files:\n            wifi = list()\n            waypoint = list()\n            with open(file) as f:\n                txt = f.readlines()\n            for line in txt:\n                line = line.strip().split()\n                if line[1] == \"TYPE_WAYPOINT\":\n                    waypoint.append(line) # Here we add the waypoints in the file .... \n                if line[1] == \"TYPE_WIFI\":\n                    wifi.append(line)\n            print(\"The dataframe is: \")\n            df = pd.DataFrame(np.array(wifi))    \n            print(df.head())\n            print(df.groupby(0).head())\n            # generate a feature, and label for each wifi block\n            # WE do a group by time here in order to get all that are connected at one point in time. \n            # this is needed to find the feature for one time point. \n            for gid, g in df.groupby(0):\n                # Each group is a timestamp where we connec to WIFI\n                # gid is a timestamp. \n                dists = list()\n                print(f\"The waypoint is : {waypoint}\")\n                print(f\"The gid is:{gid}\")\n                for e, k in enumerate(waypoint):\n                    dist = abs(int(gid) - int(k[0]))\n                    print(f\"K is :{k}\")\n                    print(f\"The dist is:{dist}\")\n                    # If they walk with constant speed this is distance ... \n                    # Sets the position of the wifi to the closes waypoint measurment in time. \n                    # Check which measurment is closest in time it seems like\n                    # There is no distance here that are measured ..... \n                    # Why call it distance. \n                    # Why not check signal strenght \n                    # Check which timepoint is the closes in time. \n                    # What we do is to find the position of each wifi measurment. As. aproxy we take the closes position we know of, which sis a waypoint\n                    \n                    dists.append(dist)\n                nearest_wp_index = np.argmin(dists)\n                print(f\"gid is: {gid}\")\n                print(\"g is:\" )\n                print(g)\n                # Drop duplicates based on the ssid, which is the same as \n                # SSID är en alfanumerisk nyckel på upp till 32 tecken som används för att unikt identifiera ett trådlöst nätverk. \n                print(f\"The nearest wp index is {nearest_wp_index}\")\n                g = g.drop_duplicates(subset=3)\n                tmp = g.iloc[:,3:5]\n                print(f\"tmp is {tmp}\")\n                feat = tmp.set_index(3).reindex(index).replace(np.nan, -999).T\n                print(f\" the feature is {feat}\")\n                ## Take the two nearest or even more. \n                feat[\"x\"] = float(waypoint[nearest_wp_index][2]) #closes way point\n                feat[\"y\"] = float(waypoint[nearest_wp_index][3]) #closes way point \n                feat[\"f\"] = floor\n                # The feature describe how strong the signals is for a waypoint, and thus howe close we are to it ish \n                \n                feat[\"path\"] = file.split('/')[-1].split('.')[0] # useful for crossvalidation\n                dfs.append(feat)\n    \n    building_df = pd.concat(dfs)\n    building_dfs[building] = df\n    building_df.to_csv(building+\"_1000_train.csv\")\n    break ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Generate the features for the test set\n\nssubm_building_g = ssubm_df.groupby(0)\nfeature_dict = dict()\n\nfor gid0, g0 in ssubm_building_g:\n    break\n    index = sorted(bssid[g0.iloc[0,0]])\n    feats = list()\n    print(gid0)\n    for gid,g in g0.groupby(1):\n\n        # get all wifi time locations, \n        with open(os.path.join(base_path, 'test/' + g.iloc[0,1] + '.txt')) as f:\n            txt = f.readlines()\n\n        wifi = list()\n\n        for line in txt:\n            line = line.strip().split()\n            if line[1] == \"TYPE_WIFI\":\n                wifi.append(line)\n\n        wifi_df = pd.DataFrame(wifi)\n        wifi_points = pd.DataFrame(wifi_df.groupby(0).count().index.tolist())\n        \n        for timepoint in g.iloc[:,2].tolist():\n\n            deltas = (wifi_points.astype(int) - int(timepoint)).abs()\n            min_delta_idx = deltas.values.argmin()\n            wifi_block_timestamp = wifi_points.iloc[min_delta_idx].values[0]\n            \n            wifi_block = wifi_df[wifi_df[0] == wifi_block_timestamp].drop_duplicates(subset=3)\n            feat = wifi_block.set_index(3)[4].reindex(index).fillna(-999)\n\n            feat['site_path_timestamp'] = g.iloc[0,0] + \"_\" + g.iloc[0,1] + \"_\" + timepoint\n            feats.append(feat)\n    feature_df = pd.concat(feats, axis=1).T\n    feature_df.to_csv(gid0+\"_1000_test.csv\")\n    feature_dict[gid0] = feature_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}