{"cells":[{"metadata":{},"cell_type":"markdown","source":"# wifi features\nHuge thanks to the great [wifi fearures](https://www.kaggle.com/devinanzelmo/wifi-features) notebook by [Devin Anzelmo](https://www.kaggle.com/devinanzelmo). I learned a lot from the notebook.\nI've made a small chage to the notebook and now it runs faster ~40min instead of 2-4 hours. Hope this helps some kagglers!\n\nIn case you find a bug please leave comments here :)"},{"metadata":{"_uuid":"864cdaf7-bc14-47d2-afd8-672ebacab5f9","_cell_guid":"c09e4adb-e879-4e9f-b45f-0be2a49319b0","trusted":true},"cell_type":"code","source":"import collections\nimport csv\nimport glob\nimport multiprocessing\nimport os\nfrom multiprocessing import Pool\nfrom pathlib import Path\nfrom typing import List, Tuple, Any\n\nimport numpy as np\nimport pandas as pd\n\n\ndef input_dir() -> Path:\n    return Path('/kaggle/input/indoor-location-navigation/')\n\n\ndef extract_wps_wifis(file: Path) -> Tuple[List[str], List[str]]:\n    wps = []\n    wifis = []\n    with open(file) as f:\n        for row in csv.reader(f, delimiter=\"\\t\", doublequote=True):\n            if row[1] == \"TYPE_WAYPOINT\":\n                # x\n                row[2] = float(row[2])  # type: ignore\n                # y\n                row[3] = float(row[3])  # type: ignore\n                wps.append(row)\n            elif row[1] == \"TYPE_WIFI\":\n                # wifi signal value\n                row[4] = int(row[4])  # type: ignore\n                wifis.append(row)\n    wps = sorted(wps, key=lambda x: x[0])  # timestamp\n    wifis = sorted(wifis, key=lambda x: x[0])  # timestamp\n    return wps, wifis\n\n\ndef top_bssids(bssids: List[str], n: int) -> List[str]:\n    df = pd.DataFrame(bssids)\n    value_counts = df[0].value_counts() # type: ignore\n    return sorted(value_counts[value_counts > n].index.tolist())\n\n\ndef top_bssids_for_building(input_dir: Path, building: str, n: int) -> List[str]:\n    folders = sorted(glob.glob(os.path.join(\n        input_dir, 'train/' + building+'/*')))\n    bssids = []\n    for folder in folders:\n        files = glob.glob(os.path.join(folder, \"*.txt\"))\n        for file in files:\n            _, wifis = extract_wps_wifis(Path(file))\n            bssids.extend([wifi[3] for wifi in wifis])\n\n    return top_bssids(bssids, n)\n\n\ndef nearest_waypoint(timestamp: int, wps: List[List[str]]) -> List[str]:\n    dists = []\n    for wp in wps:\n        # timestamp delta\n        dist = abs(timestamp - int(wp[0]))\n        dists.append(dist)\n    nearest_index = np.argmin(dists)\n    return wps[nearest_index]\n\n\n# Note: This can have exact same rows in train. Because both wifi_group_a and wifi_group_b can be nearest to a certain waypoint and wifi_group_a and wifi_group_b are the same.\ndef generate_train_for_building(building_path: Path, bssids: List[str]) -> pd.Series:\n    dfs = []\n    folders = sorted(building_path.glob('*'))\n    for folder in folders:\n        files = folder.glob(\"*.txt\")\n        for file in files:\n            rows = generate_train_for_path(file, bssids)\n            dfs.extend(rows)\n    building_df = pd.concat(dfs)\n    building_df.reset_index(drop=True, inplace=True)\n    type_map = {column: int for column in bssids}\n    building_df = building_df.astype(type_map) # type: ignore\n    return building_df\n\n\ndef generate_train_for_path(path_file: Path, bssids: List[str]) -> List[Any]:\n    floor = str(path_file.parent.name)\n    wps, wifis = extract_wps_wifis(path_file)\n    wifis_df = pd.DataFrame(wifis, columns=[\n                            'timestamp', 'type', 'ssid', 'bssid', 'value', 'channel', 'last_timestamp'])\n    rows = []\n    for timestamp, wifi_group in wifis_df.groupby('timestamp'):\n        timestamp = int(timestamp)\n        path = path_file.stem\n        row = generate_train_for_timestamp(\n            timestamp, wifi_group, wps, floor, path, bssids)\n        rows.append(row)\n    return rows\n\n\ndef generate_train_for_timestamp(timestamp: int, wifi_group: pd.DataFrame, wps: List[Any], floor: str, path: str, bssids: List[str]) -> pd.DataFrame:\n    floor_map = {\"B2\": -2, \"B1\": -1, \"F1\": 0, \"F2\": 1, \"F3\": 2, \"F4\": 3, \"F5\": 4, \"F6\": 5, \"F7\": 6, \"F8\": 7, \"F9\": 8,\n                 \"1F\": 0, \"2F\": 1, \"3F\": 2, \"4F\": 3, \"5F\": 4, \"6F\": 5, \"7F\": 6, \"8F\": 7, \"9F\": 8}\n    waypoint = nearest_waypoint(timestamp, wps)\n    wifi_group = wifi_group.drop_duplicates(subset='bssid')\n    tmp = wifi_group.iloc[:, 3:5]  # bssid and value\n    row = tmp.set_index('bssid').reindex(bssids).replace(np.nan, -999).T\n    row[\"x\"] = float(waypoint[2])\n    row[\"y\"] = float(waypoint[3])\n    row[\"f\"] = floor_map[floor]\n    row[\"path\"] = path\n    return row\n\n\ndef generate_target_buildings() -> List[str]:\n    ssubm = pd.read_csv(\n        '/kaggle/input/indoor-location-navigation/sample_submission.csv')\n    ssubm_df = ssubm[\"site_path_timestamp\"].apply(\n        lambda x: pd.Series(x.split(\"_\")))\n    return sorted(ssubm_df[0].value_counts().index.tolist()) # type: ignore\n\n\ndef generate_one(building: str):\n    print(f\"start:{building}\")\n    building_path = input_dir() / 'train' / building\n    bssids = top_bssids_for_building(input_dir(), building, 1000)\n    train_df = generate_train_for_building(building_path, bssids)\n    train_df.to_csv(f'{building}_train.csv', index=False)\n    print(f\"end:{building}\")\n\n\ndef generate_train():\n    num_cores = multiprocessing.cpu_count()\n    print(f\"num_cores={num_cores}\")\n    pool = Pool(num_cores)\n    pool.map(generate_one, generate_target_buildings())\n\n\ndef generate_test_one(building_df: pd.DataFrame):\n    building = building_df.iloc[0, 0]\n    print(f\"start: {building}\")\n    bssids = top_bssids_for_building(input_dir(), building, 1000) # type: ignore\n    feats = []\n    # group by path\n    for path, path_df in building_df.groupby('path'):\n        _, wifis = extract_wps_wifis(input_dir() / 'test' / f'{path}.txt')\n\n        wifi_df = pd.DataFrame(wifis)\n        wifi_points = pd.DataFrame(wifi_df.groupby(0).count().index.tolist())\n        for timepoint in path_df.iloc[:, 2].tolist():\n            deltas = (wifi_points.astype(int) - int(timepoint)).abs()\n            min_delta_idx = deltas.values.argmin()\n            wifi_block_timestamp = wifi_points.iloc[min_delta_idx].values[0]\n\n            wifi_block = wifi_df[wifi_df[0] ==\n                                 wifi_block_timestamp].drop_duplicates(subset=3)\n            feat = wifi_block.set_index(3)[4].reindex(bssids).fillna(-999)\n\n            feat['site_path_timestamp'] = f'{building}_{path}_{timepoint}'\n            feats.append(feat)\n    feature_df = pd.concat(feats, axis=1).T\n    feature_df.to_csv(f\"{building}_test.csv\", index=False)\n    print(f'end: {building}')\n\n\ndef generate_test():\n    sub_df = pd.read_csv(\n        '/kaggle/input/indoor-location-navigation/sample_submission.csv')\n    sub_df = sub_df[\"site_path_timestamp\"].apply(\n        lambda x: pd.Series(x.split(\"_\")))\n    sub_df.columns = ['site', 'path', 'timestamp']\n\n    building_dfs = [building_df for _, building_df in sub_df.groupby('site')]\n\n    num_cores = multiprocessing.cpu_count()\n    print(f\"num_cores={num_cores}\")\n    pool = Pool(num_cores)\n    pool.map(generate_test_one, building_dfs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"generate_train()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"generate_test()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}