{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport csv\nimport glob\nimport json\nimport pickle\nimport collections\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom pathlib import Path\nfrom typing import List, Tuple, Any\n\nimport multiprocessing\nfrom multiprocessing import Pool\n\nimport dask\nfrom dask.distributed import wait\nfrom dask.distributed import Client, wait, LocalCluster","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# set n_workers to number of cores\nclient = Client(n_workers=multiprocessing.cpu_count(), threads_per_worker=1)\nclient","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"floor_map = {\"B2\": -2, \"B1\": -1, \"F1\": 0, \"F2\": 1, \"F3\": 2, \"F4\": 3, \"F5\": 4, \"F6\": 5, \"F7\": 6, \"F8\": 7, \"F9\": 8,\n             \"1F\": 0, \"2F\": 1, \"3F\": 2, \"4F\": 3, \"5F\": 4, \"6F\": 5, \"7F\": 6, \"8F\": 7, \"9F\": 8}\n\n\nminCount = 1\nrssiFillerValue = -999.0\ndtFillerValue   = 1000.0\nfreqFillerValue = 0\noutputDir = '.'\nsampleCsvPath = '../input/indoor-location-navigation/sample_submission.csv'\nbuildingBssidPklFilePath = \"../input/idln-temp-files-version-1/buildingBssids.pkl\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def input_dir() -> Path:\n    #return Path('.')\n    return Path('../input/indoor-location-navigation')\n\ndef generate_target_buildings() -> List[str]:\n    ssubm = pd.read_csv(sampleCsvPath)\n    ssubm_df = ssubm[\"site_path_timestamp\"].apply(lambda x: pd.Series(x.split(\"_\")))\n    return sorted(ssubm_df[0].value_counts().index.tolist()) # type: ignore\n\ndef extract_wps_wifis(file: Path) -> Tuple[List[str], List[str]]:\n    wps = []\n    wifis = []\n    with open(file) as f:\n        for row in csv.reader(f, delimiter=\"\\t\", doublequote=True):\n            if row[1] == \"TYPE_WAYPOINT\":\n                # x\n                row[2] = float(row[2])  # type: ignore\n                # y\n                row[3] = float(row[3])  # type: ignore\n                wps.append([int(row[0]), row[2], row[3]])\n            elif row[1] == \"TYPE_WIFI\":\n                # wifi signal value\n                row[4] = int(row[4])  # type: ignore\n                wifis.append(row)\n    wps = sorted(wps, key=lambda x: x[0])  # timestamp\n    wifis = sorted(wifis, key=lambda x: x[0])  # timestamp\n    return wps, wifis","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generateFloorBssids_train(building : str):\n    \"\"\"\n    for a given building in train set, this function calculates\n    the unique wifi bssids within each floor\n    \n    returns :  a dict with keys = {building}_{floor}, values = list(set(floorBssids))\n    \"\"\"\n    building_path = input_dir() / 'train' / building\n    floorBssids = {}\n    folders = sorted(building_path.glob('*'))\n    for folder in folders:\n        folderData = []\n        files = folder.glob(\"*.txt\")\n        for file in files:\n            _, wifiData = extract_wps_wifis(file)\n            folderData.extend([t[3] for t in wifiData])\n        floorBssids[f\"{folder.name}\"] = list(set(folderData))\n    return floorBssids","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generateIntersectingBssids(floorBssids):\n    \"\"\"\n    given a dict with floor -> bssid list mapping,\n    returns : list of bssids which occur in more than 1 floor \n    \"\"\"\n    commonBssids = []\n    for k1, v1 in floorBssids.items():\n        for k2, v2 in floorBssids.items():\n            if (k1 != k2):\n                intersectingBssids = list(set(v1).intersection(set(v2)))\n                commonBssids.extend(intersectingBssids)\n    return commonBssids","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def printBssidsInfo(floorBssids):\n    \"\"\"\n    given a dict with floor -> bssid list mapping,m\n    Function calcuates total number of bssids and number \n    of unique bssids and prints result\n    \"\"\"\n    totalBssids = []\n    for k,v in floorBssids.items():\n        ## print(f\"{k} has {len(v)} total bssids\")\n        totalBssids.extend(v)\n    \n    print(f\"Totally, There are {len(totalBssids)} bssids\")\n    print(f\"There are {len(set(totalBssids))} unique bssids\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generateUniqueFloorBssids(floorBssids, commonBssids):\n    \"\"\"\n    floorBssids : a dict with floor -> bssid list mapping\n    commonBssids : list of bssids which are present in more than one floor\n    \"\"\"\n    uniqueFloorBssids = {}\n    for k,v in floorBssids.items():\n        uniqueFloorBssids[k] = list(set(v) - set(commonBssids))    \n    return uniqueFloorBssids","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generateFloorUniqueBssidData(buildingsList):\n    floorUniqueBssidData = {}\n    for building in buildingsList:\n        floorBssids = generateFloorBssids_train(building)    # consumes most time\n        commonBssids = generateIntersectingBssids(floorBssids)\n        uniqueBssids = generateUniqueFloorBssids(floorBssids, commonBssids)\n\n        print(building)\n        print(f\"building unique bssids information\")\n        printBssidsInfo(floorBssids)\n        print(f\"Floor unique bssids information\")\n        printBssidsInfo(uniqueBssids)\n        print('-----------------------------------')\n\n        floorUniqueBssidData[building] = uniqueBssids\n    \n    with open(\"floorUniqueBssidData.json\", \"w\") as outfile: \n        json.dump(floorUniqueBssidData, outfile)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class wifiAPFloorMapping:\n    def __init__(self, building, floorUniqueBssidMapPath):\n        self.building = building\n        self.floorList = self.getBuildingFloorList()\n        self.uniqueBssidsMap = self.getFloorUniqueBssids(floorUniqueBssidMapPath)\n        self.outputData = self.generateOutputData()\n    \n    def getFloorUniqueBssids(self, floorUniqueBssidMapPath):\n        with open(floorUniqueBssidMapPath, \"r\") as infile:\n            floorUniqueBssidMap = json.load(infile)\n        return floorUniqueBssidMap.get(self.building, None)\n    \n    def getBuildingFloorList(self):\n        buildingPath = input_dir() / 'train' / self.building\n        folders = sorted(buildingPath.glob('*'))\n        return [folder.name for folder in folders] + ['common']\n\n    def generateOutputData(self):\n        outputData = {'pathName' : []}\n        for floor in self.floorList:\n            outputData[f\"{floor}_count\"] = []\n            outputData[f\"{floor}_mean\"]  = []\n            outputData[f\"{floor}_median\"]  = []\n        return outputData\n    \n    def findMappingFloor(self, wifiAp):\n        matchingfloor = 'common'\n        if self.uniqueBssidsMap is not None:\n            for floor, floorAPList in self.uniqueBssidsMap.items():\n                if wifiAp in floorAPList:\n                    matchingfloor = floor\n                    break\n        return matchingfloor \n    \n    def getPathFileWiFiData(self, file):\n        _, wifiData = extract_wps_wifis(file)\n        wifiData = pd.DataFrame(wifiData, columns = ['timestamp','type','ssid', 'bssid', 'rssi', 'freq', 'last_ts'])\n        wifiData.drop(labels=['timestamp', 'type', 'ssid', 'freq', 'last_ts'], axis=1, inplace=True)\n        wifiData['mappedFloor'] = wifiData['bssid'].apply(self.findMappingFloor)\n        return wifiData\n        \n    def updatePathFileToOutput(self, file):\n        # add path file to output\n        self.outputData['pathName'].append(file.name.split('.')[0])\n        \n        # create wifi data of pathfile\n        wifiData = self.getPathFileWiFiData(file)\n        \n        # shortlist wifi bssids based on each floor list\n        for floor in self.floorList:\n            rssiData = wifiData[wifiData['mappedFloor']== floor]['rssi'].values\n            if len(rssiData) > 0:\n                mean, median = np.mean(rssiData), np.median(rssiData)\n            else:\n                mean, median = 0.0, 0.0\n                \n            self.outputData[f\"{floor}_count\"].append(len(rssiData))\n            self.outputData[f\"{floor}_mean\"].append(mean)\n            self.outputData[f\"{floor}_median\"].append(median)\n        del wifiData","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"buildingsList = generate_target_buildings()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"```python\n%%time\nbuilding_path = input_dir() / 'test'\n# output placeholder\ntestPathFloorPredictions = { 'building' : [], 'pathName' : [], 'predFloor':[] }\n\nfor building in tqdm(buildingsList):\n    buildingTestPathFiles = ssubm_df[ssubm_df[0] == building][1].unique()\n    #print(f\"There are {len(buildingTestPathFiles)} test path files in building\")\n    \n    temp = wifiAPFloorMapping(building, '../input/idln-temp-files-version-1/floorUniqueBssidData.json')\n    for testPathFile in buildingTestPathFiles:\n        temp.updatePathFileToOutput(building_path / f\"{testPathFile}.txt\")  \n    \n    outputDf = pd.DataFrame(temp.outputData)\n    totalColumns = list(set(temp.floorList) - set(['common']))\n    columnsOfInterest = [f\"{x}_count\" for x in totalColumns]\n    \n    # taking the maximum count as floor prediction\n    outputDf['predFloor'] = pd.Series([totalColumns[idx] for idx in outputDf[columnsOfInterest].values.argmax(axis=1)]).map(floor_map)\n    \n    # write to output variable\n    testPathFloorPredictions['pathName'].extend(outputDf['pathName'].values.tolist())\n    testPathFloorPredictions['predFloor'].extend(outputDf['predFloor'].values.tolist())\n    testPathFloorPredictions['building'].extend([building] * outputDf.shape[0])    \n```","metadata":{}},{"cell_type":"markdown","source":"```python\ntestPathFloorPredictions = pd.DataFrame(testPathFloorPredictions)\ntestPathFloorPredictions.to_csv('testPathFloorPredictions.csv',index=False)\ntestPathFloorPredictions.head(3)\n```","metadata":{}},{"cell_type":"code","source":"def pathFloorMapping(inputPath, csvMap):\n    return int(csvMap[csvMap['pathName']== inputPath]['predFloor'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read submission csv file and apply our floor mappings to it\nssubm = pd.read_csv(sampleCsvPath)\nssubm_df = ssubm[\"site_path_timestamp\"].apply(lambda x: pd.Series(x.split(\"_\")))\nownCsv = pd.read_csv('../input/idln-temp-files-version-1/testPathFloor_Mapping.csv')\nssubm_df['floor'] = ssubm_df[1].apply(pathFloorMapping, csvMap=ownCsv)\nssubm_df['site_path_timestamp'] = ssubm_df[0].astype(str) + '_' + ssubm_df[1].astype(str) + '_' + ssubm_df[2].astype(str) \nssubm_df.to_csv('testFloorPredictionsSubmssion.csv',index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ssubm_df.head(3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputCsv = pd.read_csv('../input/idln-temp-files-version-1/99_floorPredictionss_submission.csv')\nmatchingRows = (inputCsv['site_path_timestamp'] == ssubm_df['site_path_timestamp']).all()\n\nif matchingRows == True:\n    #inputCsv['floor'] = ssubm_df['floor']\n    inputCsv.to_csv('submission.csv',index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}