{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Generating basic features"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\nimport collections\nimport csv\nimport glob\nimport multiprocessing\nimport os\nfrom multiprocessing import Pool\nfrom pathlib import Path\nfrom typing import List, Tuple, Any","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def input_dir() -> Path:\n    return Path('/kaggle/input/indoor-location-navigation/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def extract_wps_wifis(file: Path) -> Tuple[List[str], List[str]]:\n    wps = []\n    wifis = []\n    with open(file) as f:\n        for row in csv.reader(f, delimiter=\"\\t\", doublequote=True):\n            if row[1] == \"TYPE_WAYPOINT\":\n                # x\n                row[2] = float(row[2])  # type: ignore\n                # y\n                row[3] = float(row[3])  # type: ignore\n                wps.append(row)\n            elif row[1] == \"TYPE_WIFI\":\n                # wifi signal value\n                row[4] = int(row[4])  # type: ignore\n                wifis.append(row)\n    wps = sorted(wps, key=lambda x: x[0])  # timestamp\n    wifis = sorted(wifis, key=lambda x: x[0])  # timestamp\n    return wps, wifis\n\n\ndef top_bssids(bssids: List[str], n: int) -> List[str]:\n    df = pd.DataFrame(bssids)\n    value_counts = df[0].value_counts() # type: ignore\n    return sorted(value_counts[value_counts > n].index.tolist())\n\n\ndef top_bssids_for_building(input_dir: Path, building: str, n: int) -> List[str]:\n    folders = sorted(glob.glob(os.path.join(\n        input_dir, 'train/' + building+'/*')))\n    bssids = []\n    for folder in folders:\n        files = glob.glob(os.path.join(folder, \"*.txt\"))\n        for file in files:\n            _, wifis = extract_wps_wifis(Path(file))\n            bssids.extend([wifi[3] for wifi in wifis])\n\n    return top_bssids(bssids, n)\n\n\ndef nearest_waypoint(timestamp: int, wps: List[List[str]]) -> List[str]:\n    dists = []\n    for wp in wps:\n        # timestamp delta\n        dist = abs(timestamp - int(wp[0]))\n        dists.append(dist)\n    nearest_index = np.argmin(dists)\n    return wps[nearest_index]\n\n\n# Note: This can have exact same rows in train. Because both wifi_group_a and wifi_group_b can be nearest to a certain waypoint and wifi_group_a and wifi_group_b are the same.\ndef generate_train_for_building(building_path: Path, bssids: List[str]) -> pd.Series:\n    dfs = []\n    folders = sorted(building_path.glob('*'))\n    for folder in folders:\n        files = folder.glob(\"*.txt\")\n        for file in files:\n            rows = generate_train_for_path(file, bssids)\n            dfs.extend(rows)\n    building_df = pd.concat(dfs)\n    building_df.reset_index(drop=True, inplace=True)\n    type_map = {column: int for column in bssids}\n    building_df = building_df.astype(type_map) # type: ignore\n    return building_df\n\n\ndef generate_train_for_path(path_file: Path, bssids: List[str]) -> List[Any]:\n    floor = str(path_file.parent.name)\n    wps, wifis = extract_wps_wifis(path_file)\n    wifis_df = pd.DataFrame(wifis, columns=[\n                            'timestamp', 'type', 'ssid', 'bssid', 'value', 'channel', 'last_timestamp'])\n    rows = []\n    for timestamp, wifi_group in wifis_df.groupby('timestamp'):\n        timestamp = int(timestamp)\n        path = path_file.stem\n        row = generate_train_for_timestamp(\n            timestamp, wifi_group, wps, floor, path, bssids)\n        rows.append(row)\n    return rows\n\n\ndef generate_train_for_timestamp(timestamp: int, wifi_group: pd.DataFrame, wps: List[Any], floor: str, path: str, bssids: List[str]) -> pd.DataFrame:\n    floor_map = {\"B2\": -2, \"B1\": -1, \"F1\": 0, \"F2\": 1, \"F3\": 2, \"F4\": 3, \"F5\": 4, \"F6\": 5, \"F7\": 6, \"F8\": 7, \"F9\": 8,\n                 \"1F\": 0, \"2F\": 1, \"3F\": 2, \"4F\": 3, \"5F\": 4, \"6F\": 5, \"7F\": 6, \"8F\": 7, \"9F\": 8}\n    waypoint = nearest_waypoint(timestamp, wps)\n    wifi_group = wifi_group.drop_duplicates(subset='bssid')\n    tmp = wifi_group.iloc[:, 3:5]  # bssid and value\n    row = tmp.set_index('bssid').reindex(bssids).replace(np.nan, -999).T\n    row[\"x\"] = float(waypoint[2])\n    row[\"y\"] = float(waypoint[3])\n    row[\"f\"] = floor_map[floor]\n    row[\"path\"] = path\n    return row\n\n\ndef generate_target_buildings() -> List[str]:\n    ssubm = pd.read_csv(\n        '/kaggle/input/indoor-location-navigation/sample_submission.csv')\n    ssubm_df = ssubm[\"site_path_timestamp\"].apply(\n        lambda x: pd.Series(x.split(\"_\")))\n    return sorted(ssubm_df[0].value_counts().index.tolist()) # type: ignore\n\n\ndef generate_one(building: str):\n    print(f\"start:{building}\")\n    building_path = input_dir() / 'train' / building\n    bssids = top_bssids_for_building(input_dir(), building, 1000)\n    train_df = generate_train_for_building(building_path, bssids)\n    train_df.to_csv(f'{building}_train.csv', index=False)\n    print(f\"end:{building}\")\n\n\ndef generate_train():\n    num_cores = multiprocessing.cpu_count()\n    print(f\"num_cores={num_cores}\")\n    pool = Pool(num_cores)\n    pool.map(generate_one, generate_target_buildings())\n\n\ndef generate_test_one(building_df: pd.DataFrame):\n    building = building_df.iloc[0, 0]\n    print(f\"start: {building}\")\n    bssids = top_bssids_for_building(input_dir(), building, 1000) # type: ignore\n    feats = []\n    # group by path\n    for path, path_df in building_df.groupby('path'):\n        _, wifis = extract_wps_wifis(input_dir() / 'test' / f'{path}.txt')\n\n        wifi_df = pd.DataFrame(wifis)\n        wifi_points = pd.DataFrame(wifi_df.groupby(0).count().index.tolist())\n        for timepoint in path_df.iloc[:, 2].tolist():\n            deltas = (wifi_points.astype(int) - int(timepoint)).abs()\n            min_delta_idx = deltas.values.argmin()\n            wifi_block_timestamp = wifi_points.iloc[min_delta_idx].values[0]\n\n            wifi_block = wifi_df[wifi_df[0] ==\n                                 wifi_block_timestamp].drop_duplicates(subset=3)\n            feat = wifi_block.set_index(3)[4].reindex(bssids).fillna(-999)\n\n            feat['site_path_timestamp'] = f'{building}_{path}_{timepoint}'\n            feats.append(feat)\n    feature_df = pd.concat(feats, axis=1).T\n    feature_df.to_csv(f\"{building}_test.csv\", index=False)\n    print(f'end: {building}')\n\n\ndef generate_test():\n    sub_df = pd.read_csv(\n        '/kaggle/input/indoor-location-navigation/sample_submission.csv')\n    sub_df = sub_df[\"site_path_timestamp\"].apply(\n        lambda x: pd.Series(x.split(\"_\")))\n    sub_df.columns = ['site', 'path', 'timestamp']\n\n    building_dfs = [building_df for _, building_df in sub_df.groupby('site')]\n\n    num_cores = multiprocessing.cpu_count()\n    print(f\"num_cores={num_cores}\")\n    pool = Pool(num_cores)\n    pool.map(generate_test_one, building_dfs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"generate_train()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"generate_test()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# LightGBM as regressor"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ------------------------------------------------------------------------------\n# Import libraries\n# ------------------------------------------------------------------------------\nimport numpy as np\nimport pandas as pd\nimport scipy.stats as stats\nfrom pathlib import Path\nimport glob\n\nfrom sklearn.model_selection import KFold\nimport lightgbm as lgb\n\nimport psutil\nimport random\nimport os\nimport time\nimport sys\nimport math\nfrom contextlib import contextmanager\n\n# ------------------------------------------------------------------------------\n# Fixed values\n# ------------------------------------------------------------------------------\nN_SPLITS = 2\nSEED = 42\n\n# ------------------------------------------------------------------------------\n# File path definition\n# ------------------------------------------------------------------------------\nLOG_PATH = Path(\"./log/\")\nLOG_PATH.mkdir(parents=True, exist_ok=True)\n\n\n# ------------------------------------------------------------------------------\n# Utilities\n# ------------------------------------------------------------------------------\n@contextmanager\ndef timer(name: str):\n    t0 = time.time()\n    p = psutil.Process(os.getpid())\n    m0 = p.memory_info()[0] / 2. ** 30\n    try:\n        yield\n    finally:\n        m1 = p.memory_info()[0] / 2. ** 30\n        delta = m1 - m0\n        sign = '+' if delta >= 0 else '-'\n        delta = math.fabs(delta)\n        print(f\"[{m1:.1f}GB({sign}{delta:.1f}GB): {time.time() - t0:.3f}sec] {name}\", file=sys.stderr)\n\n\ndef set_seed(seed=42):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n\n    \ndef comp_metric(xhat, yhat, fhat, x, y, f):\n    intermediate = np.sqrt(np.power(xhat-x, 2) + np.power(yhat-y, 2)) + 15 * np.abs(fhat-f)\n    return intermediate.sum()/xhat.shape[0]\n\n\ndef score_log(df: pd.DataFrame, num_files: int, nam_file: str, data_shape: tuple, n_fold: int, seed: int, mpe: float):\n    score_dict = {'n_files': num_files, 'file_name': nam_file, 'shape': data_shape, 'fold': n_fold, 'seed': seed, 'score': mpe}\n    # noinspection PyTypeChecker\n    df = pd.concat([df, pd.DataFrame.from_dict([score_dict])])\n    df.to_csv(LOG_PATH / f\"log_score.csv\", index=False)\n    return df\n\n\n\n\n# ------------------------------------------------------------------------------\n# Set seed\n# ------------------------------------------------------------------------------\nset_seed(SEED)\n\n# ------------------------------------------------------------------------------\n# Read data\n# ------------------------------------------------------------------------------\nfeature_dir = \"../input/indoor-navigation-and-location-wifi-features\"\ntrain_files = sorted(glob.glob(os.path.join(feature_dir, '*_train.csv')))\ntest_files = sorted(glob.glob(os.path.join(feature_dir, '*_test.csv')))\nsubm = pd.read_csv('../input/indoor-location-navigation/sample_submission.csv', index_col=0)\n\n# ------------------------------------------------------------------------------\n# Define parameters for models\n# ------------------------------------------------------------------------------\nlgb_params = {'objective': 'root_mean_squared_error',\n              'boosting_type': 'gbdt',\n              'n_estimators': 50000,\n              'learning_rate': 0.1,\n              'num_leaves': 90,\n              'colsample_bytree': 0.4,\n              'subsample': 0.6,\n              'subsample_freq': 2,\n              'bagging_seed': SEED,\n              'reg_alpha': 8,\n              'reg_lambda': 2,\n              'random_state': SEED,\n              'n_jobs': -1\n              }\n\nlgb_f_params = {'objective': 'multiclass',\n                'boosting_type': 'gbdt',\n                'n_estimators': 50000,\n                'learning_rate': 0.1,\n                'num_leaves': 90,\n                'colsample_bytree': 0.4,\n                'subsample': 0.6,\n                'subsample_freq': 2,\n                'bagging_seed': SEED,\n                'reg_alpha': 10,\n                'reg_lambda': 2,\n                'random_state': SEED,\n                'n_jobs': -1\n                }\n\n# ------------------------------------------------------------------------------\n# Training and inference\n# ------------------------------------------------------------------------------\nscore_df = pd.DataFrame()\noof = list()\npredictions = list()\nfor n_files, file in enumerate(train_files):\n    data = pd.read_csv(file, index_col=0)\n    test_data = pd.read_csv(test_files[n_files], index_col=0)\n\n    oof_x, oof_y, oof_f = np.zeros(data.shape[0]), np.zeros(data.shape[0]), np.zeros(data.shape[0])\n    preds_x, preds_y = 0, 0\n    preds_f_arr = np.zeros((test_data.shape[0], N_SPLITS))\n\n    kf = KFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED)\n    for fold, (trn_idx, val_idx) in enumerate(kf.split(data.iloc[:, :-4])):\n        X_train = data.iloc[trn_idx, :-4]\n        y_trainx = data.iloc[trn_idx, -4]\n        y_trainy = data.iloc[trn_idx, -3]\n        y_trainf = data.iloc[trn_idx, -2]\n\n        X_valid = data.iloc[val_idx, :-4]\n        y_validx = data.iloc[val_idx, -4]\n        y_validy = data.iloc[val_idx, -3]\n        y_validf = data.iloc[val_idx, -2]\n\n        modelx = lgb.LGBMRegressor(**lgb_params)\n        with timer(\"fit X\"):\n            modelx.fit(X_train, y_trainx,\n                       eval_set=[(X_valid, y_validx)],\n                       eval_metric='rmse',\n                       verbose=False,\n                       early_stopping_rounds=20\n                       )\n\n        modely = lgb.LGBMRegressor(**lgb_params)\n        with timer(\"fit Y\"):\n            modely.fit(X_train, y_trainy,\n                       eval_set=[(X_valid, y_validy)],\n                       eval_metric='rmse',\n                       verbose=False,\n                       early_stopping_rounds=20\n                       )\n\n        modelf = lgb.LGBMClassifier(**lgb_f_params)\n        with timer(\"fit F\"):\n            modelf.fit(X_train, y_trainf,\n                       eval_set=[(X_valid, y_validf)],\n                       eval_metric='multi_logloss',\n                       verbose=False,\n                       early_stopping_rounds=20\n                       )\n\n        oof_x[val_idx] = modelx.predict(X_valid)\n        oof_y[val_idx] = modely.predict(X_valid)\n        oof_f[val_idx] = modelf.predict(X_valid).astype(int)\n\n        preds_x += modelx.predict(test_data.iloc[:, :-1]) / N_SPLITS\n        preds_y += modely.predict(test_data.iloc[:, :-1]) / N_SPLITS\n        preds_f_arr[:, fold] = modelf.predict(test_data.iloc[:, :-1]).astype(int)\n\n        score = comp_metric(oof_x[val_idx], oof_y[val_idx], oof_f[val_idx],\n                            y_validx.to_numpy(), y_validy.to_numpy(), y_validf.to_numpy())\n        print(f\"fold {fold}: mean position error {score}\")\n        score_df = score_log(score_df, n_files, os.path.basename(file), data.shape, fold, SEED, score)\n\n    print(\"*+\"*40)\n    print(f\"file #{n_files}, shape={data.shape}, name={os.path.basename(file)}\")\n    score = comp_metric(oof_x, oof_y, oof_f,\n                        data.iloc[:, -4].to_numpy(), data.iloc[:, -3].to_numpy(), data.iloc[:, -2].to_numpy())\n    oof.append(score)\n    print(f\"mean position error {score}\")\n    print(\"*+\"*40)\n    score_df = score_log(score_df, n_files, os.path.basename(file), data.shape, 999, SEED, score)\n\n    preds_f_mode = stats.mode(preds_f_arr, axis=1)\n    preds_f = preds_f_mode[0].astype(int).reshape(-1)\n    test_preds = pd.DataFrame(np.stack((preds_f, preds_x, preds_y))).T\n    test_preds.columns = subm.columns\n    test_preds.index = test_data[\"site_path_timestamp\"]\n    test_preds[\"floor\"] = test_preds[\"floor\"].astype(int)\n    predictions.append(test_preds)\n\n# ------------------------------------------------------------------------------\n# Submit the result\n# ------------------------------------------------------------------------------\nall_preds = pd.concat(predictions)\nall_preds = all_preds.reindex(subm.index)\nall_preds.to_csv('submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.display import FileLink\nFileLink('submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}