{"cells":[{"metadata":{},"cell_type":"markdown","source":"### Update\n\nNow it's fast thanks to [@higepon](https://www.kaggle.com/higepon) and [@romainfabre](https://www.kaggle.com/romainfabre) !\n\n\n### Description\nThis is an example of making one dataframe to feed into one model.\nThe result is [here](https://www.kaggle.com/kokitanisaka/indoorunifiedwifids).\n\nThis code is super slow, if you know how to make it faster, please let me know.\n\nThe created dataset is here.<br>\nhttps://www.kaggle.com/kokitanisaka/indoorunifiedwifids\n\nThe oridinal dataset is here.<br>\nhttps://www.kaggle.com/hiro5299834/indoor-navigation-and-location-wifi-features"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport glob\nimport os\n\nimport multiprocessing\nfrom multiprocessing import Pool\n\nnum_cores = multiprocessing.cpu_count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ITEMS_TO_TAKE = 100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base_path = '/kaggle'\nfeature_dir = f\"{base_path}/input/indoor-navigation-and-location-wifi-features\"\ntrain_files = sorted(glob.glob(os.path.join(feature_dir, '*_train.csv')))\ntest_files = sorted(glob.glob(os.path.join(feature_dir, '*_test.csv')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def generate_training_data(train_file):\n    train = pd.read_csv(train_file)\n\n    file_name = train_file.split('/')[-1]\n\n    num_of_lines = train.shape[0]\n    print(f'{file_name} : {num_of_lines}')\n\n    data = []\n    for i in range(num_of_lines):\n        if i % 100 == 0:\n            print(f'current:{i}')\n\n        tmp = train.iloc[i,1:-4].astype(int).sort_values(ascending=False).head(ITEMS_TO_TAKE)\n        target = train.iloc[i,-4:]\n        line = [*tmp.index.astype(str), *tmp.values, *target]\n        data.append(line)\n    data = pd.DataFrame(data)\n    data.columns = [f'bssid_{str(i)}' for i in range(ITEMS_TO_TAKE)] + [f'rssi_{str(i)}' for i in range(ITEMS_TO_TAKE)] + ['x','y','floor','path']\n    data.to_csv(f'{file_name}', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"with Pool(num_cores) as pool:\n    pool.map(generate_training_data, [t for t in train_files])  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def generate_test_data(test_file):\n    train = pd.read_csv(test_file)\n\n    file_name = test_file.split('/')[-1]\n\n    num_of_lines = train.shape[0]\n    print(f'{file_name} : {num_of_lines}')\n\n    data = []\n    for i in range(num_of_lines):\n        if i % 100 == 0:\n            print(f'current:{i}')\n\n        tmp = train.iloc[i,1:-4].astype(int).sort_values(ascending=False).head(ITEMS_TO_TAKE)\n        target = train.iloc[i, [-1]]\n\n        line = [*tmp.index.astype(str), *tmp.values, *target]\n        data.append(line)\n    data = pd.DataFrame(data)\n    data.columns = [f'bssid_{str(i)}' for i in range(ITEMS_TO_TAKE)] + [f'rssi_{str(i)}' for i in range(ITEMS_TO_TAKE)] + ['site_path_timestamp']\n    data.to_csv(f'{file_name}', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"with Pool(num_cores) as pool:\n    pool.map(generate_test_data, [t for t in test_files])  ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}