{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys\nsys.path.append('../input/iln-dataset/ILN_TOOLS')\n\nimport gc, glob, time, pickle, math\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\n\nfrom visualize_f import visualize_trajectory\nimport compute_f as F\nfrom scipy.interpolate import interp1d","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_header = 'ILN_631dat'\ninput_dir = '../input/indoor-location-navigation'\nfloor_map = {\"B2\":-2, \"B1\":-1,\n             \"F1\":0, \"F2\":1, \"F3\":2, \"F4\":3, \"F5\":4,\n             \"F6\":5, \"F7\":6, \"F8\":7, \"F9\":8,\n             \"1F\":0, \"2F\":1, \"3F\":2, \"4F\":3, \"5F\":4,\n             \"6F\":5, \"7F\":6, \"8F\":7, \"9F\":8}\nsite_list = ['5d2709a003f801723c3251bf','5a0546857ecc773753327266',\n             '5c3c44b80379370013e0fd2b','5d2709b303f801723c327472',\n             '5d2709bb03f801723c32852c','5d2709c303f801723c3299ee',\n             '5d2709d403f801723c32bd39','5d2709e003f801723c32d896',\n             '5d27075f03f801723c2e360f','5d27096c03f801723c31e5e0',\n             '5d27097f03f801723c320d97','5d27099f03f801723c32511d',\n             '5da138b74db8ce0c98bd4774','5da958dd46f8266d0737457b',\n             '5da1382d4db8ce0c98bbe92e','5da1383b4db8ce0c98bc11ab',\n             '5da1389e4db8ce0c98bd0547','5da138274db8ce0c98bbd3d2',\n             '5da138314db8ce0c98bbf3a0','5da138364db8ce0c98bc00f1',\n             '5da138754db8ce0c98bca82f','5da138764db8ce0c98bcaa46',\n             '5dbc1d84c1eb61796cf7c010','5dc8cea7659e181adb076a3f']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nModify the host's code \"read_data_file\" function in \"io_f.py\"\nfor dealing with the malformed data etc.\n'''\n\nfrom dataclasses import dataclass\n\n@dataclass\nclass ReadData:\n    acce: np.ndarray\n    acce_uncali: np.ndarray\n    gyro: np.ndarray\n    gyro_uncali: np.ndarray\n    magn: np.ndarray\n    magn_uncali: np.ndarray\n    ahrs: np.ndarray\n    wifi: np.ndarray\n    ibeacon: np.ndarray\n    waypoint: np.ndarray\n\ndef split_list_as_req(line_data):\n    redo = False\n    data_BU = []\n    header_list = [i for i, itm in enumerate(line_data) if 'TYPE_' in itm]\n    if len(header_list) > 1:\n        data_BU = [line_data[header_list[1]-1][-13:]] + line_data[header_list[1]:]\n        line_data[header_list[1]-1] = line_data[header_list[1]-1][:-13]\n        line_data = line_data[:header_list[1]]\n        redo = True\n    return redo, line_data, data_BU\n\ndef read_data_file(data_filename):\n    acce = []\n    acce_uncali = []\n    gyro = []\n    gyro_uncali = []\n    magn = []\n    magn_uncali = []\n    ahrs = []\n    wifi = []\n    ibeacon = []\n    waypoint = []\n\n    with open(data_filename, 'r', encoding='utf-8') as file:\n        lines = file.readlines()\n\n    i = 0\n    redo = False\n    while i < len(lines):\n        if not redo:\n            line_data = lines[i]\n            line_data = line_data.strip()\n            if not line_data or line_data[0] == '#':\n                i += 1\n                continue\n            line_data = line_data.split('\\t')\n        else:\n            line_data = data_BU\n            redo = False\n\n        redo, line_data, data_BU = split_list_as_req(line_data)\n    \n        if line_data[1] == 'TYPE_ACCELEROMETER':\n            try:\n                acce.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n            except ValueError:\n                print(data_filename)\n                print(line_data)\n\n        elif line_data[1] == 'TYPE_ACCELEROMETER_UNCALIBRATED':\n            acce_uncali.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n\n        elif line_data[1] == 'TYPE_GYROSCOPE':\n            gyro.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n\n        elif line_data[1] == 'TYPE_GYROSCOPE_UNCALIBRATED':\n            gyro_uncali.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n\n        elif line_data[1] == 'TYPE_MAGNETIC_FIELD':\n            magn.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n\n        elif line_data[1] == 'TYPE_MAGNETIC_FIELD_UNCALIBRATED':\n            magn_uncali.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n\n        elif line_data[1] == 'TYPE_ROTATION_VECTOR':\n            ahrs.append([int(line_data[0]), float(line_data[2]), float(line_data[3]), float(line_data[4])])\n\n        elif line_data[1] == 'TYPE_WIFI':\n            sys_ts = line_data[0]\n            ssid = line_data[2]\n            bssid = line_data[3]\n            rssi = line_data[4]\n            lastseen_ts = line_data[6]\n            frequency = line_data[5]\n            wifi_data = [sys_ts, ssid, bssid, rssi, lastseen_ts, frequency]\n            wifi.append(wifi_data)\n\n        elif line_data[1] == 'TYPE_BEACON':\n            ts = line_data[0]\n            uuid = line_data[2]\n            major = line_data[3]\n            minor = line_data[4]\n            rssi = line_data[6]\n            txpow = line_data[5]\n            distance = line_data[7]\n            mac = line_data[8]\n            if len(line_data)>9:\n                ts_copy = line_data[9]\n            else:\n                ts_copy = ts\n            ibeacon_data = [ts, '_'.join([uuid, major, minor]), rssi,\n                            txpow, distance, mac, ts_copy]\n            ibeacon.append(ibeacon_data)\n\n        elif line_data[1] == 'TYPE_WAYPOINT':\n            waypoint.append([int(line_data[0]), float(line_data[2]), float(line_data[3])])\n\n        if not redo:\n            i += 1\n            \n    acce = np.array(acce)\n    acce_uncali = np.array(acce_uncali)\n    gyro = np.array(gyro)\n    gyro_uncali = np.array(gyro_uncali)\n    magn = np.array(magn)\n    magn_uncali = np.array(magn_uncali)\n    ahrs = np.array(ahrs)\n    wifi = np.array(wifi)\n    ibeacon = np.array(ibeacon)\n    waypoint = np.array(waypoint)\n\n    return ReadData(acce, acce_uncali, gyro, gyro_uncali, magn, magn_uncali, ahrs, wifi, ibeacon, waypoint)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\"test_site_dict.pkl\" in the below cell is the dictionary consists of wifi bssid and the ibeacon MAC address in the test data, and created with [this notebook](https://www.kaggle.com/horsek/ilnpre1-create-testsitedict).  \n\n\"df_TestTimeLag\" in the below cell is a time lag table for each path in the test data, and created with [this notebook](https://www.kaggle.com/horsek/iln-preprocess-time-lag-table-of-test-data).","metadata":{}},{"cell_type":"code","source":"with open(\"../input/iln-preprocess-id-dict-of-the-test-dataset/test_site_dict.pkl\", \"rb\") as f:\n    test_site_dict = pickle.load(f)\ndf_TestTimeLag = pd.read_csv('../input/iln-preprocess-time-lag-table-of-test-data/test_ts_lag.csv',index_col=0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nCreate sequence of time & relative position data for each path\nOutputs are as follows:\n  PathSeq_t: timestamp\n  PathSeq_l: length position (cumulative summation (cumsum) of walking distance)\n             at each timestamp\n  PathSeq_rx, PathSeq_ry: relative position\n'''\ndef Path_Sequence(PathData):\n    step_timestamps, step_indexs, step_acce_max_mins = \\\n        F.compute_steps(PathData.acce)\n    headings = F.compute_headings(PathData.ahrs)\n    stride_lengths = F.compute_stride_length(step_acce_max_mins)\n    step_headings = F.compute_step_heading(step_timestamps, headings)\n    PathSeq_t = stride_lengths[:,0]\n    PathSeq_t = np.insert(PathSeq_t, 0, \n                          PathData.acce[0,0]).astype('int64')\n    PathSeq_l = stride_lengths[:,1].cumsum()\n    PathSeq_l = np.append(np.array(0),PathSeq_l)\n\n    rel_pos = F.compute_rel_positions(stride_lengths, step_headings)\n    PathSeq_rx = np.append(np.array(0),rel_pos[:,1]).cumsum()\n    PathSeq_ry = np.append(np.array(0),rel_pos[:,2]).cumsum()\n    return PathSeq_t, PathSeq_l, PathSeq_rx, PathSeq_ry","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nCreate pd.DataFrame of wifi signal data for each path\nBasic format is the same as that created with Kouki's great notebook.\n(https://www.kaggle.com/kokitanisaka/create-unified-wifi-features-example)\n- rows: timestamp (or corresponding Length position)\n- columns: signal id and strength, which are sorted by strength at the same timestamp\n'''\ndef dfSignalSequence(df_sign, ref_time, id_list,\n                     id_, values, tslabel, n_in_seq):\n    \n    ''' Drop the data out of \"tslabel\" range '''\n    mask = df_sign[id_].apply(lambda x: x in id_list)\n    mask &= (df_sign[tslabel] >= ref_time[0])\n    mask &= (df_sign[tslabel] <= ref_time[-1])\n    df_sign = df_sign[mask]\n    if df_sign.shape[0]==0:\n        return\n    \n    ''' Drop the data before the previous timestamp '''\n    df_tmp = pd.DataFrame(df_sign[tslabel].unique(),columns=[tslabel])\n    df_tmp['last_ts'] = np.append(ref_time[0],\n                                  df_tmp[tslabel].values[:-1])\n    df_sign = df_sign.merge(df_tmp, how='left', on=tslabel)\n    del df_tmp; gc.collect()\n    df_sign = df_sign[df_sign['lastseen_ts']>=df_sign['last_ts']]\n    df_sign.drop('last_ts', axis=1, inplace=True)\n    if df_sign.shape[0]==0:\n        return\n    \n    ''' create pivot table '''\n    df_sign.set_index(tslabel, inplace=True)\n    df_pivot = df_sign.pivot_table(index=df_sign.index,\n                                   columns=id_, values=values,\n                                   aggfunc=max)\n    feat = []\n    for i in range(df_pivot.shape[0]):\n        tmp = df_pivot.iloc[i,:].sort_values(ascending=False)[:n_in_seq]\n        if tmp.shape[0]!=n_in_seq:\n            n_col = tmp.shape[0]\n            add_col = list(set(id_list)-set(tmp.index))[:n_in_seq-n_col]\n            tmp = pd.concat([tmp, pd.Series(np.nan, index=add_col)])\n            assert tmp.shape[0]==n_in_seq\n        tmp = tmp.reset_index().values.T.reshape(-1)\n        feat.append(tmp)\n    feat = np.stack(feat,axis=1).T\n    df_out = pd.DataFrame(feat,\n                          columns=[f'id_{i}' for i in range(n_in_seq)]\n                                 +[f'strength_{i}' for i in range(n_in_seq)])\n    df_out[tslabel] = df_pivot.index\n    df_out.set_index(tslabel, drop=True, inplace=True)\n    return df_out","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"''' Retrieve the Site ID from txt file '''\ndef SiteID(txt):\n    p1 = txt[1].find('SiteID:')+7\n    p2 = txt[1].find('\\tSiteName:')\n    assert p1!=-1+7 and p2!=-1, 'SiteID not found'\n    return txt[1][p1:p2]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def headings_from_magn(PathData):\n    mag_df = pd.DataFrame(PathData.magn)\n    mag_df.columns = [\"timestamp\",\"x\",\"y\",\"z\"]\n    acce_df = pd.DataFrame(PathData.acce)\n    acce_df.columns = [\"timestamp\",\"ax\",\"ay\",\"az\"]\n    mag_df = pd.merge(mag_df,acce_df,on=\"timestamp\").dropna()\n\n    m_trans = 0\n    time_di_list = []\n    for i in mag_df.iterrows():\n\n        ''' https://www.kaggle.com/museas/with-magn-cost-minimization '''\n        gx,gy,gz = i[1][1],i[1][2],i[1][3]\n        ax,ay,az = i[1][4],i[1][5],i[1][6]\n        roll = math.atan2(ay,az)\n        pitch = math.atan2(-1*ax , (ay * math.sin(roll) + az * math.cos(roll)))\n        q = m_trans - math.atan2(\n            (gz*math.sin(roll)-gy*math.cos(roll)),(gx*math.cos(pitch) + gy*math.sin(roll)*math.sin(pitch) + gz*math.sin(pitch)*math.cos(roll))\n        ) -np.pi/2\n        q = (q+np.pi)%(2*np.pi)-np.pi\n\n        ''' The following is a different calculation for verification '''    \n#         mag, acce = i[1][1:4].values, i[1][4:7].values\n#         axsz = acce/np.linalg.norm(acce)\n#         axsy = mag/np.linalg.norm(mag)\n#         axsy -= np.sum(axsy*axsz) * axsz\n#         axsy = axsy/np.linalg.norm(axsy)\n#         axsx = np.cross(axsy, axsz)\n#         q1 = np.arctan2(axsx[1],axsy[1])\n#         q1 = (q1+np.pi)%(2*np.pi)-np.pi\n\n        time_di_list.append((i[1][0],q))\n    return np.array(time_di_list)\n\ndef add_IMU(PathData, df_inpt, time_delta=0):\n\n    headmagn = headings_from_magn(PathData)\n    df_IMU = pd.DataFrame(PathData.gyro,columns=['IMU_ts','gyro_x','gyro_y','gyro_z'])\n    df_IMU['IMU_ts'] = (df_IMU['IMU_ts']+time_delta).astype('int64')\n#     assert (df_IMU['IMU_ts'].values==headmagn[:,0]).all()\n#     assert (df_IMU['IMU_ts'].values==PathData.acce[:,0]).all()\n#     assert (df_IMU['IMU_ts'].values==PathData.ahrs[:,0]).all()\n    df_IMU['head_magn_x'] = np.sin(headmagn[:,1])\n    df_IMU['head_magn_y'] = np.cos(headmagn[:,1])\n    df_IMU[['acce_x','acce_y','acce_z']] = PathData.acce[:,1:]\n    df_IMU[['ahrs_x','ahrs_y','ahrs_z']] = PathData.ahrs[:,1:]\n    df_IMU[['magn_x','magn_y','magn_z']] = PathData.magn[:,1:]\n\n    ''' set window label '''\n    grp_ts = df_inpt.index.name\n    df_IMU = pd.concat([pd.DataFrame([df_inpt.index]*2,index=['IMU_ts',grp_ts]).T,\n                        df_IMU], axis=0)\n    df_IMU = df_IMU.sort_values('IMU_ts')\n    df_IMU[grp_ts].fillna(method='bfill',inplace=True)\n    df_IMU.dropna(inplace=True)\n    df_IMU[grp_ts] = df_IMU[grp_ts].astype('int64')\n    df_IMU.drop('IMU_ts', axis=1, inplace=True)\n\n    ''' grouping by timestamp of wifi data '''\n    gdf_IMU = pd.concat([df_IMU.groupby(grp_ts).mean().add_suffix('_mean'),\n                         df_IMU.groupby(grp_ts).std().add_suffix('_std'),\n                         df_IMU.groupby(grp_ts).max().add_suffix('_max'),\n                         df_IMU.groupby(grp_ts).min().add_suffix('_min'),\n                         df_IMU.groupby(grp_ts).skew().add_suffix('_skew')],axis=1)\n\n    return df_inpt.merge(gdf_IMU, how='left', left_index=True, right_index=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp_list = []","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"''' train data processing '''\nfor i_site, site in enumerate(site_list):\n\n    print(f'========== {i_site+1}/{len(site_list)} ==========')\n    print(f'site: {site}')\n    test_wifi = sorted(test_site_dict[site]['wifi'])\n    print(f'num of wifi BSSID in test set: {len(test_wifi)}')\n\n    AllPathData = []\n    floor_dirs = sorted(glob.glob(f'{input_dir}/train/{site}/*'))\n    floors = [dir_.split('/')[-1] for dir_ in floor_dirs]\n\n    print(f'floors: {floors}')\n\n    print('\\n=== Read Path Data ===')\n    time.sleep(1)\n    for i, floor_dir_ in enumerate(floor_dirs):\n        path_files = sorted(glob.glob(f'{floor_dir_}/*'))\n        dic_temp = {}\n        for path_file_ in tqdm(path_files, desc=floors[i]):\n            path_name = path_file_.split('/')[-1].replace('.txt','')\n            dic_temp[path_name] = read_data_file(path_file_)\n        AllPathData.append(dic_temp)\n    del dic_temp; gc.collect()\n\n    print('\\n=== Create features ===')\n    time.sleep(1)\n    for i_floor, FloorPathData in enumerate(AllPathData):\n        for PathName, PathData in tqdm(FloorPathData.items(),\n                                       desc=floors[i_floor]):\n            ''' Path Sequence '''\n            PathSeq_t, PathSeq_l, PathSeq_rx, PathSeq_ry = Path_Sequence(PathData)\n    \n            ''' wifi DataFrame '''\n            if PathData.wifi.shape[0]==0:\n                continue\n            df_wifi = pd.DataFrame(PathData.wifi,\n                                   columns=['sys_ts','ssid','bssid','rssi',\n                                            'lastseen_ts','frequency'])\n            df_wifi['sys_ts']=df_wifi['sys_ts'].astype(np.int64)\n            df_wifi['lastseen_ts']=df_wifi['lastseen_ts'].astype(np.int64)\n            df_wifi['rssi']=df_wifi['rssi'].astype(np.int64)\n            \n            ''' Signal Sequence '''\n            df_feat = dfSignalSequence(df_wifi, PathSeq_t, test_wifi,\n                                       'bssid','rssi','sys_ts',100)\n            if df_feat is None:\n                continue\n\n            ''' Compute length position & relative position by interpolation '''\n            itp_l = interp1d(PathSeq_t, PathSeq_l,\n                             fill_value=(PathSeq_l[0],PathSeq_l[-1]),\n                             bounds_error=False)\n            df_feat['len_pos'] = itp_l(df_feat.index)\n            itp_rx = interp1d(PathSeq_t, PathSeq_rx,\n                              fill_value=(PathSeq_rx[0],PathSeq_rx[-1]),\n                              bounds_error=False)\n            itp_ry = interp1d(PathSeq_t, PathSeq_ry,\n                              fill_value=(PathSeq_ry[0],PathSeq_ry[-1]),\n                              bounds_error=False)\n            df_feat['rel_x'] = itp_rx(df_feat.index)\n            df_feat['rel_y'] = itp_ry(df_feat.index)\n            \n            ''' IMU data '''\n            df_feat = add_IMU(PathData, df_feat)\n                \n            ''' x, y, floor, labels '''\n#             idx = np.abs(PathData.waypoint[:,0:1]-\n#                          df_feat.index.values.reshape(1,-1)).argmin(axis=0)\n#             df_feat[['x','y']]=PathData.waypoint[idx,1:]\n\n            GT_t, GT_x, GT_y = [PathData.waypoint[:,i] for i in range(3)]\n            GT_l = itp_l(GT_t)\n            itp_x = interp1d(GT_l, GT_x, fill_value=(GT_x[0],GT_x[-1]),\n                             bounds_error=False)\n            itp_y = interp1d(GT_l, GT_y, fill_value=(GT_y[0],GT_y[-1]),\n                             bounds_error=False)\n            df_feat['x'] = itp_x(df_feat['len_pos'])\n            df_feat['y'] = itp_y(df_feat['len_pos'])\n            df_feat['floor'] = floor_map[floors[i_floor]]\n            df_feat['path'] = PathName\n            df_feat['site'] = site\n            df_feat['train/test'] = 'train'\n            \n            tmp_list.append(df_feat)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"''' test data processing '''\nprint('\\n=== Read Path Data ===')\ntest_files = sorted(glob.glob(f'{input_dir}/test/*.txt'))\nTestPathData,TestSiteName = {},{}\ntime.sleep(1)\nfor path_file_ in tqdm(test_files):\n    path_name = path_file_.split('/')[-1].replace('.txt','')\n    TestPathData[path_name] = read_data_file(path_file_)\n    \n    with open(path_file_, 'r', encoding=\"utf-8\") as f:\n        txt = f.readlines()\n    TestSiteName[path_name] = SiteID(txt)\n\nprint('\\n=== Create features ===')\ntime.sleep(1)\nfor PathName, PathData in tqdm(TestPathData.items()):\n    site = TestSiteName[PathName]\n    test_wifi = sorted(test_site_dict[site]['wifi'])\n    time_lag = df_TestTimeLag.loc[PathName,'time_lag']\n    \n    ''' Path Sequence '''\n    PathSeq_t, PathSeq_l, PathSeq_rx, PathSeq_ry = Path_Sequence(PathData)\n    PathSeq_t += time_lag\n    \n    ''' wifi DataFrame '''\n    if PathData.wifi.shape[0]==0:\n        continue\n    df_wifi = pd.DataFrame(PathData.wifi,\n                           columns=['sys_ts','ssid','bssid','rssi',\n                                    'lastseen_ts','frequency'])\n    df_wifi['sys_ts']=df_wifi['sys_ts'].astype(np.int64)\n    df_wifi['sys_ts'] += time_lag\n    df_wifi['lastseen_ts']=df_wifi['lastseen_ts'].astype(np.int64)\n    df_wifi['rssi']=df_wifi['rssi'].astype(np.int64)\n\n    ''' Signal Sequence '''\n    df_feat = dfSignalSequence(df_wifi, PathSeq_t, test_wifi,\n                               'bssid','rssi','sys_ts',100)\n    if df_feat is None:\n        continue\n\n    ''' Compute length position & relative position by interpolation '''\n    itp_l = interp1d(PathSeq_t, PathSeq_l,\n                     fill_value=(PathSeq_l[0],PathSeq_l[-1]),\n                     bounds_error=False)\n    df_feat['len_pos'] = itp_l(df_feat.index)\n    itp_rx = interp1d(PathSeq_t, PathSeq_rx,\n                      fill_value=(PathSeq_rx[0],PathSeq_rx[-1]),\n                      bounds_error=False)\n    itp_ry = interp1d(PathSeq_t, PathSeq_ry,\n                      fill_value=(PathSeq_ry[0],PathSeq_ry[-1]),\n                      bounds_error=False)\n    df_feat['rel_x'] = itp_rx(df_feat.index)\n    df_feat['rel_y'] = itp_ry(df_feat.index)\n    \n    ''' IMU data '''\n    df_feat = add_IMU(PathData, df_feat, time_lag)\n                \n    ''' x, y, floor, labels '''\n    df_feat[['x','y']] = np.nan\n    df_feat['floor'] = np.nan\n    df_feat['path'] = PathName\n    df_feat['site'] = site\n    df_feat['train/test'] = 'test'\n\n    tmp_list.append(df_feat)    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_wifi_all = pd.concat(tmp_list)\ndf_wifi_all.reset_index(inplace=True)\n\n#null check\ndf_wifi_all[[f'id_{i}' for i in range(100)]].isnull().any().any()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(f'{file_header}_df_wifi_all.pkl','wb') as f:\n    pickle.dump(df_wifi_all, f)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}