{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"In this notebook, I use single model trained separately for each of 24 test sites first for floors then for coordinates. My model uses fastai.\nThere is almost no postprocessing except for floor prediction. I corrected floor predictions to make all points being on the same floor (majority of predictions).\nAll this code (5 folds CV) runs more than 12 hours and I run it locally. So after checking everything works I upload my submission file and will submit it instead of waiting of the whole run here. Also on Kaggle I cannot run on GPU (not enough memory), so use only cpu here.\n\nAbout the model per se.\nIt uses RSSI from BSSIDs met more than X times in given site. Threshold is defined for each building. To define coordinates of records with wifi-data we use calibrate_magnetic_wifi_ibeacon_to_position provided by host, fix for new lines used too. But as we add dataset with this information (ie txt files were already parsed) we don't attach fixes for hosts files.\n\nFurther details like augmenations I'll clarify below.\n\nI think the score for the single model without ensembles and postprocessing is apropriate so may be it will be useful for someone.\n\nThank you","metadata":{}},{"cell_type":"code","source":"import os.path as op\nimport sys\nimport glob\nimport pandas as pd\nimport numpy as np\nimport json\nfrom IPython.core.debugger import Tracer\nimport os\nimport matplotlib.pyplot as plt\nimport time\nimport re\nfrom tqdm.notebook import tqdm\nfrom sklearn.preprocessing import OneHotEncoder\n\nfrom fastai.tabular.all import *\nimport random\nimport datetime\nfrom sklearn.model_selection import StratifiedKFold","metadata":{"_uuid":"a961b7f6-fbfc-4ede-b35e-b97e4b405837","_cell_guid":"65546a69-abff-484e-927a-fe275ded9cab","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-05-21T18:43:10.637050Z","iopub.execute_input":"2021-05-21T18:43:10.637485Z","iopub.status.idle":"2021-05-21T18:43:13.203515Z","shell.execute_reply.started":"2021-05-21T18:43:10.637369Z","shell.execute_reply":"2021-05-21T18:43:13.202515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CSV_PATH = \"../input/indoor-generated\" #this is for csv files which I generated before, If you wish regenerate - handle this to see test and train txt\nTXT_PATH = \"../input/indoor-location-navigation\"\nOUTPUT = \"\"\n#TEST_PATH = op.join(TXT_PATH, \"test\") \n#TRAIN_PATH = op.join(TXT_PATH, \"train\")\n#META_PATH = op.join(PATH, \"metadata\")\n#SCRIPTS_PATH = op.join(PATH, \"scripts\", \"indoor-location-competition-20\") #see comments above\nSS_PATH = op.join(TXT_PATH, \"sample_submission.csv\")\nTRAIN_CSV_PATH = op.join(CSV_PATH, \"train_csv\", \"train_csv\")\nTEST_CSV_PATH = op.join(CSV_PATH, \"test_csv\", \"test_csv\")\n#TEST_ALL_PATHS_CSV = op.join(TEST_CSV_PATH, \"test_all_paths.csv\")\nMODEL_DIR = op.join(OUTPUT, \"models\")\n#CLOSEST_WAYPOINT_CSV_DIR = op.join(TRAIN_CSV_PATH, \"train_df_closest_waypoint\")\nTODAY = datetime.date.today().strftime(\"%d%m%Y\")\n#os.mkdir(op.join(PATH, TODAY))\n#sys.path.append(SCRIPTS_PATH) \n#from main import calibrate_magnetic_wifi_ibeacon_to_position\nsys.path.append(\"../input/iterative-stratification/iterative-stratification-master\") \nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold\n# from io_f import read_data_file\n# from compute_f import compute_step_positions\n# from compute_f import compute_steps\n# from compute_f import compute_headings\n# from compute_f import compute_stride_length\n# from compute_f import compute_step_heading\n# from compute_f import compute_rel_positions\n# from compute_f import correct_positions\n# from compute_f import correct_trajectory\n!dir $CSV_PATH\n","metadata":{"_uuid":"dc2a3dae-2e97-4834-906e-f9e52f731018","_cell_guid":"8500af8d-9ae4-408e-91c7-c0ab07c7b981","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-05-21T18:43:13.205007Z","iopub.execute_input":"2021-05-21T18:43:13.205475Z","iopub.status.idle":"2021-05-21T18:43:13.988464Z","shell.execute_reply.started":"2021-05-21T18:43:13.205396Z","shell.execute_reply":"2021-05-21T18:43:13.987635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    \nseed_everything(seed=42)","metadata":{"_uuid":"17bd3aa5-bdd0-4a88-b3a3-9d35b319ac4d","_cell_guid":"839fb1ad-4f75-40ee-9b90-022af0da2494","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-05-21T18:43:13.990663Z","iopub.execute_input":"2021-05-21T18:43:13.991256Z","iopub.status.idle":"2021-05-21T18:43:14.001534Z","shell.execute_reply.started":"2021-05-21T18:43:13.991203Z","shell.execute_reply":"2021-05-21T18:43:14.000275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss_df = pd.read_csv(SS_PATH)\nss_df_tmp = ss_df['site_path_timestamp'].apply(lambda x: pd.Series(x.split(\"_\")))\nss_df_tmp = ss_df_tmp.rename(columns={0: \"site\", 1: \"path\", 2:\"timestamp\"})\nss_df_tmp['timestamp'] = pd.to_numeric(ss_df_tmp['timestamp'])\ntest_sites = np.unique(ss_df_tmp.site.values)#TODO will train only with sites from test, check later if we adding others sites can help","metadata":{"_uuid":"25b0935d-ba69-40e9-a78b-575e81cff560","_cell_guid":"a71e9dfa-04d2-4519-bc37-fe336a9f5763","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-05-21T18:43:14.003271Z","iopub.execute_input":"2021-05-21T18:43:14.003746Z","iopub.status.idle":"2021-05-21T18:43:16.238280Z","shell.execute_reply.started":"2021-05-21T18:43:14.003711Z","shell.execute_reply":"2021-05-21T18:43:16.237264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Below we generate csv files from txt for training and prediction. But as we added dataset already, will skip this. I got no results with IBeacon so I don't used it.","metadata":{}},{"cell_type":"code","source":"def get_df_from_file(file):\n    data = calibrate_magnetic_wifi_ibeacon_to_position([file])\n    \n    wifi_dfs = []\n    ibeacon_dfs = []\n     \n    for key in data:\n        x,y = key       \n               \n        if ('wifi' in data[key] and len(data[key]['wifi']) > 0):\n            wifi_df = pd.DataFrame(columns=['path','floor','timestamp','x','y','bssid','rssi', 'lastseen'])\n            wifi_data = data[key]['wifi']             \n            wifi_df['timestamp'] = wifi_data[:,0].astype(float)\n            wifi_df['bssid'] = wifi_data[:,2]\n            wifi_df['rssi'] = wifi_data[:,3].astype(float)\n            wifi_df['lastseen'] = wifi_data[:,4].astype(float)\n            wifi_df['x'] = x\n            wifi_df['y'] = y   #next time add last seen\n            wifi_dfs.append(wifi_df)\n        elif ('ibeacon' in data[key] and len(data[key]['ibeacon']) > 0):   \n            ibeacon_df = pd.DataFrame(columns=['path', 'floor', 'timestamp', 'uuid', 'major', 'minor', 'txpower',\n                                               'rssi', 'distance', 'mac', 'lastseen'])\n          \n            ibeacon_data = data[key]['ibeacon'] \n            ibeacon_df['timestamp'] = ibeacon_data[:,0].astype(float)\n            ibeacon_df['uuid'] = ibeacon_data[:,1]\n            ibeacon_df['major']= ibeacon_data[:,2]\n            ibeacon_df['minor'] = ibeacon_data[:,3]\n            ibeacon_df['txpower'] = ibeacon_data[:,4].astype(float)\n            ibeacon_df['rssi'] = ibeacon_data[:,5].astype(float)\n            ibeacon_df['distance'] = ibeacon_data[:,6].astype(float)\n            ibeacon_df['mac'] = ibeacon_data[:,7]\n            ibeacon_df['lastseen'] = ibeacon_data[:,8].astype(float)\n            ibeacon_df['x'] = x\n            ibeacon_df['y'] = y\n            ibeacon_dfs.append(ibeacon_df)\n           \n    return (pd.concat(wifi_dfs) if len(wifi_dfs) > 0 else None),\\\n           (pd.concat(ibeacon_dfs) if len(ibeacon_dfs) > 0 else None)\n\ndef generate_pre_csv(generate=False):\n    if not generate:\n        return    \n\n    for i in range(1,len(test_sites)):\n        site = test_sites[i]\n        print(i, site)\n        floor_files = glob.glob(op.join(TRAIN_PATH, site,'*'))  \n\n        floor_wifi_dfs = []\n        floor_ibeacon_dfs = []\n        \n        for floor_file in floor_files:\n            \n            floor = floor_file.split(\"\\\\\")[-1]\n            files = glob.glob(op.join(floor_file, \"*\"))\n\n            for file in files:\n                path = file.split(\"\\\\\")[-1][:-4]\n                #print(file)\n               \n                wifi_df, ibeacon_df = get_df_from_file(file)\n\n                if (wifi_df is not None):\n                    wifi_df['path']=path\n                    wifi_df['floor']=floor                     \n                    floor_wifi_dfs.append(wifi_df)    \n                \n                if (ibeacon_df is not None):\n                    ibeacon_df['path']=path\n                    ibeacon_df['floor']=floor\n                    floor_ibeacon_dfs.append(ibeacon_df)\n            #break\n        site_wifi_df = pd.concat(floor_wifi_dfs)\n        site_wifi_df.to_csv(op.join(TRAIN_CSV_PATH, site + '.wifi.pre.csv'),index=False)  \n        \n        site_ibeacon_df = pd.concat(floor_ibeacon_dfs)\n        site_ibeacon_df.to_csv(op.join(TRAIN_CSV_PATH, site + '.ibeacon.pre.csv'),index=False)  \n        #break\n        \ndef generate_test_pre_csv(generate=False):    \n    if not generate:\n        return    \n\n    for i in range(len(test_sites)):\n        site = test_sites[i]\n        print(i, site)\n        paths = ss_df_tmp[ss_df_tmp['site']==site]['path'].values\n        wifi_paths_df = []\n        ibeacon_paths_df = []\n        \n        for path in paths:\n            file = op.join(TEST_PATH, path + \".txt\")\n            data = read_data_file(file)\n            \n            wifi_path_df = pd.DataFrame(columns=['timestamp', 'ssid', 'bssid', 'rssi', 'lastseen'])            \n            wifi_data = data.wifi\n            for i in range(len(wifi_path_df.columns)):\n                wifi_path_df[wifi_path_df.columns[i]] = wifi_data[:,i]\n            wifi_path_df['path'] = path\n            wifi_paths_df.append(wifi_path_df)\n            \n            ibeacon_path_df = pd.DataFrame(columns=['timestamp', 'uuid', 'major', 'minor', 'txpower',\n                                                    'rssi', 'distance', 'mac', 'lastseen'])            \n            ibeacon_data = data.ibeacon\n            if (len(ibeacon_data) > 0):\n                for i in range(len(ibeacon_path_df.columns)):\n                    ibeacon_path_df[ibeacon_path_df.columns[i]] = ibeacon_data[:,i]\n                ibeacon_path_df['path'] = path\n            ibeacon_paths_df.append(ibeacon_path_df)\n            \n                       \n            \n        wifi_site_df = pd.concat(wifi_paths_df)\n        wifi_site_df.to_csv(op.join(TEST_CSV_PATH, site + '.wifi.pre.csv'),index=False)  \n        \n        ibeacon_site_df = pd.concat(ibeacon_paths_df)\n        ibeacon_site_df.to_csv(op.join(TEST_CSV_PATH, site + '.ibeacon.pre.csv'),index=False)  \n       \nprint(\"generating train pre csv\")       \ngenerate_pre_csv(generate=False)\nprint(\"generating test pre csv\")       \ngenerate_test_pre_csv(False)","metadata":{"_uuid":"19386383-c184-4a4e-8ec1-31ad40baaea7","_cell_guid":"b19ad1d1-97a2-476b-bad9-8657890de279","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-05-21T18:43:16.239505Z","iopub.execute_input":"2021-05-21T18:43:16.239744Z","iopub.status.idle":"2021-05-21T18:43:16.261528Z","shell.execute_reply.started":"2021-05-21T18:43:16.239721Z","shell.execute_reply":"2021-05-21T18:43:16.260312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#So for now see no good from choosing the closest wifi records\n\ndef generate_wifi_files(generate = False):\n    if (not generate):\n        return\n    for i in range(14,len(test_sites)):\n        site = test_sites[i]\n        print(i, site)\n        df = pd.read_csv(op.join(TRAIN_CSV_PATH, site + \".wifi.pre.csv\"))#TODO store with index=False      \n        \n        bssids = list(np.unique(df['bssid'].values))\n        train_df = pd.DataFrame(columns=['timestamp','x','y','floor','path'] + bssids)\n        for (i,(a,b)) in tqdm(enumerate(df.groupby(['timestamp','x','y']))):\n            #v = b['time_diff']\n            \n            timestamp, x, y = a\n            bssid_values = b['bssid'].values    \n            rssi_values = b['rssi'].values            \n            row = {'timestamp':timestamp, 'x':x, 'y':y}\n            row.update({k:v for (k,v) in zip(bssid_values,rssi_values)})    \n            train_df = train_df.append(row, ignore_index=True)\n            \n        for (i,(a,b)) in tqdm(enumerate(df[['timestamp', 'floor', 'path']].groupby(['timestamp']))):\n            timestamp  = a\n            floor = b['floor'].values[0]\n            path = b['path'].values[0]\n            \n            train_df.loc[train_df[train_df['timestamp']==a].index,['floor', 'path']] = [floor, path]\n            \n        train_df = train_df.fillna(0)        \n        \n        train_df.to_csv(op.join(TRAIN_CSV_PATH, site + \".wifi.csv\"), index=False) \n\ndef generate_ibeacon_files(generate = False):\n    if (not generate):\n        return        \n        \n    for i in range(len(test_sites)):\n        site = test_sites[i]\n        print(i, site)\n        df = pd.read_csv(op.join(TRAIN_CSV_PATH, site + \".ibeacon.pre.csv\"))#TODO store with index=False      \n        \n        macs = list(np.unique(df['mac'].values))\n        rssi_cols = ['rssi_' + mac for mac in macs]\n        txp_cols = ['txp_' + mac for mac in macs]\n        dist_cols =  ['dist_' + mac for mac in macs]\n        cols = rssi_cols + txp_cols + dist_cols\n        \n        train_df = pd.DataFrame(columns=['timestamp','x','y','floor','path'] + cols)\n        for (i,(a,b)) in tqdm(enumerate(df.groupby(['timestamp','x','y']))):\n                     \n            timestamp, x, y = a\n            mac_values = b['mac'].values\n            rssi_values = b['rssi'].values\n            dist_values = b['distance'].values\n            txpower_values = b['txpower'].values\n            \n                      \n            row = {'timestamp':timestamp, 'x':x, 'y':y}\n            row.update({'rssi_'+ k:v for (k,v) in zip(mac_values,rssi_values)})   \n            row.update({'txpower_'+ k:v for (k,v) in zip(mac_values,txpower_values)})\n            row.update({'dist_'+ k:v for (k,v) in zip(mac_values,dist_values)})\n            \n            train_df = train_df.append(row, ignore_index=True)\n            \n        for (i,(a,b)) in tqdm(enumerate(df[['timestamp', 'floor', 'path']].groupby(['timestamp']))):\n            timestamp  = a\n            floor = b['floor'].values[0]\n            path = b['path'].values[0]\n            \n            train_df.loc[train_df[train_df['timestamp']==a].index,['floor', 'path']] = [floor, path]\n            \n        train_df = train_df.fillna(0)        \n        \n        train_df.to_csv(op.join(TRAIN_CSV_PATH, site + \".ibeacon.csv\"), index=False)        \n\nprint(\"Will generate train csv\")            \ngenerate_wifi_files(False) \nprint(\"Will generate ibeacon csv\")\ngenerate_ibeacon_files(False)","metadata":{"_uuid":"96d0a567-a78f-492c-a7f6-1ce0f0cf2c69","_cell_guid":"60cf393b-8c9c-4a84-8d66-f658e5978621","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-05-21T18:43:16.263331Z","iopub.execute_input":"2021-05-21T18:43:16.263764Z","iopub.status.idle":"2021-05-21T18:43:16.283980Z","shell.execute_reply.started":"2021-05-21T18:43:16.263721Z","shell.execute_reply":"2021-05-21T18:43:16.282669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So next goes my main cell.","metadata":{}},{"cell_type":"code","source":"EPOCHS = 1 # I used 20 epochs !!!!! this is for quick run\nFOLDS = 5\n\n# As RSSI is always negative and zero in my csv means no signal there is a contradiction. Zero is much more than very strong signal. So I move \n# RSSI to be above zero, so 0 means very bad signal. Than I factorize by 2.5 (the best value for me, round and /5 was worse/.)\ndef handle_values(df):\n    needed_cols = [c for c in df.columns if len(c) > 15]\n    values = df[needed_cols].values    \n    values=np.where(values==0, -100, values)\n    values = np.maximum(0, values + 90)\n    df[needed_cols] = np.round(values/2.5)\n    \n    return df\n#these my thresholds for different sites (how often bssid should be met to participate in train)\nthr_dict = {\n    test_sites[0]:200,\n    test_sites[1]:500,\n    test_sites[2]:500,\n    test_sites[3]:250,\n    test_sites[4]:200,\n    test_sites[5]:150,\n    \n    test_sites[6]:200,\n    test_sites[7]:250,\n    test_sites[8]:250,\n    test_sites[9]:200,\n    test_sites[10]:250,\n    test_sites[11]:500,\n    \n    test_sites[12]:250,\n    test_sites[13]:200,\n    test_sites[14]:250,\n    test_sites[15]:200,\n    test_sites[16]:500,\n    test_sites[17]:200,\n    \n    test_sites[18]:250,\n    test_sites[19]:150,\n    test_sites[20]:200,\n    test_sites[21]:500,\n    test_sites[22]:500,\n    test_sites[23]:400##200  \n    \n    \n}\n\n# here we compute which columns should be used (based on threshold), make fold stratification and one-hot encoding\ndef get_train_df(site, pixel_strat=True, do_thr=True, wifi=True):\n    suffix = \".wifi.csv\" if wifi else '.ibeacon.csv'\n    train_df = pd.read_csv(op.join(TRAIN_CSV_PATH, site + suffix))\n    train_df = handle_values(train_df)\n    enc = OneHotEncoder(sparse=False)\n    result = enc.fit_transform(train_df['floor'].values.reshape(-1,1))\n    cat_names = enc.categories_[0]\n    train_df[cat_names] = result    \n    \n    train_df['kfold'] = -1\n    \n    \n    if (pixel_strat):        \n        kf = MultilabelStratifiedKFold(n_splits=FOLDS)#, random_state=12)\n        y = train_df[enc.categories_[0]].values\n        for fold, (t_,v_) in enumerate(kf.split(X=train_df,y=y)):\n            train_df.loc[v_,'kfold'] = fold\n    else:\n        skf = StratifiedKFold(n_splits=5)#, random_state=12)\n        help_df = train_df[['path','floor']]\n        help_df = help_df.drop_duplicates(subset=['path', 'floor'])\n        y_for_split = help_df['floor'].values\n        for fold, (t_,v_) in enumerate(skf.split(help_df, y_for_split)):\n            for p, f in help_df.iloc[v_][['path','floor']].values:\n                index = train_df[(train_df['path']==p) & (train_df['floor']==f)].index\n                train_df.loc[index, 'kfold']= fold\n    \n    columns = [c for c in train_df.columns if len(c) > 20]\n    if (do_thr):\n        thr = thr_dict[site]#min(thr_dict[site],250)#thr_dict[site]\n        \n        print(\"threshold: \", str(thr))\n        cont_names = [c for c in columns if train_df[c].ne(0).sum() > thr]\n    else:\n        cont_names = columns\n    return train_df, list(enc.categories_[0]), cont_names, columns\n\n#just return floor for submission by letter\ndef get_floor(floor_str):\n    num = re.search('\\d+',floor_str)[0]\n    if ('F'  in floor_str):\n        return int(num)-1\n    elif ('B'  in floor_str):\n        return -int(num)\n    else:\n        raise ValueError('A very specific bad thing happened.')  \n        \n#Augmentation is very simple:  sometimes some of wifi  points can be switche off, so we replace true RSSI by zero.    \nclass AugCallback(Callback):\n    #run_after,run_valid = [Normalize],False\n    #run_before = [Normalize]\n    #def __init__(self, alpha=0.4): self.distrib = Beta(tensor(alpha), tensor(alpha))\n    \n    def before_batch(self, **kwargs):      \n        #Tracer()()\n        ret_condition = (self.learn.dls[0] != self.learn.dl)\n        \n        if (ret_condition): #if not train do nothing\n            return \n        \n        cat,x = self.xb\n        #y,_ =self.learn.yb        \n        non_zeros_counts = self.xb[1].count_nonzero(dim=1)\n        make_zero_counts = (self.xb[1].count_nonzero(dim=1)/50).round()\n        changed_counts = (self.xb[1].count_nonzero(dim=1)/50).round()\n        bs = x.shape[0]\n        for i in range(bs):\n            elem = x[i]\n            #Tracer()()\n            indexes = elem.nonzero().reshape(-1)\n            for_zero_indexes = np.random.randint(0, len(indexes), int(make_zero_counts[i]))\n            change_count_i = int(changed_counts[i])\n            for_change_indexes = np.random.randint(0, len(indexes), change_count_i)\n            x[i, indexes[for_zero_indexes]]=0\n            #Tracer()()\n            #factor =  torch.tensor(0.95 + 0.1*np.random.rand(change_count_i)).to(x.device)\n            #x[i, indexes[for_change_indexes]] *=factor\n            \n        #Tracer()()\n        self.learn.xb = (cat, x)\n        #print(a.shape, b.shape)   \n        \n\n#Nothing special, cat names is empty, experiments with categories were not successful :)\ndef get_data(train_df,\n             fold, target_names = [], \n             procs = [Categorify, FillMissing],#, Normalize],\n             cat_names = [],\n             cont_names = []):\n    val_idx = train_df[train_df.kfold==fold].index\n    \n    dls = TabularDataLoaders.from_df(train_df, path=CSV_PATH, \n                                        y_names=target_names,#['B1', 'F1', 'F2', 'F3', 'F4'],list(enc.categories_[0])\n                                        cat_names = cat_names,\n                                        \n                                        cont_names = cont_names,\n                                        procs = procs,#, Normalize],\n                                        valid_idx=val_idx,\n                                        bs=64)\n    return dls\n\n\n\ndef run_train(train_df, fold, test_df, y_names, cat_names, cont_names, loss_func, y_range, name):\n    #Tracer()()\n    #new = train_df.loc[train_df['floor']=='B1'].reset_index()\n    dls = get_data(train_df, fold, target_names=y_names,cat_names=cat_names, cont_names= cont_names) # Data\n    model_dir = '/kaggle/working/'\n    learn = tabular_learner(dls, \n                        y_range=y_range,\n                            layers = [1024, 512, 512, 256],     \n                            #layers = [2048, 1024, 512, 256], \n                            #layers = [1536, 1024, 512, 256],\n                           \n                                #loss_func = BCELossFlat(),\n                                loss_func = loss_func,\n                                #config=config,\n                                model_dir=model_dir,\n                            #cbs=cbs\n                                cbs= [AugCallback()]\n                   ) # Model\n    cb = SaveModelCallback(monitor='valid_loss',fname=name )\n    lr=1e-3   \n    learn.fit_one_cycle(EPOCHS, lr_max=slice(lr/(2.6**4),lr)) # Training\n    lr=1e-4\n    learn.fit_one_cycle(EPOCHS, lr_max=slice(lr/(2.6**4),lr), cbs = cb) # Training  \n    \n    learn.load(name)\n    test_dl = learn.dls.test_dl(test_df)#learn.dls.valid#learn.valid_dllearn.dls.test_dl(test_features)\n    test_preds, _ = learn.get_preds(dl=test_dl) # prediction\n    #Tracer()()\n    \n    #test_dl = learn.dls.test_dl(test_df)#learn.dls.valid#learn.valid_dllearn.dls.test_dl(test_features)\n    val_preds, val_labels = learn.get_preds(dl=learn.dls.valid) # prediction\n    return val_preds, val_labels, test_preds\n\n\ndef get_test_df(site, columns,generate = False, wifi = False):\n    suffix = \".wifi.csv\" if wifi else '.ibeacon.csv'\n    if (not generate):\n        df = pd.read_csv(op.join(TEST_CSV_PATH, site + suffix))\n        df = handle_values(df)\n        return df\n    suffix2 = \".wifi.pre.csv\" if wifi else '.ibeacon.pre.csv'\n    df = pd.read_csv(op.join(TEST_CSV_PATH, site + suffix2))\n    result_df = pd.DataFrame(columns = ['timestamp', 'path'] + columns)\n    for (i,(a,b)) in tqdm(enumerate(df.groupby(['timestamp']))):\n        timestamp= a\n        row = {'timestamp':timestamp}\n        if (wifi):\n            bssid_values = b['bssid'].values    \n            rssi_values = b['rssi'].values           \n            row.update({k:v for (k,v) in zip(bssid_values,rssi_values)})                \n        else:\n            mac_values = b['mac'].values\n            rssi_values = b['rssi'].values\n            dist_values = b['distance'].values\n            txpower_values = b['txpower'].values                    \n            \n            row.update({'rssi_'+ k:v for (k,v) in zip(mac_values,rssi_values)})   \n            row.update({'txpower_'+ k:v for (k,v) in zip(mac_values,txpower_values)})\n            row.update({'dist_'+ k:v for (k,v) in zip(mac_values,dist_values)})\n        result_df = result_df.append(row, ignore_index=True)     \n            \n    for (i,(a,b)) in tqdm(enumerate(df[['timestamp', 'path']].groupby(['timestamp', 'path']))):\n        timestamp, path  = a\n      \n        #path = b['path'].values[0]\n\n        result_df.loc[result_df[result_df['timestamp']==timestamp].index,['path']] = path\n\n\n    result_df = result_df.fillna(0)        \n    result_df.to_csv(op.join(TEST_CSV_PATH, site  + suffix), index=False)  \n    result_df = handle_values(result_df)\n    return result_df\n\ndef update_ss_df(test_df, f_test_preds, xy_test_preds, site, cat_names):\n    floors_ind = np.argmax(f_test_preds, axis=1)\n    test_df['floor'] = floors_ind.tolist()\n    test_df['x'] = xy_test_preds[:,0].tolist()\n    test_df['y'] = xy_test_preds[:,1].tolist()\n    \n    for path, ss_time in ss_df_tmp[ss_df_tmp['site']==site][['path', 'timestamp']].values:\n        timestamps = test_df[test_df['path'] == path]['timestamp'].values\n        distances = np.abs(timestamps - ss_time)\n        argmin = np.argmin(distances)\n        #print(\"min dist found\", distances[argmin])\n        timestamp = timestamps[argmin]\n        #result['floor'] = [get_floor(cat_names[i]) for i in np.argmax(e,axis=1)]\n        #Tracer()()\n        row = test_df[(test_df['path']==path)&(test_df['timestamp']==timestamp)]\n    \n        floor = get_floor(cat_names[row['floor'].values[0]])\n        xy = row[['x','y']].values[0]\n        record_id = site + \"_\" + path + \"_\" + str(ss_time+ 10000000000000)[1:]\n        index = ss_df[ss_df['site_path_timestamp']==record_id].index    \n        ss_df.loc[index, ['floor','x','y']] = [floor, xy[0], xy[1]]\n        \nvalid_dict = {}\n\ndef fix_floor_in_df(df):\n    for a,b in df[['site', 'floor','path']].groupby(['site', 'path']):\n        site, path = a\n        floors = b['floor']\n        \n        uniques = np.unique(floors)\n        if (len(uniques) > 1):\n            #Tracer()()\n            print(\"site: \", site, \", path:\", path)\n            ind = np.argmax([np.where(floors==elem, 1, 0).sum() for elem in uniques])\n            floor = uniques[ind]\n            df_index = df[(df['path']==path)&(df['site']==site)].index\n            df.loc[df_index, 'floor']=floor\n            print(floors)\n            print(\"!!!\", floor)\n\n    for a,b in df[['site', 'floor','path']].groupby(['site', 'path']):\n        site, path = a\n        floors = b['floor']\n\n        uniques = np.unique(floors)\n        if (len(uniques) > 1):\n            #Tracer()()        \n            print(\"Not fixed\", floors)\n    return df\n\nPREFIX = TODAY\n  \nfor i, site in enumerate(test_sites):\n    #seed_everything(i) \n    xy_test_all = []\n    floor_test_all = []\n    print(\"_______Staring\", i, \" of \", len(test_sites), \" sites\")\n    \n    train_df, cat_names,cont_names, columns = get_train_df(site, True, True, True)\n    test_df = get_test_df(site, columns, False, True)\n    print(i, \" \", len(cont_names), \" \",len(train_df), \" \")#, len(test_site_df))\n        #TODO add number of folds\n    \n    for fold in range(FOLDS):\n        print(\"Starting fold \", fold)\n        seed_everything(fold)         \n\n        f_val_preds, f_val_labels, f_test_preds = run_train(train_df, fold, test_df, cat_names, [],cont_names, BCELossFlat(),\n                                                             (0,1),site + \"_\" + str(fold) + \"_f\")\n        \n        #f_val_preds, f_val_labels, f_test_preds = run_train(train_df, fold, test_df, cat_names, [],cont_names, None,\n        #                                                    None,site + \"_\" + str(fold) + \"_f\")\n\n        valid_dict[site + \"_f_labels\"] = f_val_labels\n        valid_dict[site + \"_f_preds\"] = f_val_preds\n        \n        floor_test_all.append(f_test_preds.tolist())\n   \n    \n    \n#     for fold in range(5):\n#         print(\"Starting fold for xy\", fold)\n#          seed_everything(fold)\n\n    \n    \n    \n        if i == 1 or i == 20 or i ==22:\n            width = 500\n        else:\n            width = 350\n        \n        xy_val_preds, xy_val_labels, xy_test_preds = run_train(train_df, #train_df[train_df[(train_df['B1'] != 1) & (train_df['B2'] != 1)]],\n\n                                                               fold, test_df, ['x','y'], [],cont_names, MSELossFlat(),\n                                                               (0, width), site + \"_\" + str(fold) + \"_xy\") \n        \n        xy_test_all.append(xy_test_preds.tolist())\n    \n    xy_test_all = np.array(xy_test_all).mean(axis=0)  \n    test_df['x'] = xy_test_all[:,0]\n    test_df['y'] = xy_test_all[:,1]\n    floor_test_all = np.array(floor_test_all).mean(axis=0)\n    test_df['floor'] = [cat_names[f] for f in floor_test_all.argmax(axis=1).tolist()]\n    test_df['site'] = site\n    \n    #test_df.to_csv(op.join(PATH, PREFIX, \"test_df_\" + site + \".csv\"))\n    \n    test_df = fix_floor_in_df(test_df)\n    update_ss_df(test_df, floor_test_all, xy_test_all, site, cat_names) \n    valid_dict[site + \"_xy_labels\"] = xy_val_labels\n    valid_dict[site + \"_xy_preds\"] = xy_val_preds\n    \n    #ss_df.to_csv(op.join(PATH, PREFIX, \"ss_df_\" + site + \".csv\"))","metadata":{"_uuid":"5b3f5c77-9c6b-4d96-8484-b438b6dcae9f","_cell_guid":"124fa735-0a87-4ead-aa02-eb517b98ed3c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-05-21T18:43:16.286564Z","iopub.execute_input":"2021-05-21T18:43:16.286973Z","iopub.status.idle":"2021-05-21T18:59:34.761683Z","shell.execute_reply.started":"2021-05-21T18:43:16.286925Z","shell.execute_reply":"2021-05-21T18:59:34.759274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nss_df_tmp['floor'] = ss_df['floor']\nss_df_tmp = fix_floor_in_df(ss_df_tmp)           \nss_df['floor']=ss_df_tmp['floor']           \nss_df.to_csv(op.join(\"\", \"submission-generated.csv\"), index=False)\n\n##submission to don't wait long run\n#This file was gotten on my local machine with EPOCHS=20\nsubm = pd.read_csv(op.join(CSV_PATH, \"04.05.1.csv\"))\nsubm.to_csv(op.join(\"\", \"subm.csv\"), index=False)","metadata":{"_uuid":"8f2d26eb-139c-4c6a-ad73-5e0fce012a24","_cell_guid":"d633dd72-46c7-49f3-9f67-8e3c1e80fc43","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-05-21T18:59:34.762630Z","iopub.status.idle":"2021-05-21T18:59:34.762997Z"},"trusted":true},"execution_count":null,"outputs":[]}]}