{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"#\n# Continuous embedding:\n#  https://www.kaggle.com/c/riiid-test-answer-prediction/discussion/210171\n#  https://github.com/dkletran/riiid-challenge-4th-place/blob/main/modeling_training/modeling.py\n#  https://arxiv.org/pdf/2010.12042.pdf\n#\n\nimport os\nimport glob\nimport numpy as np\nimport pandas as pd\nimport torch\nimport tqdm\nimport matplotlib\nimport time\nimport pickle\n\nsettings = {\n    'beacon_seq_len' : 50\n    ,'beacon_embed_dim' : 256\n    ,'device' : torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    ,'floor_mapping' : {'B1' : -1, 'B2' : -2, 'B3' : -3\n                         ,'F1' : 0, 'F2' : 1, 'F3' : 2, 'F4' : 3, 'F5' : 4\n                         ,'F6' : 5, 'F7' : 6 , 'F8' : 7, 'F9' : 8, 'F10' : 9\n                         ,'1F' : 0, '2F' : 1, '3F' : 2, '4F' : 3, '5F' : 4\n                         ,'6F' : 5, '7F' : 6, '8F' : 7, '9F' : 8\n                        }\n    ,'n_floor' : 13\n    ,'max_beacon_distance' : 200\n    ,'path_train' : '../input/indoor-location-navigation/train/*/*/*'\n    ,'path_test' : '../input/indoor-location-navigation/test/*'\n    ,'path_sample' : '../input/indoor-location-navigation/sample_submission.csv'\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('../input/location-data/wp_train.pkl', 'rb') as handle:\n    train = pickle.load(handle)\n    \nwith open('../input/location-data/beacon_train.pkl', 'rb') as handle:\n    tmp = pickle.load(handle)\n    \n# Handle this in other script eventually\ntrain['wp'] = train.pop('df')\ntrain['beacon'] = tmp.pop('df')\ntrain['wp'].x = train['wp'].x.astype('float')\ntrain['wp'].y = train['wp'].y.astype('float')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('../input/location-data/wp_train.pkl', 'rb') as handle:\n    train = pickle.load(handle)\n    \nwith open('../input/location-data/beacon_train.pkl', 'rb') as handle:\n    tmp = pickle.load(handle)\n    \n# Handle this in other script eventually\ntrain['wp'] = train.pop('df')\ntrain['beacon'] = tmp.pop('df')\ntrain['wp'].x = train['wp'].x.astype('float')\ntrain['wp'].y = train['wp'].y.astype('float')\n\n#\n# Delete trace_ids not in beacon\n# ***** Delete this eventually\n#\ntrain['wp'] = train['wp'].loc[train['wp'].trace_id.isin(train['beacon'].trace_id)]\n\n# Record settings for embedding\nsettings['n_uuids'] = train['beacon']['UUID'].nunique() + 1\nsettings['n_minor_ids'] = train['beacon']['MinorID'].nunique() + 1\nsettings['n_major_ids'] = train['beacon']['MajorID'].nunique() + 1\nsettings['n_macs'] = train['beacon']['MAC_Address'].nunique() + 1\nsettings['n_sites'] = train['beacon']['site_id'].nunique() + 1\n\n#\n# Make validation set\n#\n\n# Get trace_ids to be in validation set\nnp.random.seed(1)\ntmp = train['wp'].trace_id.unique()\nvalid_ids = np.random.choice(tmp, int(.2 * tmp.shape[0]), replace=False)\n\n# Make validation set\nvalid = {}\nvalid['beacon'] = train['beacon'].loc[train['beacon'].trace_id.isin(valid_ids)]\nvalid['wp'] = train['wp'].loc[train['wp'].trace_id.isin(valid_ids)]\n\n# Delete validation set from train\ntrain['beacon'] = train['beacon'].loc[~train['beacon'].trace_id.isin(valid_ids)]\ntrain['wp'] = train['wp'].loc[~train['wp'].trace_id.isin(valid_ids)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_data(data, settings):\n    \n    # Delete rows with 'non-testing' floors\n    i = data['wp'][~data['wp']['floor'].isin(settings['floor_mapping'].keys())].index\n    data['wp'].drop(i, inplace=True)\n    i = data['beacon'][~data['beacon']['floor']\\\n                        .isin(settings['floor_mapping'].keys())].index\n    data['beacon'].drop(i, inplace=True)\n    \n    # Reset indices\n    data['wp'].reset_index(drop=True, inplace=True)\n    data['beacon'].reset_index(drop=True, inplace=True)\n\n    # Only keep initial rows of a trace\n    data['beacon'] = data['beacon'].groupby('trace_id')\\\n                    .head(settings['beacon_seq_len']).reset_index(drop=True)\n\n    # Convert distance column to int\n    tmp = settings['max_beacon_distance']\n    data['beacon']['distance'] = np.where(data['beacon'][\"distance\"] > tmp\n                                       ,tmp, data['beacon'][\"distance\"])\n    data['beacon']['distance'] = np.where(data['beacon'][\"distance\"] < 0\n                                       ,0, data['beacon'][\"distance\"])\n    data['beacon']['distance'] = data['beacon']['distance'].astype(int) + 1\n\n    # Map columns\n    data['beacon']['site_id'] = data['beacon']['site_id'].astype('category').cat.codes + 1\n    data['beacon']['UUID'] = data['beacon']['UUID'].astype('category').cat.codes + 1\n    data['beacon']['MinorID'] = data['beacon']['MinorID'].astype('category').cat.codes + 1\n    data['beacon']['MajorID'] = data['beacon']['MajorID'].astype('category').cat.codes + 1\n    data['beacon']['MAC_Address'] = data['beacon']['MAC_Address'].astype('category').cat.codes + 1\n    data['beacon'].replace({'floor' : settings['floor_mapping']}, inplace=True)\n\n    # Convert to dictionary\n    data['beacon'] = {k: table for k, table in data['beacon'].groupby(\"trace_id\")} # Can optimize\n    data['wp'] = data['wp'].to_dict(orient='index')\n    \n    # Return\n    return(data)\n\ntrain = clean_data(train, settings)\nvalid = clean_data(valid, settings)\n\nprint(train['wp'][0])\nvalid['beacon'][list(valid['beacon'].keys())[0]].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class location_dataset(torch.utils.data.Dataset):\n    \n    def __init__(self, data, settings):\n        super(location_dataset, self).__init__()\n        self.beacon_seq_len = settings['beacon_seq_len']\n        self.n_floor = settings['n_floor']\n        self.data = data\n        \n    def __len__(self):\n        return(len(self.data['wp'].keys()))\n    \n    def __getitem__(self, index):\n        # Get the relevant user data\n        wp = self.data['wp'][index]\n        beacon = self.data['beacon'][wp['trace_id']]\n        \n        # Get contents as np.int64s\n        uuids = beacon['UUID'].values\n        distances = beacon['distance'].values\n        minor_ids = beacon['MinorID'].values\n        major_ids = beacon['MajorID'].values\n        macs = beacon['MAC_Address'].values\n        sites = beacon['site_id'].values\n        \n        # Pad if needed\n        n_pad = self.beacon_seq_len - len(uuids)\n        if n_pad > 0:\n            uuids = np.concatenate((uuids, np.full(n_pad, 0).astype(np.int64)))\n            distances = np.concatenate((distances, np.full(n_pad, 0).astype(np.int64)))\n            minor_ids = np.concatenate((minor_ids, np.full(n_pad, 0).astype(np.int64))) \n            major_ids = np.concatenate((major_ids, np.full(n_pad, 0).astype(np.int64)))\n            macs = np.concatenate((macs, np.full(n_pad, 0).astype(np.int64)))\n            sites = np.concatenate((sites, np.full(n_pad, 0).astype(np.int64)))\n        else:\n            uuids = uuids[:self.beacon_seq_len]\n            distances = distances[:self.beacon_seq_len]\n            minor_ids = minor_ids[:self.beacon_seq_len]\n            major_ids = major_ids[:self.beacon_seq_len]\n            macs = macs[:self.beacon_seq_len]\n            sites = sites[:self.beacon_seq_len]\n            \n        # Location of waypoint timestamp\n        #tmp = train['beacon']['5d09b23ccfb49b00085466a6'].timestamp\n        #print(tmp)\n        #tmp2 = tmp.loc[tmp > 15609164469999].index.min()\n\n        \n        # Return\n        return({\n            'x' : wp['x']\n            ,'y' : wp['y']\n            ,'uuids' : uuids\n            ,'distances' : distances\n            ,'minor_ids' : minor_ids\n            ,'major_ids' : major_ids\n            ,'macs' : macs\n            ,'sites' : sites\n        })\n    \n\ntrain_dataset = location_dataset(data = train\n                              ,settings = settings\n                              )\ntrain_dataloader = torch.utils.data.DataLoader(train_dataset\n                                                ,batch_size = 256\n                                                ,drop_last = True\n                                                ,shuffle = True\n                                                ,num_workers = 4\n                                               )\n\nvalid_dataset = location_dataset(data = valid\n                              ,settings = settings\n                              )\nvalid_dataloader = torch.utils.data.DataLoader(valid_dataset\n                                                ,batch_size = 1000\n                                                ,num_workers = 4\n                                               )\nvalid_dataset.__getitem__(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class xy_model(torch.nn.Module):\n    def __init__(self, settings):\n        super(xy_model, self).__init__()\n        self.embed_dim = settings['beacon_embed_dim']\n        self.seq_len = settings['beacon_seq_len']\n        self.device = settings['device']\n        \n        self.minor_id_embedding = torch.nn.Embedding(settings['n_minor_ids']\n                                                    ,self.embed_dim)\n        self.major_id_embedding = torch.nn.Embedding(settings['n_major_ids']\n                                                    ,self.embed_dim)\n        self.uuid_embedding = torch.nn.Embedding(settings['n_uuids']\n                                                 ,self.embed_dim)\n        self.mac_embedding = torch.nn.Embedding(settings['n_macs']\n                                                 ,self.embed_dim)\n        # Site embedding doesn't make to much sense (same site always)\n        self.site_embedding = torch.nn.Embedding(settings['n_sites']\n                                                ,self.embed_dim)\n        self.distance_embedding = torch.nn.Embedding(settings['max_beacon_distance']+2\n                                                     ,self.embed_dim)\n        self.pos_embedding = torch.nn.Embedding(self.seq_len, self.embed_dim)\n        self.multi_att = torch.nn.MultiheadAttention(embed_dim = self.embed_dim\n                                                     ,num_heads = 4\n                                                     ,dropout = 0.2)\n\n        self.lin_1 = torch.nn.Linear(self.embed_dim, self.embed_dim)\n        self.relu = torch.nn.ReLU()\n        self.lin_2 = torch.nn.Linear(self.embed_dim, 1)\n        self.dropout = torch.nn.Dropout(0.2)\n        \n        self.pred = torch.nn.Linear(self.seq_len, 2)\n        \n            \n    def forward(self, batch):        \n        # Minor id embedding\n        x = self.minor_id_embedding(batch['minor_ids'].long())\n        \n        # MAC Address embedding\n        x = x + self.mac_embedding(batch['macs'].long())\n        \n        # Site embedding\n        x = x + self.site_embedding(batch['sites'].long())\n        \n        # Major Id embedding\n        x = x + self.major_id_embedding(batch['major_ids'].long())\n        \n        # UUID embedding\n        #x = x + self.uuid_embedding(batch['uuids'].long())\n        \n        # Distance embedding\n        x = x + self.distance_embedding(batch['distances'])\n        \n        # Position embedding\n        pos_id = torch.arange(x.shape[1])[None, :].to(self.device)\n        x = x + self.pos_embedding(pos_id)\n        \n        # Permute\n        x = x.permute(1, 0, 2) # x: [bs, s_len, embed] => [s_len, bs, embed]\n        \n        # MultiHead Attention and permute back\n        attn_output, _ = self.multi_att(x, x, x)\n        x = x + attn_output\n        x = x.permute(1, 0, 2)\n        \n        # Feed forward\n        x = self.lin_1(x)\n        x = self.relu(x)\n        x = self.lin_2(x)\n        x = self.dropout(x)\n        \n        # Predict\n        x = x[:, :, -1]\n        x = self.pred(x)\n        \n        # Return\n        return(x)\n        \n\n# Setup model, optimizer and criterion\nmodel = xy_model(settings)\noptimizer = torch.optim.Adam(model.parameters(), lr=.002)\ncriterion = torch.nn.MSELoss()\n\n# Move model and criteriod to device\nmodel.to(settings['device'])\ncriterion.to(settings['device'])\nall_loss = []\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for _ in range(1):\n    tbar = tqdm.tqdm(train_dataloader)\n    for batch in tbar:\n        for k in batch.keys():\n            batch[k] = batch[k].to(settings['device'])\n        optimizer.zero_grad()\n        pred = model(batch)\n        targ = torch.cat((batch['x'][:, None], batch['y'][:, None]), 1).float()\n        loss = criterion(pred, targ)\n        loss.backward()\n        optimizer.step()\n        \n        # Record metrics\n        all_loss.append(loss.item())\n\nprint(np.array(all_loss[-200:]).mean())\nmatplotlib.pyplot.plot(all_loss)\nmatplotlib.pyplot.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def score(dataset, settings):\n    if dataset == 'valid':\n        dl = valid_dataloader\n    else:\n        dl = torch.utils.data.DataLoader(train_dataset\n                                        ,batch_size = 1000\n                                        ,num_workers = 4\n                                       )\n    # Accuracy mesurments\n    pred_x = np.empty(0, dtype=np.float)\n    pred_y = np.empty(0, dtype=np.float)\n    targ_x = np.empty(0, dtype=np.float)\n    targ_y = np.empty(0, dtype=np.float)\n\n    model.eval()\n    for batch in dl:\n        for k in batch.keys():\n            batch[k] = batch[k].to(settings['device'])\n\n        # Get predictions\n        pred = model(batch)\n        p = pred.detach().to('cpu').numpy()\n        p_x = p[:, 0]\n        p_y = p[:, 1]\n        pred_x = np.concatenate((pred_x, p_x))\n        pred_y = np.concatenate((pred_y, p_y))\n\n        # Target\n        t_x = batch['x'].detach().to('cpu').numpy()\n        t_y = batch['x'].detach().to('cpu').numpy()\n        targ_x = np.concatenate((targ_x, t_x))\n        targ_y = np.concatenate((targ_y, t_y))\n        \n    model.train()\n    tmp = sum(np.sqrt(np.square(pred_x - targ_x) + np.square(pred_y - targ_y)))/pred_x.shape[0]\n    print(tmp)\n    \n    \n# Get accuracy\nscore('valid', settings)\nscore('train', settings)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}