{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"* Saito's post-processing, alpha = 7.2, beta = 0.3\n* Adding time_gap feature from Kouki, improves LB by 0.02 (4.327 --> 4.307), CV improves by 0.1\n* Incorporate IMU data into the training process, improves LB by 0.014 (4.307 --> 4.293)\n* Interpolate the prediction improves LB by 0.096 (4.293 --> 4.197)\n* Leakage-based and magnetic Saito's post-processing gave 0.15 improvement (4.197 --> 4.049);\n* Use aggregated IMU data, LB: 4.049 --> 4.019;\n* Model IMU v4, adding delta waypoints and user ID, LB: 4.019 --> 3.937\n* Apply Darich's pp: LB: 3.937 --> ...","metadata":{"papermill":{"duration":0.03875,"end_time":"2021-05-12T04:46:06.658345","exception":false,"start_time":"2021-05-12T04:46:06.619595","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Necessary packages","metadata":{"papermill":{"duration":0.035241,"end_time":"2021-05-12T04:46:06.729467","exception":false,"start_time":"2021-05-12T04:46:06.694226","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\nfrom glob import glob\nimport numpy as np\nimport math\nimport pandas as pd\nfrom tqdm.notebook import tqdm\nimport copy\nimport pickle as pkl\nimport json\nimport matplotlib.pyplot as plt\nimport multiprocessing\n\nfrom sklearn.model_selection import GroupKFold\nfrom scipy.ndimage import gaussian_filter1d\nfrom scipy.spatial.distance import cdist\nimport scipy\nfrom scipy import optimize\nfrom scipy.optimize import minimize\n\nimport torch\nfrom torch import nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import AdamW, lr_scheduler\n\nimport gc\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":2.195854,"end_time":"2021-05-12T04:46:08.960802","exception":false,"start_time":"2021-05-12T04:46:06.764948","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import data","metadata":{"papermill":{"duration":0.03542,"end_time":"2021-05-12T04:46:09.032328","exception":false,"start_time":"2021-05-12T04:46:08.996908","status":"completed"},"tags":[]}},{"cell_type":"code","source":"test_data = pd.read_pickle('../input/indoor-test-set-based-on-wifi/test_all.pkl')   # ../input/indoor-interpolated-with-gap/test_all.pkl\ntest_data.rename(columns = {'site_id': 'building'}, inplace = True)\ntest_data['siteid_path'] = ['_'.join([i, j]) for i, j in zip(test_data['building'], test_data['path'])]","metadata":{"papermill":{"duration":3.463748,"end_time":"2021-05-12T04:46:12.531767","exception":false,"start_time":"2021-05-12T04:46:09.068019","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data['timestamp'] = [(13 - len(str(i))) * '0' + str(i) for i in test_data['timestamp']]\ntest_data['site_path_timestamp'] = ['_'.join([i, j, str(k)]) for i, j, k in zip(test_data.building, test_data.path, test_data.timestamp)]","metadata":{"papermill":{"duration":0.113406,"end_time":"2021-05-12T04:46:12.681855","exception":false,"start_time":"2021-05-12T04:46:12.568449","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Add delta waypoints for test data","metadata":{"papermill":{"duration":0.03583,"end_time":"2021-05-12T04:46:12.753771","exception":false,"start_time":"2021-05-12T04:46:12.717941","status":"completed"},"tags":[]}},{"cell_type":"code","source":"test_delta_waypoints = pd.read_csv('../input/indoor-test-set-based-on-wifi/test_all.csv')\ntest_data = test_data.merge(test_delta_waypoints, left_on = 'site_path_timestamp', right_on = 'site_path_timestamp')\ntest_data['timestamp'] = test_data['timestamp'].astype(int)","metadata":{"papermill":{"duration":0.667284,"end_time":"2021-05-12T04:46:13.459169","exception":false,"start_time":"2021-05-12T04:46:12.791885","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data.head()","metadata":{"papermill":{"duration":0.078467,"end_time":"2021-05-12T04:46:13.573788","exception":false,"start_time":"2021-05-12T04:46:13.495321","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imu_data = pd.read_pickle('../input/ilnaggregated-imu/test_imu_all.pkl')\nimu_data['path'] = [i.split('_')[-1] for i in imu_data['site_floor_path']]\nimu_data.head()","metadata":{"papermill":{"duration":0.225435,"end_time":"2021-05-12T04:46:13.837208","exception":false,"start_time":"2021-05-12T04:46:13.611773","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Wifi encoder and building map","metadata":{"papermill":{"duration":0.037745,"end_time":"2021-05-12T04:46:13.912897","exception":false,"start_time":"2021-05-12T04:46:13.875152","status":"completed"},"tags":[]}},{"cell_type":"code","source":"with open('../input/iln-wifi-and-building-mapping/label_encoder_bssid.pkl', 'rb') as f:\n    lbl_bssid = pkl.load(f)\n    \nwith open('../input/iln-wifi-and-building-mapping/building_map.json', 'r') as f:\n    building_map = json.load(f)\n\nbssid_map = dict(zip(lbl_bssid.classes_, lbl_bssid.transform(lbl_bssid.classes_)))","metadata":{"papermill":{"duration":0.645829,"end_time":"2021-05-12T04:46:14.597939","exception":false,"start_time":"2021-05-12T04:46:13.952110","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Encode BSSID\nbssid_features = [i for i in test_data.columns if i.startswith('bssid_')]\nrssi_features = [i for i in test_data.columns if i.startswith('rssi_')]\ntimegap_features = [i for i in test_data.columns if i.startswith('gap_')]\ntest_data[bssid_features] = test_data[bssid_features].applymap(lambda x: bssid_map[x])","metadata":{"papermill":{"duration":23.498144,"end_time":"2021-05-12T04:46:38.134532","exception":false,"start_time":"2021-05-12T04:46:14.636388","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data[bssid_features][test_data[rssi_features] == -999] = 0","metadata":{"papermill":{"duration":26.629725,"end_time":"2021-05-12T04:47:04.802490","exception":false,"start_time":"2021-05-12T04:46:38.172765","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Encode building\ntest_data['building'] = test_data.building.map(building_map)\ntest_data.head()","metadata":{"papermill":{"duration":0.217623,"end_time":"2021-05-12T04:47:05.059829","exception":false,"start_time":"2021-05-12T04:47:04.842206","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Add user ID","metadata":{"papermill":{"duration":0.05411,"end_time":"2021-05-12T04:47:05.165022","exception":false,"start_time":"2021-05-12T04:47:05.110912","status":"completed"},"tags":[]}},{"cell_type":"code","source":"users = pd.read_csv('../input/retrieving-user-id-from-leaked-wifi-feature/df.csv', usecols = ['path_id', 'user_id'])\nuser_map = dict(zip(users['path_id'], users['user_id']))\ntest_data['user_id'] = test_data['path'].map(user_map)","metadata":{"papermill":{"duration":0.22353,"end_time":"2021-05-12T04:47:05.433540","exception":false,"start_time":"2021-05-12T04:47:05.210010","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature extraction","metadata":{"papermill":{"duration":0.039107,"end_time":"2021-05-12T04:47:05.512166","exception":false,"start_time":"2021-05-12T04:47:05.473059","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def feature_extraction_wifi(x):\n    ## Path ID\n    path = x['path'].unique()[0]\n    \n    # Building\n    building = x['building'].unique().astype(int)\n    \n    # User ID\n    user = x['user_id'].unique().astype(int)\n    \n    # BSSID\n    bssid_feature = x[bssid_features].values.astype(int)\n    \n    # RSSI\n    rssi_feature = x[rssi_features].values.astype(float)\n    for i in range(len(rssi_features)):\n        rssi_feature[:,i] = gaussian_filter1d(rssi_feature[:,i], sigma = 2)\n    \n    # Timegap\n    timegap_feature = x[timegap_features].values.astype(float)\n    \n     # Delta waypoints\n    del_waypoints = x[['delta_x_hat', 'delta_y_hat']].values.astype(float)\n        \n    return path, building, user, bssid_feature, rssi_feature, timegap_feature, del_waypoints","metadata":{"papermill":{"duration":0.049443,"end_time":"2021-05-12T04:47:05.600903","exception":false,"start_time":"2021-05-12T04:47:05.551460","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Raw IMU\nacce_ = ['acce_x', 'acce_y', 'acce_z']\ngyro_ = ['gyro_x', 'gyro_y', 'gyro_z']\nmagn_ = ['magn_x', 'magn_y', 'magn_z']\nahrs_ = ['ahrs_x', 'ahrs_y', 'ahrs_z']\nimu_ = acce_ + gyro_ + magn_ + ahrs_\n\ndef feature_extraction_imu(df):\n    imu = df[imu_].values.astype(int)\n    return imu\n\nimu_group = imu_data.groupby('path').apply(feature_extraction_imu)","metadata":{"papermill":{"duration":0.44409,"end_time":"2021-05-12T04:47:06.084603","exception":false,"start_time":"2021-05-12T04:47:05.640513","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{"papermill":{"duration":0.039292,"end_time":"2021-05-12T04:47:06.163339","exception":false,"start_time":"2021-05-12T04:47:06.124047","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class ILN_Dataset(Dataset):\n    def __init__(self, group, imu_group, max_len_wifi = 512, max_len_imu = 25_000):\n        self.group = group\n        self.imu_group = imu_group\n        self.max_len_wifi = max_len_wifi\n        self.max_len_imu = max_len_imu\n        \n    def __len__(self):\n        return len(self.group)\n    \n    def __getitem__(self, idx):\n        # Extract sequences\n        path, building, user, bssid_feature, rssi_feature, timegap_feature, del_waypoints = self.group.iloc[idx]\n        \n        # Load IMU data\n        imu = self.imu_group.loc[path]\n        \n        seq_len_imu = imu.shape[0]\n        mask_imu = np.ones(seq_len_imu)\n        mask_imu_ = np.zeros(self.max_len_imu)\n        \n        # Sequence length\n        seq_len_wifi = bssid_feature.shape[0]\n        mask = np.ones(seq_len_wifi)\n        \n        mask_ = np.zeros(self.max_len_wifi)\n        bssid_feature_ = np.zeros((self.max_len_wifi, bssid_feature.shape[1]))\n        rssi_feature_ = np.zeros((self.max_len_wifi, rssi_feature.shape[1]))\n        timegap_feature_ = np.zeros((self.max_len_wifi, timegap_feature.shape[1]))\n        del_waypoints_ = np.zeros((self.max_len_wifi, del_waypoints.shape[1]))\n        \n        imu_ = np.zeros((self.max_len_imu, imu.shape[1]))\n        \n        if seq_len_wifi <= self.max_len_wifi:   # Pad\n            mask_[-seq_len_wifi:] = mask\n            bssid_feature_[-seq_len_wifi:,:] = bssid_feature\n            rssi_feature_[-seq_len_wifi:,:] = rssi_feature\n            timegap_feature_[-seq_len_wifi:,:] = timegap_feature\n            del_waypoints_[-seq_len_wifi:,:] = del_waypoints\n        else:    # Cut\n            mask_ = mask[-self.max_len_wifi:]\n            bssid_feature_ = bssid_feature[-self.max_len_wifi:,:]\n            rssi_feature_ = rssi_feature[-self.max_len_wifi:,:]\n            timegap_feature_ = timegap_feature[-self.max_len_wifi:,:]\n            del_waypoints_ = del_waypoints[-self.max_len_wifi:,:]\n            \n        if seq_len_imu > 0:\n            if seq_len_imu <= self.max_len_imu:   # Pad\n                mask_imu_[-seq_len_imu:] = mask_imu\n                imu_[-seq_len_imu:,:] = imu\n            else:    # Cut\n                mask_imu_ = mask_imu[-self.max_len_imu:]\n                imu_ = imu[-self.max_len_imu:,:]\n            \n        return {\n            'mask': torch.tensor(mask_, dtype = torch.bool),\n            'mask_imu': torch.tensor(mask_imu_, dtype = torch.bool),\n            'imu': torch.tensor(imu_, dtype = torch.float),\n            'building': torch.tensor(building, dtype = torch.long),\n            'user': torch.tensor(user, dtype = torch.long),\n            'bssid': torch.tensor(bssid_feature_, dtype = torch.long),\n            'rssi': torch.tensor(rssi_feature_, dtype = torch.float),\n            'timegap': torch.tensor(timegap_feature_, dtype = torch.float),\n            'del_waypoints': torch.tensor(del_waypoints_, dtype = torch.float)\n        }","metadata":{"papermill":{"duration":0.060405,"end_time":"2021-05-12T04:47:06.263162","exception":false,"start_time":"2021-05-12T04:47:06.202757","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{"papermill":{"duration":0.039906,"end_time":"2021-05-12T04:47:06.343504","exception":false,"start_time":"2021-05-12T04:47:06.303598","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def clones(module, N):\n    \"Produce N identical layers.\"\n    return nn.ModuleList([copy.deepcopy(module) for _ in range(N)])\n\ndef attention(query, key, value, mask=None, dropout=None):\n    \"Compute 'Scaled Dot Product Attention'\"\n    d_k = query.size(-1)\n    scores = torch.matmul(query, key.transpose(-2, -1)) \\\n             / math.sqrt(d_k)\n    if mask is not None:\n        scores = scores.masked_fill(mask == 0, -1e9)\n    p_attn = F.softmax(scores, dim=-1)\n    if dropout is not None:\n        p_attn = dropout(p_attn)\n    return torch.matmul(p_attn, value), p_attn\n\n\nclass MultiHeadedAttention(nn.Module):\n    def __init__(self, d_model, nhead, dropout=0.1):\n        \"Take in model size and number of heads.\"\n        super(MultiHeadedAttention, self).__init__()\n        assert d_model % nhead == 0\n        # We assume d_v always equals d_k\n        self.d_k = d_model // nhead\n        self.nhead = nhead\n        self.linears = clones(nn.Linear(d_model, d_model, bias=False), 4) # Q, K, V, last\n        self.attn = None\n        self.dropout = nn.Dropout(p=dropout)\n\n    def forward(self, query, key, value, mask=None):\n        \"Implements Figure 2\"\n        if mask is not None:\n            # Same mask applied to all h heads.\n            mask = mask.unsqueeze(1)\n        nbatches = query.size(0)\n\n        # 1) Do all the linear projections in batch from d_model => h x d_k\n        query, key, value = \\\n            [l(x).view(nbatches, -1, self.nhead, self.d_k).transpose(1, 2)\n             for l, x in zip(self.linears, (query, key, value))]\n\n        # 2) Apply attention on all the projected vectors in batch.\n        x, self.attn = attention(query, key, value, mask=mask,\n                                 dropout=self.dropout)\n\n        # 3) \"Concat\" using a view and apply a final linear.\n        x = x.transpose(1, 2).contiguous() \\\n            .view(nbatches, -1, self.nhead * self.d_k)\n        return self.linears[-1](x)\n\n\nclass PositionwiseFeedForward(nn.Module):\n    \"Implements FFN equation.\"\n    def __init__(self, d_model, d_ff, dropout=0.1):\n        super(PositionwiseFeedForward, self).__init__()\n        self.w_1 = nn.Linear(d_model, d_ff)\n        self.w_2 = nn.Linear(d_ff, d_model)\n        self.dropout = nn.Dropout(dropout)\n\n    def forward(self, x):\n        return self.w_2(self.dropout(F.relu(self.w_1(x))))\n    \nclass EncoderLayer(nn.Module):\n    \"\"\"\n    Single Encoder block of SAINT\n    \"\"\"\n    def __init__(self, d_model, nhead, dim_feedforward = 1024, dropout = 0.1):\n        super().__init__()\n        self._self_attn = MultiHeadedAttention(d_model, nhead, dropout)\n        self._ffn = PositionwiseFeedForward(d_model, dim_feedforward, dropout)\n        self._layernorms = clones(nn.LayerNorm(d_model, eps=1e-6), 2)\n        self._dropout = nn.Dropout(dropout)\n\n    def forward(self, src, mask = None):\n        \"\"\"\n        query: question embeddings\n        key: interaction embeddings\n        \"\"\"\n        # self-attention block\n        src2 = self._self_attn(query=src, key=src, value=src, mask=mask)\n        src = src + self._dropout(src2)\n        src = self._layernorms[0](src)\n        src2 = self._ffn(src)\n        src = src + self._dropout(src2)\n        src = self._layernorms[1](src)\n        return src","metadata":{"papermill":{"duration":0.064724,"end_time":"2021-05-12T04:47:06.447711","exception":false,"start_time":"2021-05-12T04:47:06.382987","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Linformer Layer","metadata":{"papermill":{"duration":0.03987,"end_time":"2021-05-12T04:47:06.528016","exception":false,"start_time":"2021-05-12T04:47:06.488146","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class Lin_MultiHeadedAttention(nn.Module):\n    def __init__(self, d_model, nhead, dropout=0.1, max_len = 512, target_len = 400):\n        \"Take in model size and number of heads.\"\n        super(Lin_MultiHeadedAttention, self).__init__()\n        assert d_model % nhead == 0\n        # We assume d_v always equals d_k\n        self.d_k = d_model // nhead\n        self.nhead = nhead\n        self.linears = clones(nn.Linear(d_model, d_model, bias=False), 4) # Q, K, V, last\n        self.linear_key_value = clones(nn.Linear(max_len, 256, bias= False), 3)\n        self.linear_query = nn.Linear(max_len, target_len)\n        self.attn = None\n        self.dropout = nn.Dropout(p=dropout)\n\n    def forward(self, query, key, value, mask=None):\n        \"Implements Figure 2\"\n        if mask is not None:\n            # Same mask applied to all h heads.\n            mask = mask.unsqueeze(-1)\n            # Mask query, key and value\n            query = query * mask\n            key = key * mask\n            mask = value * mask\n        nbatches = query.size(0)\n        \n        # 1) Do all the linear projections in batch from d_model => h x d_k\n        query, key, value = \\\n            [l(x).view(nbatches, -1, self.nhead, self.d_k).transpose(1, 2)\n             for l, x in zip(self.linears, (query, key, value))]\n        \n        # 2) Linear projections\n        key, value = \\\n            [l(x).transpose(-1, -2)\n             for l, x in zip(self.linear_key_value, (key.transpose(-1,-2), value.transpose(-1,-2)))]\n        \n        query = self.linear_query(query.transpose(-1, -2)).transpose(-1, -2)\n\n        # 3) Apply attention on all the projected vectors in batch.\n        x, self.attn = attention(query, key, value, mask=None,\n                                 dropout=self.dropout)\n\n        # 4) \"Concat\" using a view and apply a final linear.\n        x = x.transpose(1, 2).contiguous() \\\n            .view(nbatches, -1, self.nhead * self.d_k)\n        return self.linears[-1](x)\n    \nclass Lin_EncoderLayer(nn.Module):\n    \"\"\"\n    Single Encoder block of SAINT\n    \"\"\"\n    def __init__(self, d_model, nhead, dim_feedforward = 1024, dropout = 0.1, max_len = 512, target_len = 400):\n        super().__init__()\n        self._self_attn = Lin_MultiHeadedAttention(d_model, nhead, dropout = dropout, max_len = max_len, target_len = target_len)\n        self._ffn = PositionwiseFeedForward(d_model, dim_feedforward, dropout)\n        self._layernorms = clones(nn.LayerNorm(d_model, eps=1e-6), 2)\n        self._dropout = nn.Dropout(dropout)\n\n    def forward(self, src, mask = None):\n        \"\"\"\n        query: question embeddings\n        key: interaction embeddings\n        \"\"\"\n        # self-attention block\n        src2 = self._self_attn(query=src, key=src, value=src, mask=mask)\n        src = self._dropout(src2)\n        src = self._layernorms[0](src)\n        src2 = self._ffn(src)\n        src = src + self._dropout(src2)\n        src = self._layernorms[1](src)\n        return src","metadata":{"papermill":{"duration":0.062514,"end_time":"2021-05-12T04:47:06.630980","exception":false,"start_time":"2021-05-12T04:47:06.568466","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PositionalEncoding(nn.Module):\n\n    def __init__(self, d_model, dropout=0.1, max_len=5000):\n        super(PositionalEncoding, self).__init__()\n        self.max_len = max_len\n        self.dropout = nn.Dropout(p=dropout)\n        self.div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))\n\n    def forward(self, x, mask = None):\n        if mask is not None:\n            position = torch.cumsum(mask.unsqueeze(-1), dim = 1)\n        else:\n            position = torch.repeat_interleave(torch.arange(0, self.max_len).unsqueeze(-1).unsqueeze(0), x.shape[0], dim = 0)\n        \n        pe = torch.zeros(x.shape).to(x.device)\n        div_term = self.div_term.to(x.device)\n        pe[:,:,0::2] = torch.sin(position * div_term)\n        pe[:,:,1::2] = torch.cos(position * div_term)\n        \n        x = x + pe\n        \n        return self.dropout(x)","metadata":{"papermill":{"duration":0.052624,"end_time":"2021-05-12T04:47:06.723629","exception":false,"start_time":"2021-05-12T04:47:06.671005","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ILN_Transformer(nn.Module):\n    def __init__(self, num_feature_bssid = 100, num_feature_rssi = 100, num_building = 24, num_bssid = 216210, num_user = 27551, \n                 num_floor = 11, num_feature_imu = 12, d_model = 512, nhead = 4, max_len = 512, max_len_imu = 25_000, droprate = 0.1):\n        super().__init__()\n        self.num_feature_bssid = num_feature_bssid\n        self.num_feature_rssi = num_feature_rssi\n        self.num_building = num_building\n        self.num_bssid = num_bssid\n        self.num_user = num_user\n        self.num_floor = num_floor\n        self.d_model = d_model\n        self.nhead = nhead\n        self.max_len = max_len\n        self.max_len_imu = max_len_imu\n        self.droprate = droprate\n        \n        ############################################ Wifi ############################################\n        # Embedding layers\n        self.building_embedding = nn.Embedding(num_embeddings = self.num_building, embedding_dim = self.d_model)\n        self.user_embedding = nn.Embedding(num_embeddings = self.num_user, embedding_dim = self.d_model)\n        self.bssid_embedding = nn.Embedding(num_embeddings = self.num_bssid, embedding_dim = self.d_model // 2, padding_idx = 0)\n        \n        # Linear layers for BSSID\n        self.linear_bssid = nn.Linear(self.num_feature_bssid, 1)\n        \n        # Linear layers for RSSI and timegap\n        self.linear_rssi_timegap = nn.Linear(2, 1)\n        self.linear_rssi = nn.Linear(self.num_feature_rssi, self.d_model // 2)\n        self.layer_norm_rssi = nn.LayerNorm(self.d_model // 2)\n        self.dropout_rssi = nn.Dropout(droprate)\n        \n        ############################################ IMU ############################################\n        # Linear projection\n        self.linear_imu_rotation = nn.Sequential(\n            nn.Linear(num_feature_imu, d_model // 2),\n            nn.LayerNorm(d_model // 2),\n            nn.ReLU(),\n            nn.Dropout(droprate)\n        )\n        \n        # Positional encoder\n        self.position_imu = PositionalEncoding(d_model = self.d_model // 2, dropout = self.droprate, max_len = self.max_len_imu)   # Because the IMU sequences are super long, we need a smaller model size to fit to the memory capacity\n        \n        # Attention\n        self.attn_encoder_imu = Lin_EncoderLayer(d_model = self.d_model // 2, nhead = self.nhead, dim_feedforward = 256, dropout = self.droprate, max_len = self.max_len_imu, target_len = self.max_len)\n        \n        # self.shrink_time = nn.Linear(self.max_len_imu, self.max_len)\n        \n        # Output layer\n        self.imu_output = nn.LSTM(input_size = d_model // 2, hidden_size = d_model // 2, bidirectional = True, batch_first = True)\n        \n        ############################################ Concatenate and prediction ############################################\n        # Middle linear layers\n        self.middle_linear = nn.Sequential(\n            nn.Linear(2 * d_model, d_model),\n            nn.LayerNorm(d_model),\n            nn.ReLU(),\n            nn.Dropout(droprate)\n        )\n        \n        # LSTM\n        self.lstms = clones(nn.LSTM(input_size = self.d_model, hidden_size = self.d_model // 2, bidirectional = True, batch_first = True), 3)\n        \n        # Positional encoder\n        self.position = PositionalEncoding(d_model = self.d_model, dropout = self.droprate, max_len = self.max_len)\n        \n        # Attention\n        self.attn_encoder = EncoderLayer(d_model = self.d_model, nhead = self.nhead, dim_feedforward = 512, dropout = self.droprate)\n        \n        self.lstm = nn.LSTM(input_size = self.d_model + 2, hidden_size = self.d_model // 2, bidirectional = True, batch_first = True)\n        self.linear_after_encoder = nn.Linear(self.d_model, self.d_model)\n        self.layernorm_after_encoder = nn.LayerNorm(self.d_model)\n        \n        # Output\n        self.linear_waypoints = nn.Linear(self.d_model, 2)\n        \n        # Activation\n        self.softmax = nn.Softmax()\n        self.prelu = nn.PReLU()\n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(self.droprate)\n        \n    def _get_pad_mask(self, seq, pad_idx):\n        return (seq == pad_idx).unsqueeze(-2).to(seq.device)\n    \n    def _get_subsequent_mask(self, seq):\n        sz_b, len_s = seq.size()\n        subsequent_mask = torch.triu(torch.ones((1, len_s, len_s), device = seq.device), diagonal = 1).bool()\n        return subsequent_mask\n        \n    def forward(self, mask, mask_imu, building, user, bssid, rssi, timegap, imu, del_waypoints):\n        ############################################ Wifi ############################################\n        # Embed\n        building = self.building_embedding(building)\n        user = self.user_embedding(user)\n        bssid = self.bssid_embedding(bssid)\n        \n        # BSSID\n        bssid = self.linear_bssid(bssid.transpose(-2,-1)).squeeze(-1)\n        \n        # RSSI and timegap\n        rssi = torch.cat((rssi.unsqueeze(-1), timegap.unsqueeze(-1)), dim = -1)\n        rssi = self.linear_rssi_timegap(rssi).squeeze(-1)\n        rssi = self.linear_rssi(rssi)\n        rssi = self.layer_norm_rssi(rssi)\n        rssi = self.dropout_rssi(rssi)\n        \n        ############################################ IMU ############################################\n        # Projection\n        imu = self.linear_imu_rotation(imu)\n        \n        # Positional embedding\n        imu = self.position_imu(imu, mask = mask_imu)\n        \n        # Attention\n        imu = self.attn_encoder_imu(imu, mask = mask_imu)\n        \n        # Shrink time\n        # imu = self.shrink_time(imu.transpose(-1,-2)).transpose(-1,-2)\n        \n        # Output IMU\n        imu, _ = self.imu_output(imu)\n        \n        ############################################ Concatenate and prediction ############################################\n        x_wifi = torch.cat((bssid, rssi), dim = -1)\n        \n        x = self.middle_linear(torch.cat((bssid, rssi, imu), dim = -1)) + building + user\n        \n        for i, layer in enumerate(self.lstms):\n            x, _ = layer(x)\n        \n        # Positional encoding\n        x = self.position(x, mask = mask)\n        \n        # Mask\n        mask = ~(self._get_pad_mask(mask, False) | self._get_subsequent_mask(mask))\n        \n        # Feed it to the Encoder\n        x = self.attn_encoder(x, mask)\n        \n        # Feed it to the lstm\n        x, _ = self.lstm(torch.cat((x + x_wifi + imu, del_waypoints), dim = -1))\n        \n        # Feed over one more linear layer\n        x = self.linear_after_encoder(self.dropout(self.prelu(x)))\n        x = self.layernorm_after_encoder(x)\n        x = self.dropout(self.prelu(x))\n        \n        # Concatenate floor output and x + building\n        x = self.linear_waypoints(x)\n        \n        return x","metadata":{"papermill":{"duration":0.075236,"end_time":"2021-05-12T04:47:06.839028","exception":false,"start_time":"2021-05-12T04:47:06.763792","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configuration and util functions","metadata":{"papermill":{"duration":0.04067,"end_time":"2021-05-12T04:47:06.920047","exception":false,"start_time":"2021-05-12T04:47:06.879377","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def infer_fn(model, infer_dataloader, device = 'cpu'):\n    model.eval()\n    \n    pad = []\n    way_pred = []\n    floor_pred = []\n    \n    loss = 0\n    \n    for item in infer_dataloader:\n        padding_mask = item['mask'].to(device)\n        padding_mask_imu = item['mask_imu'].to(device)\n        imu = item['imu'].to(device)\n        building = item['building'].to(device)\n        user = item['user'].to(device)\n        bssid = item['bssid'].to(device)\n        rssi = item['rssi'].to(device)\n        timegap = item['timegap'].to(device)\n        del_waypoints = item['del_waypoints'].to(device)\n        \n        # Feed input to the model\n        with torch.no_grad():\n            output_waypoints = model(padding_mask, padding_mask_imu, building, user, bssid, rssi, timegap, imu, del_waypoints)\n        \n        # Extract non-padded waypoint outputs\n        output_waypoints = output_waypoints[padding_mask]\n        \n        # Store results\n        pad.append(padding_mask.cpu().detach().numpy())\n        way_pred.append(output_waypoints.cpu().detach().numpy())\n        \n    # Stack\n    pad = np.vstack(pad)\n    way_pred = np.vstack(way_pred)\n    \n    return pad, way_pred","metadata":{"papermill":{"duration":0.052819,"end_time":"2021-05-12T04:47:07.013176","exception":false,"start_time":"2021-05-12T04:47:06.960357","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Configuration","metadata":{"papermill":{"duration":0.040265,"end_time":"2021-05-12T04:47:07.094500","exception":false,"start_time":"2021-05-12T04:47:07.054235","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class config():\n    # For inference\n    device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n    # For dataloader\n    batch_size = 64\n    num_workers = 4\n    # For model\n    N = 2    # Number of encoder layers\n    d_model = 256\n    nhead = 2\n    max_len = 400\n    max_len_imu = 1024\n    droprate = 0.3\n    \ncfg = config()","metadata":{"papermill":{"duration":0.049932,"end_time":"2021-05-12T04:47:07.185643","exception":false,"start_time":"2021-05-12T04:47:07.135711","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Main","metadata":{"papermill":{"duration":0.041189,"end_time":"2021-05-12T04:47:07.267794","exception":false,"start_time":"2021-05-12T04:47:07.226605","status":"completed"},"tags":[]}},{"cell_type":"code","source":"model_path = '../input/iln-models'\n\n# Dataloader\ninfer_group = test_data.groupby('siteid_path').apply(feature_extraction_wifi)\ninfer_dataset = ILN_Dataset(infer_group, imu_group, max_len_wifi = cfg.max_len, max_len_imu = cfg.max_len_imu)\ninfer_dataloader = DataLoader(infer_dataset, batch_size = cfg.batch_size, num_workers = cfg.num_workers, shuffle = False)\n\nwaypoints_prediction = []\n\nfor i in tqdm(range(5)):\n    ckp = torch.load(os.path.join(model_path, f'model_best_fold_{i}_imu_v4.pt'), map_location = cfg.device)\n    model = ILN_Transformer(d_model = cfg.d_model, nhead = cfg.nhead, max_len = cfg.max_len, \n                            max_len_imu = cfg.max_len_imu, droprate = cfg.droprate).to(cfg.device)\n    model.load_state_dict(ckp['model_state_dict'])\n    padding_mask, waypoints_pred = infer_fn(model, infer_dataloader, device = cfg.device)\n    waypoints_prediction.append(waypoints_pred)\n    \n# Average all folds\nwaypoints_prediction = np.mean(waypoints_prediction, axis = 0)\n\n# Prediction\nprediction = waypoints_prediction","metadata":{"papermill":{"duration":260.317077,"end_time":"2021-05-12T04:51:27.626024","exception":false,"start_time":"2021-05-12T04:47:07.308947","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = pd.DataFrame(prediction, columns = ['x', 'y'])\nresult[['site', 'path', 'timestamp']] = test_data[['site', 'path', 'timestamp']].values\nresult['siteit_path'] = ['_'.join([i, j]) for i, j in zip(result['site'], result['path'])]\nresult.set_index('siteit_path', inplace = True)","metadata":{"papermill":{"duration":0.115481,"end_time":"2021-05-12T04:51:27.782459","exception":false,"start_time":"2021-05-12T04:51:27.666978","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_waypoints = pd.read_csv('../input/indoor-location-train-waypoints/train_waypoints.csv')\nss = pd.read_csv('../input/indoor-location-navigation/sample_submission.csv')\nss[['site','path', 'timestamp']] = [i.split('_') for i in ss.site_path_timestamp]\n\nsamples = pd.DataFrame(ss.groupby(['site','path'])['timestamp'].apply(lambda x: list(x)))\nbuildings = np.unique([x[0] for x in samples.index])\nsamples.head()","metadata":{"papermill":{"duration":0.383479,"end_time":"2021-05-12T04:51:28.207082","exception":false,"start_time":"2021-05-12T04:51:27.823603","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!git clone --depth 1 https://github.com/location-competition/indoor-location-competition-20 indoor_location_competition_20\n!rm -rf indoor_location_competition_20/data\n\nfrom indoor_location_competition_20.io_f import read_data_file\nimport indoor_location_competition_20.compute_f as compute_f","metadata":{"papermill":{"duration":38.085491,"end_time":"2021-05-12T04:52:06.335187","exception":false,"start_time":"2021-05-12T04:51:28.249696","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.interpolate import interp1d\nfrom scipy.ndimage.filters import uniform_filter1d\n\ncolacce = ['xyz_time','x_acce','y_acce','z_acce']\ncolahrs = ['xyz_time','x_ahrs','y_ahrs','z_ahrs']\n\nfor building in buildings:\n    print(building)\n    paths = samples.loc[building].index\n    # Acceleration info:\n    tfm = pd.read_csv(f'../input/indoor-gbm-postprocessing-xy-prediction/indoor_testing_accel/{building}.txt',index_col = 0)\n    for path_id in paths:\n        # Original predicted values:\n        xy = result.loc[building + '_' + path_id]\n        tfmi = tfm.loc[path_id]\n        acce_datas = np.array(tfmi[colacce],dtype = np.float)\n        ahrs_datas = np.array(tfmi[colahrs],dtype = np.float)\n        posi_datas = np.array(xy[['timestamp', 'x', 'y']], dtype = np.float)\n        # Outlier removal:\n        xyout = uniform_filter1d(posi_datas, size = 3, axis = 0, mode = 'reflect')\n        xydiff = np.abs(posi_datas - xyout)\n        xystd = np.std(xydiff,axis = 0) * 3\n        posi_datas = posi_datas[(xydiff [:,1] < xystd[1]) & (xydiff[:,2] < xystd[2])]\n        # Step detection:\n        step_timestamps, step_indexs, step_acce_max_mins = compute_f.compute_steps(acce_datas)\n        stride_lengths = compute_f.compute_stride_length(step_acce_max_mins)\n        # Orientation detection:\n        headings = compute_f.compute_headings(ahrs_datas)\n        step_headings = compute_f.compute_step_heading(step_timestamps, headings)\n        rel_positions = compute_f.compute_rel_positions(stride_lengths, step_headings)\n        # Running average:\n        posi_datas = uniform_filter1d(posi_datas,size = 3,axis = 0,mode = 'reflect')[0::3,:]\n        # The 1st prediction timepoint should be earlier than the 1st step timepoint.\n        rel_positions = rel_positions[rel_positions[:,0] > posi_datas[0,0],:]\n        # If two consecutive predictions are in-between two step datapoints,\n        # the last one is removed, causing error (in the \"split_ts_seq\" function).\n        posi_index = [np.searchsorted(rel_positions[:,0], x, side = 'right') for x in posi_datas[:,0]]\n        u, i1, i2 = np.unique(posi_index, return_index = True, return_inverse = True)\n        posi_datas = np.vstack([np.mean(posi_datas[i2 == i],axis = 0) for i in np.unique(i2)])\n        # Position correction:\n        step_positions = compute_f.correct_positions(rel_positions, posi_datas)\n        # Interpolate for timestamps in the testing set:\n        t = step_positions[:,0]\n        x = step_positions[:,1]\n        y = step_positions[:,2]\n        fx = interp1d(t, x, kind = 'linear', fill_value = (x[0], x[-1]), bounds_error = False) #fill_value=\"extrapolate\"\n        fy = interp1d(t, y, kind = 'linear', fill_value = (y[0], y[-1]), bounds_error = False)\n        # Output result:\n        t0 = np.array(samples.loc[(building,path_id),'timestamp'], dtype = np.float64)\n        \n        ss.loc[(ss.site == building) & (ss.path == path_id), 'x'] = fx(t0)\n        ss.loc[(ss.site == building) & (ss.path == path_id), 'y'] = fy(t0)\n\nss.head()","metadata":{"papermill":{"duration":746.938912,"end_time":"2021-05-12T05:04:47.085940","exception":false,"start_time":"2021-05-12T04:52:20.147028","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Use floor prediction from Kouki","metadata":{"papermill":{"duration":0.111989,"end_time":"2021-05-12T05:04:47.311889","exception":false,"start_time":"2021-05-12T05:04:47.199900","status":"completed"},"tags":[]}},{"cell_type":"code","source":"floor_prediction = pd.read_csv('../input/indoor-support-data/floor_pred_0507.csv', usecols = ['path', 'floor'])\nfloor_prediction = dict(zip(floor_prediction['path'], floor_prediction['floor']))\nss['floor'] = ss['path'].map(floor_prediction)\nss['timestamp'] = ss['timestamp'].astype(int)","metadata":{"papermill":{"duration":0.147214,"end_time":"2021-05-12T05:04:47.571597","exception":false,"start_time":"2021-05-12T05:04:47.424383","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss[['site_path_timestamp', 'floor', 'x', 'y']].to_csv('submission_raw.csv', index = None)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Visualizing function","metadata":{"papermill":{"duration":0.112026,"end_time":"2021-05-12T05:04:47.795866","exception":false,"start_time":"2021-05-12T05:04:47.683840","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Helper Functions\ndef split_col(df):\n    df = pd.concat([\n        df['site_path_timestamp'].str.split('_', expand=True) \\\n        .rename(columns={0:'site',\n                         1:'path',\n                         2:'timestamp'}),\n        df\n    ], axis=1).copy()\n    return df\n\nfloor_map = {\"B2\":-2, \"B1\":-1, \"F1\":0, \"F2\": 1, \"F3\":2,\n             \"F4\":3, \"F5\":4, \"F6\":5, \"F7\":6,\"F8\":7,\"F9\":8,\n             \"1F\":0, \"2F\":1, \"3F\":2, \"4F\":3, \"5F\":4, \"6F\":5,\n             \"7F\":6, \"8F\": 7, \"9F\":8}\n\ndef plot_preds(\n    site,\n    floorNo,\n    sub=None,\n    true_locs=None,\n    base=\"../input/indoor-location-navigation\",\n    show_train=True,\n    show_preds=True,\n    show_smoothed=False,\n    fix_labels=True,\n    map_floor=None\n):\n    \"\"\"\n    Plots predictions on floorplan map.\n    \n    map_floor : use a different floor's map\n    \"\"\"\n    if map_floor is None:\n        map_floor = floorNo\n    # Prepare width_meter & height_meter (taken from the .json file)\n    floor_plan_filename = f\"{base}/metadata/{site}/{map_floor}/floor_image.png\"\n    json_plan_filename = f\"{base}/metadata/{site}/{map_floor}/floor_info.json\"\n    with open(json_plan_filename) as json_file:\n        json_data = json.load(json_file)\n\n    width_meter = json_data[\"map_info\"][\"width\"]\n    height_meter = json_data[\"map_info\"][\"height\"]\n\n    floor_img = plt.imread(f\"{base}/metadata/{site}/{map_floor}/floor_image.png\")\n\n    fig, ax = plt.subplots(figsize=(12, 12))\n    plt.imshow(floor_img)\n\n    if show_train:\n        true_locs = true_locs.query('site == @site and floorNo == @map_floor').copy()\n        true_locs[\"x_\"] = true_locs[\"x\"] * floor_img.shape[0] / height_meter\n        true_locs[\"y_\"] = (\n            true_locs[\"y\"] * -1 * floor_img.shape[1] / width_meter\n        ) + floor_img.shape[0]\n        true_locs.query(\"site == @site and floorNo == @map_floor\").groupby(\"path\").plot(\n            x=\"x_\",\n            y=\"y_\",\n            style=\"+\",\n            ax=ax,\n            label=\"train waypoint location\",\n            color=\"grey\",\n            alpha=0.5,\n        )\n\n    if show_preds:\n        if not show_smoothed:\n            sub = sub.query('site == @site and floorNo == @floorNo').copy()\n            sub[\"x_\"] = sub[\"x\"] * floor_img.shape[0] / height_meter\n            sub[\"y_\"] = (\n                sub[\"y\"] * -1 * floor_img.shape[1] / width_meter\n            ) + floor_img.shape[0]\n            for path, path_data in sub.query(\n                \"site == @site and floorNo == @floorNo\"\n            ).groupby(\"path\"):\n                path_data.plot(\n                    x=\"x_\",\n                    y=\"y_\",\n                    style=\".-\",\n                    ax=ax,\n                    title=f\"{site} - floor - {floorNo}\",\n                    alpha=1,\n                    label=path,\n                )\n        else:\n            sub = sub.query('site == @site and floorNo == @floorNo').copy()\n            sub[\"x_\"] = sub[\"x\"] * floor_img.shape[0] / height_meter\n            sub[\"y_\"] = (\n                sub[\"y\"] * -1 * floor_img.shape[1] / width_meter\n            ) + floor_img.shape[0]\n            for path, path_data in sub.query(\n                \"site == @site and floorNo == @floorNo\"\n            ).groupby(\"path\"):\n                path_data.plot(\n                    x=\"x_\",\n                    y=\"y_\",\n                    style=\".-\",\n                    ax=ax,\n                    title=f\"{site} - floor - {floorNo}\",\n                    alpha=1,\n                    label=path,\n                )\n    if fix_labels:\n        handles, labels = ax.get_legend_handles_labels()\n        by_label = dict(zip(labels, handles))\n        plt.legend(\n            by_label.values(), by_label.keys(), loc=\"center left\", bbox_to_anchor=(1, 0.5)\n        )\n    return fig, ax\n\ndef sub_process(sub, train_waypoints):\n    train_waypoints['isTrainWaypoint'] = True\n    sub = split_col(sub[['site_path_timestamp','floor','x','y']]).copy()\n    sub = sub.merge(train_waypoints[['site','floorNo','floor']].drop_duplicates(), how='left')\n    sub = sub.merge(\n        train_waypoints[['x','y','site','floor','isTrainWaypoint']].drop_duplicates(),\n        how='left',\n        on=['site','x','y','floor']\n             )\n    sub['isTrainWaypoint'] = sub['isTrainWaypoint'].fillna(False)\n    return sub.copy()","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.143941,"end_time":"2021-05-12T05:04:48.052295","exception":false,"start_time":"2021-05-12T05:04:47.908354","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Visualize the raw prediction","metadata":{"papermill":{"duration":0.113248,"end_time":"2021-05-12T05:04:48.282803","exception":false,"start_time":"2021-05-12T05:04:48.169555","status":"completed"},"tags":[]}},{"cell_type":"code","source":"sub = sub_process(ss, train_waypoints)\n\n# Plot the training Data For an example Floor\nexample_site = '5d2709bb03f801723c32852c'\nexample_floorNo = 'F4'\n\nplot_preds(example_site, example_floorNo, sub,\n           train_waypoints, show_preds=True)\nplt.show()","metadata":{"papermill":{"duration":2.083305,"end_time":"2021-05-12T05:04:50.478595","exception":false,"start_time":"2021-05-12T05:04:48.395290","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Saito's post-processing","metadata":{"papermill":{"duration":0.117721,"end_time":"2021-05-12T05:04:50.716619","exception":false,"start_time":"2021-05-12T05:04:50.598898","status":"completed"},"tags":[]}},{"cell_type":"code","source":"ss2 = pd.read_csv('../input/indoorcsv/ILN_594_fold_all_submission_.csv')\nss2[['site', 'path', 'timestamp']] = [i.split('_') for i in ss2.site_path_timestamp]\nss2['timestamp'] = ss2['timestamp'].astype(float)\n\nss = ss.merge(train_waypoints[['site','floor','floorNo']].drop_duplicates())\nss2 = ss2.merge(train_waypoints[['site','floor','floorNo']].drop_duplicates())","metadata":{"papermill":{"duration":0.292624,"end_time":"2021-05-12T05:04:51.126802","exception":false,"start_time":"2021-05-12T05:04:50.834178","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\n\norder = 3\nfs = 50.0  # sample rate, Hz\n# fs = 100\n# cutoff = 3.667  # desired cutoff frequency of the filter, Hz\ncutoff = 3\n\nstep_distance = 0.8\nw_height = 1.7\nm_trans = -5\n\nfrom scipy.signal import butter, lfilter\n\ndef butter_lowpass(cutoff, fs, order=5):\n    nyq = 0.5 * fs\n    normal_cutoff = cutoff / nyq\n    b, a = butter(order, normal_cutoff, btype='low', analog=False)\n    return b, a\n\ndef butter_lowpass_filter(data, cutoff, fs, order=5):\n    b, a = butter_lowpass(cutoff, fs, order=order)\n    y = lfilter(b, a, data)\n    return y\n\ndef peak_accel_threshold(data, timestamps, threshold):\n    d_acc = []\n    last_state = 'below'\n    crest_troughs = 0\n    crossings = []\n\n    for i, datum in enumerate(data):\n        \n        current_state = last_state\n        if datum < threshold:\n            current_state = 'below'\n        elif datum > threshold:\n            current_state = 'above'\n\n        if current_state is not last_state:\n            if current_state is 'above':\n                crossing = [timestamps[i], threshold]\n                crossings.append(crossing)\n            else:\n                crossing = [timestamps[i], threshold]\n                crossings.append(crossing)\n\n            crest_troughs += 1\n        last_state = current_state\n    return np.array(crossings)\n\ndef steps_compute_rel_positions(sample_file):\n    \n    mix_acce = np.sqrt(sample_file.acce[:,1:2]**2 + sample_file.acce[:,2:3]**2 + sample_file.acce[:,3:4]**2)\n    mix_acce = np.concatenate([sample_file.acce[:,0:1], mix_acce], 1)\n    mix_df = pd.DataFrame(mix_acce)\n    mix_df.columns = [\"timestamp\",\"acce\"]\n    \n    filtered = butter_lowpass_filter(mix_df[\"acce\"], cutoff, fs, order)\n\n    threshold = filtered.mean() * 1.1\n    crossings = peak_accel_threshold(filtered, mix_df[\"timestamp\"], threshold)\n\n    step_sum = len(crossings)/2\n    distance = w_height * 0.4 * step_sum\n\n    mag_df = pd.DataFrame(sample_file.magn)\n    mag_df.columns = [\"timestamp\",\"x\",\"y\",\"z\"]\n    \n    acce_df = pd.DataFrame(sample_file.acce)\n    acce_df.columns = [\"timestamp\",\"ax\",\"ay\",\"az\"]\n    \n    mag_df = pd.merge(mag_df,acce_df,on=\"timestamp\")\n    mag_df.dropna()\n    \n    time_di_list = []\n\n    for i in mag_df.iterrows():\n\n        gx,gy,gz = i[1][1],i[1][2],i[1][3]\n        ax,ay,az = i[1][4],i[1][5],i[1][6]\n\n        roll = math.atan2(ay,az)\n        pitch = math.atan2(-1*ax , (ay * math.sin(roll) + az * math.cos(roll)))\n\n        q = m_trans - math.degrees(math.atan2(\n            (gz*math.sin(roll)-gy*math.cos(roll)),(gx*math.cos(pitch) + gy*math.sin(roll)*math.sin(pitch) + gz*math.sin(pitch)*math.cos(roll))\n        )) -90\n        if q <= 0:\n            q += 360\n        time_di_list.append((i[1][0],q))\n\n    d_list = [x[1] for x in time_di_list]\n    \n    steps = []\n    step_time = []\n    di_dict = dict(time_di_list)\n\n    for n,i in enumerate(crossings[:,:1]):\n        if n % 2 == 1:\n            continue\n        direct_now = di_dict[i[0]]\n        dx = math.sin(math.radians(direct_now))\n        dy = math.cos(math.radians(direct_now))\n#         print(int(n/2+1),\"歩目/x:\",dx,\"/y:\",dy,\"/角度：\",direct_now)\n        steps.append((i[0],dx,dy))\n        step_time.append(i[0])\n    \n        step_dtime = np.diff(step_time)/1000\n        step_dtime = step_dtime.tolist()\n        step_dtime.insert(0,5)\n        \n        rel_position = []\n\n        wp_idx = 0\n#         print(\"WP:\",round(sample_file.waypoint[0,1],3),round(sample_file.waypoint[0,2],3),sample_file.waypoint[0,0])\n#         print(\"------------------\")\n        for p,i in enumerate(steps):\n            step_distance = 0\n            if step_dtime[p] >= 1:\n                step_distance = w_height*0.25\n            elif step_dtime[p] >= 0.75:\n                step_distance = w_height*0.3\n            elif step_dtime[p] >= 0.5:\n                step_distance = w_height*0.4\n            elif step_dtime[p] >= 0.35:\n                step_distance = w_height*0.45\n            elif step_dtime[p] >= 0.2:\n                step_distance = w_height*0.5\n            else:\n                step_distance = w_height*0.4\n\n#             step_x += i[1]*step_distance\n#             step_y += i[2]*step_distance\n            \n            rel_position.append([i[0], i[1]*step_distance, i[2]*step_distance])\n#     print(rel_position)\n    \n    return np.array(rel_position)","metadata":{"papermill":{"duration":0.151659,"end_time":"2021-05-12T05:04:51.397869","exception":false,"start_time":"2021-05-12T05:04:51.246210","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compute_rel_positions(acce_datas, ahrs_datas):\n    step_timestamps, step_indexs, step_acce_max_mins = compute_f.compute_steps(acce_datas)\n    headings = compute_f.compute_headings(ahrs_datas)\n    stride_lengths = compute_f.compute_stride_length(step_acce_max_mins)\n    step_headings = compute_f.compute_step_heading(step_timestamps, headings)\n    rel_positions = compute_f.compute_rel_positions(stride_lengths, step_headings)\n    return rel_positions\n\ndef calc_points(x1_realnum, y1_realnum, x2_realnum, y2_realnum):\n    \n    x1 = math.floor(x1_realnum)\n    y1 = math.floor(y1_realnum)\n    x2 = math.floor(x2_realnum)\n    y2 = math.floor(y2_realnum)\n    \n    \n    points = []\n    \n    if (x1 == x2) and (y1 == y2):\n        points.append([x1_realnum,y1_realnum])\n        points.append([x2_realnum,y2_realnum])\n        return np.array(points)\n\n    x1, y1 = x1_realnum, y1_realnum\n    x2, y2 = x2_realnum, y2_realnum\n    \n    step_x = np.sign(x2 - x1)\n    step_y = np.sign(y2 - y1)\n    a = (y2-y1)/(x2-x1)\n    dx = 1/((a**2+1)**0.5)\n    dy = a/((a**2+1)**0.5)\n    n = math.floor(((y2-y1)**2 + (x2-x1)**2)**0.5)\n    \n    x = x1_realnum\n    y = y1_realnum\n    points.append([x1_realnum, y1_realnum])\n    for i in range(n-1):\n        x = x + step_x * dx\n        y = y + step_y * dy\n        points.append([x, y])\n    points.append([x2_realnum, y2_realnum])\n    \n    return np.array(points)\n\ndef f_coarse(v,xy_hat_,dist): \n    \n    length = int(len(xy_hat_)/2)\n    \n    x = xy_hat_[0::2]\n    y = xy_hat_[1::2]\n    \n    cost = 0.0\n    x_ = math.cos(v[2]) * x - math.sin(v[2]) * y + v[0]\n    y_ = math.sin(v[2]) * x + math.cos(v[2]) * y + v[1]\n    cost_gamma = 0\n    for i in range(length-1):\n        #points = calc_points(math.floor(x[i]),math.floor(y[i]),math.floor(x[i+1]),math.floor(y[i+1]))\n        points = calc_points(x_[i],y_[i],x_[i+1],y_[i+1])\n        cost_point = 0\n        for point in points:\n            gamma = 1000000000\n            if point[0] < 0 or point[1] < 0 or point[0] > dist.shape[1] or point[1] > dist.shape[0]:\n                cost_point += gamma\n            else:\n                try:\n                    px = point[0]\n                    py = point[1]\n                    pxf = math.floor(px)\n                    pyf = math.floor(py)\n                    dist_bilinear = (pxf+1-px)*((pyf+1-py)*dist[pyf][pxf]+(py-pyf)*dist[pyf+1][pxf])+(px-pxf)*((pyf+1-py)*dist[pyf][pxf+1]+(py-pyf)*dist[pyf+1][pxf+1])\n                    #cost_gamma += gamma*(dist[point[1]][point[0]])\n                    cost_point += gamma*dist_bilinear\n                    #print(px,py,dist_bilinear)\n                except:\n                    cost_point += gamma\n        cost_gamma += cost_point/len(points)\n    \n    cost_alpha = sum((x-x_)**2+(y-y_)**2)\n    \n    cost = cost_alpha + cost_gamma\n    \n    return cost\n\ndef f_with_(v,path_df, alpha_, beta_, xy_hat_, delta_xy_hat_, dist, delta_t):\n    length = int(len(v)/2)-1\n    \n    dx,dy = v[-2], v[-1]\n    x = v[0:-2:2] + dx\n    y = v[1:-1:2] + dy\n    v = v[0:-2]\n    \n    #x = v[0::2]\n    #y = v[1::2]\n    delta_x = delta_xy_hat_[0::2]\n    delta_y = delta_xy_hat_[1::2]\n    \n    cost_alpha = sum(alpha_ * (v - xy_hat_)**2)\n    cost_beta = sum(beta_ * (v[2:2*length] - v[0:2*length-2] - delta_xy_hat_)**2)\n    \n    cost_gamma = 0\n    for i in range(length-1):\n        points = calc_points(x[i],y[i],x[i+1],y[i+1])\n        for j, point in enumerate(points):\n            gamma = 100*3\n            if j==0 or j==len(points)-1:\n                start_end_coeff = 100\n            else:\n                start_end_coeff = 1\n            \n            if point[0] < 0 or point[1] < 0 or point[0] > dist.shape[1] or point[1] > dist.shape[0]:\n                cost_gamma += gamma * start_end_coeff\n            else:\n                try:\n                    px = point[0]\n                    py = point[1]\n                    pxf = math.floor(px)\n                    pyf = math.floor(py)\n                    dist_bilinear = (pxf+1-px)*((pyf+1-py)*dist[pyf][pxf]+(py-pyf)*dist[pyf+1][pxf])+(px-pxf)*((pyf+1-py)*dist[pyf][pxf+1]+(py-pyf)*dist[pyf+1][pxf+1])\n                    #cost_gamma += gamma*(dist[point[1]][point[0]])**2\n                    cost_gamma += start_end_coeff * gamma*dist_bilinear**2\n                    #print(px,py,dist_bilinear)\n                except:\n                    cost_gamma += gamma * start_end_coeff\n        \n    return cost_alpha + cost_beta + cost_gamma","metadata":{"papermill":{"duration":0.154581,"end_time":"2021-05-12T05:04:51.670885","exception":false,"start_time":"2021-05-12T05:04:51.516304","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def correct_path(args):\n    path, path_df = args\n    \n    site = path_df['site'].iloc[0]\n    floorNo = path_df['floorNo'].iloc[0]\n    \n    initial_v = np.ravel(path_df[['x', 'y']].values)\n    \n    T_ref  = path_df['timestamp'].values\n    xy_hat = path_df[['x', 'y']].values\n    \n    example = read_data_file(f'../input/indoor-location-navigation/test/{path}.txt')\n    \n    rel_positions1 = compute_rel_positions(example.acce, example.ahrs)\n    rel_positions2 = steps_compute_rel_positions(example)\n    rel1 = rel_positions1.copy()\n    rel2 = rel_positions2.copy()\n    rel1[:,1:] = rel_positions1[:,1:] / 2\n    rel2[:,1:] = rel_positions2[:,1:] / 2\n    rel_positions = np.vstack([rel1,rel2])\n    rel_positions = rel_positions[np.argsort(rel_positions[:, 0])]\n    \n    if T_ref[-1] > rel_positions[-1, 0]:\n        rel_positions = [np.array([[0, 0, 0]]), rel_positions, np.array([[T_ref[-1], 0, 0]])]\n    else:\n        rel_positions = [np.array([[0, 0, 0]]), rel_positions]\n    rel_positions = np.concatenate(rel_positions)\n    \n    T_rel = rel_positions[:, 0]\n    delta_xy_hat = np.diff(scipy.interpolate.interp1d(T_rel, np.cumsum(rel_positions[:, 1:3], axis=0), axis=0)(T_ref), axis=0)\n\n    N = xy_hat.shape[0]\n    delta_t = np.diff(T_ref)\n    alpha = (7.2)**(-2) * np.ones(N)\n    beta  = (0.3 + 0.3 * 1e-3 * delta_t)**(-2)\n    \n    alpha_ = np.ravel(np.stack([alpha, alpha],1))\n    beta_ = np.ravel(np.stack([beta, beta],1))\n    \n    xy_hat_ = np.ravel(xy_hat)\n    delta_xy_hat_ = np.ravel(delta_xy_hat)\n    \n    dist = plt.imread(f'../input/indoorlocationnavigationbwdist/metadata/{site}/{floorNo}/geojson_map_cv_dist_x16.png')\n    \n    x = xy_hat_[0::2]\n    y = xy_hat_[1::2]\n    \n    x2 = ss2[ss2['path'] == path]['x']\n    y2 = ss2[ss2['path'] == path]['y']\n    initial_v[0::2] = x2\n    initial_v[1::2] = y2\n    \n    initial_v = np.append(initial_v, [0, 0])\n    \n    v_with_ = minimize(f_with_, x0 = initial_v * 16, args = (path_df, alpha_, beta_, xy_hat_ * 16, delta_xy_hat_ * 16, dist, delta_t), \n                       method = \"cobyla\", options = {'rhobeg': 1.0, 'maxiter': 10000, 'disp': False, 'catol': 0.0002})\n    \n    cost = f_with_(v_with_['x'] * 16, path_df, alpha_, beta_, xy_hat_ * 16, delta_xy_hat_ * 16, dist, delta_t)\n    \n    dx, dy = v_with_['x'][-2], v_with_['x'][-1]\n    x_with_ = v_with_['x'][0:-2:2] + dx\n    y_with_ = v_with_['x'][1:-1:2] + dy\n\n    return pd.DataFrame({\n        'site_path_timestamp' : path_df['site_path_timestamp'],\n        'floor' : path_df['floor'],\n        'x' : x_with_/16,\n        'y' : y_with_/16,\n    })","metadata":{"papermill":{"duration":0.144213,"end_time":"2021-05-12T05:04:51.935225","exception":false,"start_time":"2021-05-12T05:04:51.791012","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"processes = multiprocessing.cpu_count()\nwith multiprocessing.Pool(processes = processes) as pool:\n    dfs = pool.imap_unordered(correct_path, ss.groupby('path'))\n    dfs = tqdm(dfs)\n    dfs = list(dfs)\nss = pd.concat(dfs).sort_values('site_path_timestamp')","metadata":{"papermill":{"duration":29889.678778,"end_time":"2021-05-12T13:23:01.734425","exception":false,"start_time":"2021-05-12T05:04:52.055647","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss[['site_path_timestamp', 'floor', 'x', 'y']].to_csv('submission_after_Darich.csv', index = None)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Visualize post-processing after Saito's post processing","metadata":{"papermill":{"duration":0.120385,"end_time":"2021-05-12T13:23:01.978898","exception":false,"start_time":"2021-05-12T13:23:01.858513","status":"completed"},"tags":[]}},{"cell_type":"code","source":"sub = sub_process(ss, train_waypoints)\n\n# Plot the training Data For an example Floor\nexample_site = '5d2709bb03f801723c32852c'\nexample_floorNo = 'F4'\n\nplot_preds(example_site, example_floorNo, sub,\n           train_waypoints, show_preds = True)\nplt.show()","metadata":{"papermill":{"duration":2.269382,"end_time":"2021-05-12T13:23:04.370379","exception":false,"start_time":"2021-05-12T13:23:02.100997","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Push to corridor post-processing","metadata":{"papermill":{"duration":0.133454,"end_time":"2021-05-12T13:23:04.636912","exception":false,"start_time":"2021-05-12T13:23:04.503458","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from shapely.geometry import Polygon\nfrom shapely.ops import nearest_points\nfrom shapely.geometry import Point\n\ndef fix_prediction(args):\n    # Unpack\n    (site, floor), df = args\n    \n    # Find the file path\n    floor_name = os.listdir('../input/indoor-location-navigation-scaled-geojson/scaled_geojson/' + site)\n    for name in floor_name:\n        if floor_map[name] == floor:\n            file = '../input/indoor-location-navigation-scaled-geojson/scaled_geojson/' + site + '/' + name + '/shapely_geometry.pkl'\n            break\n            \n    # Open the corridor\n    with open(file, 'rb') as f:\n        corridor = pkl.load(f)\n        \n    # Find the outside-corridor points and force them into the corridor\n    out_corridor = []\n    out_corridor_idx = []\n    corridor_nearest_points = []\n    for i in range(df.shape[0]):\n        p = Point(df[['x', 'y']].iloc[i].values)\n        if not p.within(corridor):\n            out_corridor.append(p)\n            out_corridor_idx.append(df[['x', 'y']].index[i])\n            nearest_p, _ = nearest_points(corridor, p)\n            x, y = nearest_p.xy[0][0], nearest_p.xy[1][0]\n            corridor_nearest_points.append([x, y])\n    \n    if len(corridor_nearest_points) != 0:\n        df.loc[out_corridor_idx, ['x', 'y']] = np.array(corridor_nearest_points)\n    \n    return df","metadata":{"papermill":{"duration":0.449158,"end_time":"2021-05-12T13:23:05.218193","exception":false,"start_time":"2021-05-12T13:23:04.769035","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss[['site', 'path', 'timestamp']] = np.array([i.split('_') for i in ss.site_path_timestamp])\n\nprocesses = multiprocessing.cpu_count()\nwith multiprocessing.Pool(processes = processes) as pool:\n    dfs = pool.imap_unordered(fix_prediction, ss.groupby(['site', 'floor']))\n    dfs = tqdm(dfs)\n    dfs = list(dfs)\nss = pd.concat(dfs).sort_values('site_path_timestamp')[['site_path_timestamp', 'floor', 'x', 'y']]","metadata":{"papermill":{"duration":8.172792,"end_time":"2021-05-12T13:23:13.519644","exception":false,"start_time":"2021-05-12T13:23:05.346852","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss[['site_path_timestamp', 'floor', 'x', 'y']].to_csv('submission_after_push.csv', index = None)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Visualize after Push-to-Corridor post-processing","metadata":{"papermill":{"duration":0.12834,"end_time":"2021-05-12T13:23:13.775906","exception":false,"start_time":"2021-05-12T13:23:13.647566","status":"completed"},"tags":[]}},{"cell_type":"code","source":"sub = sub_process(ss, train_waypoints)\n\n# Plot the training Data For an example Floor\nexample_site = '5d2709bb03f801723c32852c'\nexample_floorNo = 'F4'\n\nplot_preds(example_site, example_floorNo, sub, \n           train_waypoints, show_preds = True)\nplt.show()","metadata":{"papermill":{"duration":1.858597,"end_time":"2021-05-12T13:23:15.764307","exception":false,"start_time":"2021-05-12T13:23:13.905710","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# \"Snap to Grid\" Post Processing","metadata":{"papermill":{"duration":0.137596,"end_time":"2021-05-12T13:23:16.038215","exception":false,"start_time":"2021-05-12T13:23:15.900619","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def add_xy_site(df):\n    df['xy'] = [(x, y) for x,y in zip(df['x'], df['y'])]\n    try:\n        df['site'] = [i.split('_')[0] for i in df.site_path_timestamp]\n    except:\n        pass\n    return df\n\ndef closest_point(point, points):\n    \"\"\" Find closest point from a list of points. \"\"\"\n    return points[cdist([point], points).argmin()]\n\nsub = add_xy_site(ss)\ntrain_waypoints = add_xy_site(train_waypoints)\n\nds = []\nfor (site, myfloor), d in tqdm(sub.groupby(['site','floor'])):\n    true_floor_locs = train_waypoints.loc[(train_waypoints['floor'] == myfloor) &\n                                          (train_waypoints['site'] == site)].reset_index(drop=True)\n    if len(true_floor_locs) == 0:\n        print(f'Skipping {site} {myfloor}')\n        continue\n    d['matched_point'] = [closest_point(x, list(true_floor_locs['xy'])) for x in d['xy']]\n    d['x_'] = d['matched_point'].apply(lambda x: x[0])\n    d['y_'] = d['matched_point'].apply(lambda x: x[1])\n    ds.append(d)\n\nsub = pd.concat(ds)","metadata":{"papermill":{"duration":13.024869,"end_time":"2021-05-12T13:23:29.200824","exception":false,"start_time":"2021-05-12T13:23:16.175955","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def snap_to_grid(sub, threshold):\n    \"\"\"\n    Snap to grid if within a threshold.\n    \n    x, y are the predicted points.\n    x_, y_ are the closest grid points.\n    _x_, _y_ are the new predictions after post processing.\n    \"\"\"\n    sub['_x_'] = sub['x']\n    sub['_y_'] = sub['y']\n    sub.loc[sub['dist'] < threshold, '_x_'] = sub.loc[sub['dist'] < threshold]['x_']\n    sub.loc[sub['dist'] < threshold, '_y_'] = sub.loc[sub['dist'] < threshold]['y_']\n    return sub.copy()\n\n# Calculate the distances\nsub['dist'] = np.sqrt((sub.x-sub.x_)**2 + (sub.y-sub.y_)**2)\n\nss = snap_to_grid(sub, threshold = 4)\n\nss = ss[['site_path_timestamp','floor','_x_','_y_']].rename(columns = {'_x_':'x', '_y_':'y'}).sort_values('site_path_timestamp')","metadata":{"papermill":{"duration":0.17951,"end_time":"2021-05-12T13:23:29.531857","exception":false,"start_time":"2021-05-12T13:23:29.352347","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss[['site_path_timestamp', 'floor', 'x', 'y']].to_csv('submission_after_snap.csv', index = None)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Visualize after Snap-to-Grip post-processing","metadata":{"papermill":{"duration":0.135725,"end_time":"2021-05-12T13:23:29.807787","exception":false,"start_time":"2021-05-12T13:23:29.672062","status":"completed"},"tags":[]}},{"cell_type":"code","source":"sub = sub_process(ss, train_waypoints)\n\n# Plot the training Data For an example Floor\nexample_site = '5d2709bb03f801723c32852c'\nexample_floorNo = 'F4'\n\nplot_preds(example_site, example_floorNo, sub,\n           train_waypoints, show_preds=True)\nplt.show()","metadata":{"papermill":{"duration":1.848686,"end_time":"2021-05-12T13:23:31.792022","exception":false,"start_time":"2021-05-12T13:23:29.943336","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Leakage-based post-processing","metadata":{"papermill":{"duration":0.153781,"end_time":"2021-05-12T13:23:32.091924","exception":false,"start_time":"2021-05-12T13:23:31.938143","status":"completed"},"tags":[]}},{"cell_type":"code","source":"ss['path'] = [i.split('_')[1] for i in ss['site_path_timestamp']]\n\ndef device_based_leak_pp(sub):\n    df_leak = pd.read_pickle('../input/indoor-support-data/df_leak.pkl')\n    df_leak = df_leak.rename({'path_id':'path'}, axis=1)    \n    df_sub = sub.copy()\n    list_path = df_sub[\"path\"].unique()\n    for path in tqdm(list_path):\n        df_sub_path = df_sub.query(\"path == @path\")\n        start_idx = df_sub.loc[df_sub[\"path\"] == path].index.min()\n        end_idx = df_sub.loc[df_sub[\"path\"] == path].index.max()\n        start_x = df_sub_path.at[start_idx,\"x\"]\n        start_y = df_sub_path.at[start_idx,\"y\"]\n        end_x = df_sub_path.at[end_idx,\"x\"]\n        end_y = df_sub_path.at[end_idx,\"y\"]\n        start_x_leak = df_leak.query(\"path == @path\")[\"start_waypoint_x\"].iloc[0]\n        start_y_leak = df_leak.query(\"path == @path\")[\"start_waypoint_y\"].iloc[0]\n        end_x_leak = df_leak.query(\"path == @path\")[\"end_waypoint_x\"].iloc[0]\n        end_y_leak = df_leak.query(\"path == @path\")[\"end_waypoint_y\"].iloc[0]\n        if not np.isnan(start_x_leak):\n            df_sub.at[start_idx,\"x\"] = start_x_leak\n            df_sub.at[start_idx,\"y\"] = start_y_leak\n        if not np.isnan(end_x_leak):\n            df_sub.at[end_idx,\"x\"] = end_x_leak\n            df_sub.at[end_idx,\"y\"] = end_y_leak\n    return df_sub\n\nss = device_based_leak_pp(ss)","metadata":{"papermill":{"duration":12.898987,"end_time":"2021-05-12T13:23:45.151336","exception":false,"start_time":"2021-05-12T13:23:32.252349","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = sub_process(ss, train_waypoints)\n\n# Plot the training Data For an example Floor\nexample_site = '5d2709bb03f801723c32852c'\nexample_floorNo = 'F4'\n\nplot_preds(example_site, example_floorNo, sub,\n           train_waypoints, show_preds=True)\nplt.show()","metadata":{"papermill":{"duration":1.85164,"end_time":"2021-05-12T13:23:47.141990","exception":false,"start_time":"2021-05-12T13:23:45.290350","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{"papermill":{"duration":0.145167,"end_time":"2021-05-12T13:23:47.432599","exception":false,"start_time":"2021-05-12T13:23:47.287432","status":"completed"},"tags":[]}},{"cell_type":"code","source":"ss[['site_path_timestamp', 'floor', 'x', 'y']].to_csv('submission.csv', index = None)","metadata":{"papermill":{"duration":0.227999,"end_time":"2021-05-12T13:23:47.805461","exception":false,"start_time":"2021-05-12T13:23:47.577462","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]}]}