{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This is a pytorch lightning training code that I used and wrote for this competiton.\n\nA lot were inspired from https://www.kaggle.com/luffy521/lstm-by-pytorch-with-unified-wi-fi-feats\n\nThe code will not work right away in Kaggle notebooks (you will have to change details) and I used it in my local machine. \n\nHere is the link to my github repository in case you want to see all of the original code.\n\nhttps://github.com/dongkyuk/Kaggle_Indoor_Loc\n\nThe current scores for this code is 4.965 with postprocessing and 6.74 without postprocessing.\n\nI used this notebook https://www.kaggle.com/higepon/visualize-submissions-with-post-processing for postprocessing.","metadata":{}},{"cell_type":"markdown","source":"Config","metadata":{}},{"cell_type":"code","source":"class Config():\n    DATA_DIR = 'data'\n    SAVE_DIR = 'save'\n    \n    seed = 42\n    epochs = 300\n    num_wifi_feats = 20\n    fold_num = 5\n    train_batch_size = 256\n    val_batch_size = 256\n    num_workers = 16\n    device = 'gpu'\n    lr = 5e-3","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"DataModule","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nfrom pytorch_lightning import LightningDataModule\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.model_selection import StratifiedGroupKFold, StratifiedKFold\nimport numpy as np\n\n\nclass IndoorDataset(Dataset):\n    def __init__(self, data, bssid_feats, rssi_feats, flag='TRAIN'):\n        self.data = data\n        self.flag = flag\n        self.bssid_feats = bssid_feats\n        self.rssi_feats = rssi_feats\n\n    def __len__(self):\n        return self.data.shape[0]\n\n    def __getitem__(self, index):\n        tmp_data = self.data.iloc[index]\n        if self.flag == 'TRAIN':\n            return {\n                'BSSID_FEATS': tmp_data[self.bssid_feats].values.astype(int),\n                'RSSI_FEATS': tmp_data[self.rssi_feats].values.astype(np.float32),\n                'site_id': tmp_data['site_id'].astype(int),\n                'x': tmp_data['x'],\n                'y': tmp_data['y'],\n                'floor': tmp_data['floor'],\n            }\n        elif self.flag == 'TEST':\n            return {\n                'BSSID_FEATS': tmp_data[self.bssid_feats].values.astype(int),\n                'RSSI_FEATS': tmp_data[self.rssi_feats].values.astype(np.float32),\n                'site_id': tmp_data['site_id'].astype(int)\n            }\n\n\nclass IndoorDataModule(LightningDataModule):\n    def __init__(self, train_data, test_data, kfold=False):\n        self.train_data = train_data\n        self.test_data = test_data\n        self.kfold = kfold\n\n    def set_fold_num(self, fold_num):\n        self.fold_num = fold_num\n\n    def _init_feats(self):\n        self.bssid_feats = [f'bssid_{i}' for i in range(Config.num_wifi_feats)]\n        self.rssi_feats = [f'rssi_{i}' for i in range(Config.num_wifi_feats)]\n\n    def _init_wifi_bssids(self):\n        wifi_bssids = []\n        for i in range(100):\n            wifi_bssids += self.train_data[f'bssid_{i}'].values.tolist()\n            wifi_bssids += self.test_data[f'bssid_{i}'].values.tolist()\n\n        self.wifi_bssids = list(set(wifi_bssids))\n        self.wifi_bssids_size = len(self.wifi_bssids)\n\n    def _init_transforms(self):\n        self.wifi_bssids_encoder = LabelEncoder()\n        self.wifi_bssids_encoder.fit(self.wifi_bssids)\n\n        self.site_id_encoder = LabelEncoder()\n        self.site_id_encoder = self.site_id_encoder.fit(\n            self.train_data['site_id'])\n\n        self.rssi_normalizer = StandardScaler()\n        self.rssi_normalizer.fit(self.train_data[self.rssi_feats])\n\n    def _transform(self, data):\n        for bssid_feat in self.bssid_feats:\n            data[bssid_feat] = self.wifi_bssids_encoder.transform(\n                data[bssid_feat])\n        data['site_id'] = self.site_id_encoder.transform(data['site_id'])\n        data[self.rssi_feats] = self.rssi_normalizer.transform(\n            data[self.rssi_feats])\n        return data\n\n    def _kfold(self):\n        ''' Group Kfold wrt path and Stratified Kfold wrt site_id\n        '''\n        skf = StratifiedKFold(n_splits=Config.fold_num,\n                                   shuffle=True, random_state=Config.seed)\n        self.train_data['site_id_f'] = self.train_data['site_id'] + self.train_data['floor'].astype(str)\n        for n, (train_index, val_index) in enumerate(\n            skf.split(\n                X = self.train_data['path'],\n                y = self.train_data['path']\n            )\n        ):\n            self.train_data.loc[val_index, 'kfold'] = int(n)\n\n    def prepare_data(self):\n        # Init cross validation\n        if self.kfold:\n            self._kfold()\n\n        # Init preprocessing\n        self._init_feats()\n        self._init_wifi_bssids()\n        self._init_transforms()\n        self.site_id_dim = len(self.train_data['site_id'].unique())\n        self.train_data = self._transform(self.train_data)\n        self.test_data = self._transform(self.test_data)\n\n    def setup(self, stage=None):\n        # Assign train/val datasets for use in dataloaders\n        if stage == 'fit' or stage is None:\n            if self.kfold:\n                train_df = self.train_data[self.train_data['kfold'] !=\n                                           self.fold_num].reset_index(drop=True)\n                val_df = self.train_data[self.train_data['kfold'] ==\n                                         self.fold_num].reset_index(drop=True)\n            self.train = IndoorDataset(\n                train_df, self.bssid_feats, self.rssi_feats, flag=\"TRAIN\")\n            self.val = IndoorDataset(\n                val_df, self.bssid_feats, self.rssi_feats, flag=\"TRAIN\")\n\n        # Assign test dataset for use in dataloader(s)\n        if stage == 'test' or stage is None:\n            self.test = IndoorDataset(\n                self.test_data, self.bssid_feats, self.rssi_feats, flag=\"TEST\")\n\n    def train_dataloader(self):\n        return DataLoader(self.train, batch_size=Config.train_batch_size, num_workers=Config.num_workers, shuffle=True, pin_memory=True)\n\n    def val_dataloader(self):\n        return DataLoader(self.val, batch_size=Config.val_batch_size, num_workers=Config.num_workers, shuffle=True, pin_memory=True)\n\n    def test_dataloader(self):\n        return DataLoader(self.test, batch_size=Config.val_batch_size, num_workers=Config.num_workers, shuffle=False, pin_memory=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Model (Torch NN Module)","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport torch\nimport torch.nn as nn\n\nclass SeqLSTM(nn.Module):\n    def __init__(self, wifi_num, bssid_dim, site_id_dim, embedding_dim=64):\n        \"\"\"SeqLSTM Model\n        Args:\n            wifi_num (int): number of wifi signals to use\n            bssid_dim (int): total number of unique bssids\n            site_id_dim (int): total number of unique site ids\n            embedding_dim (int): Dimension of bssid embedding. Defaults to 64.\n        \"\"\"\n        super(SeqLSTM, self).__init__()\n        self.wifi_num = wifi_num\n        self.feature_dim = 256\n\n        # Embedding\n        self.embd_bssid = nn.Embedding(bssid_dim, embedding_dim)\n        self.embd_site_id = nn.Embedding(site_id_dim, embedding_dim)\n\n        # Linear\n        self.fc_rssi = nn.Linear(1, embedding_dim)\n        self.fc_features = nn.Linear(embedding_dim * 3, self.feature_dim)\n        self.fc_output = nn.Linear(16, 3)\n\n        # Other\n        self.bn_rssi = nn.BatchNorm1d(embedding_dim)\n        self.bn_features = nn.BatchNorm1d(self.feature_dim)\n        self.dropout = nn.Dropout(0.3),\n\n        self.lstm1 = nn.LSTM(input_size=256, hidden_size=128,\n                             dropout=0.3, bidirectional=False)\n        self.lstm2 = nn.LSTM(input_size=128, hidden_size=16,\n                             dropout=0.1, bidirectional=False)\n\n    def forward(self, x):\n        embd_bssid = self.embd_bssid(x['BSSID_FEATS'])  # (,wifi_num,embedding_dim)\n\n        embd_site_id = self.embd_site_id(x['site_id'])  # (,embedding_dim)\n        embd_site_id = torch.unsqueeze(embd_site_id, dim=1)  # (,1,embedding_dim)\n        embd_site_id = embd_site_id.repeat(\n            1, self.wifi_num, 1)  # (,wifi_num,embedding_dim)\n\n        rssi_feat = x['RSSI_FEATS']  # (,wifi_num)\n        rssi_feat = torch.unsqueeze(rssi_feat, dim=-1)   # (,wifi_num,1)\n        rssi_feat = self.fc_rssi(rssi_feat)              # (,wifi_num,embedding_dim)\n        rssi_feat = self.bn_rssi(rssi_feat.transpose(1, 2)).transpose(1, 2)\n        rssi_feat = torch.relu(rssi_feat)\n\n        x = torch.cat([embd_bssid, embd_site_id, rssi_feat],\n                      dim=-1)  # (,wifi_num,embedding_dim*3)\n\n        x = self.fc_features(x)  # (,wifi_num, feature_dim)\n        x = self.bn_features(x.transpose(1, 2)).transpose(1, 2)\n        x = torch.relu(x)\n\n        x = torch.transpose(x, 0, 1)  # (wifi_num,,128)\n        x, _ = self.lstm1(x)\n\n        x = x[-1] # (256,16)\n        x = torch.relu(x)\n\n        output = self.fc_output(x).squeeze()  # (,3)\n\n        return output","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"LightningModule ","metadata":{}},{"cell_type":"code","source":"import torch.nn as nn\nimport torch\nimport numpy as np\nfrom pytorch_lightning import LightningModule\nfrom pytorch_lightning.metrics import Accuracy\nfrom torch import optim\n\ndef xy_loss(xy_hat, xy_label):\n    xy_loss = torch.mean(torch.sqrt(\n        (xy_hat[:, 0]-xy_label[:, 0])**2 + (xy_hat[:, 1]-xy_label[:, 1])**2))\n    return xy_loss\n\n\ndef floor_loss(floor_hat, floor_label):\n    floor_loss = 15 * torch.mean(torch.abs(floor_hat-floor_label))\n    return floor_loss\n\n\nclass IndoorLocModel(LightningModule):\n    def __init__(self, model: nn.Module):\n        super().__init__()\n        self.model = model\n        self.lr = Config().lr\n\n        self.critertion_xy = xy_loss\n        self.criterion_floor = floor_loss\n\n    def forward(self, x):\n        x = self.model(x)\n        return x\n\n    def training_step(self, batch, batch_nb):\n        x, y, f = batch['x'].unsqueeze(\n            -1), batch['y'].unsqueeze(-1), batch['floor']\n\n        xy_label = torch.cat([x, y], dim=-1)\n\n        output = self(batch)\n        xy_hat = output[:, 0:2]\n        f_hat = output[:, 2]\n\n        loss_xy = self.critertion_xy(xy_hat, xy_label)\n        loss_floor = self.criterion_floor(f_hat, f)\n        loss = loss_xy + loss_floor\n\n        return {'loss': loss, 'loss_xy': loss_xy, 'loss_floor': loss_floor, 'xy_label': xy_label, 'xy_hat': xy_hat, 'floor_hat': f_hat, 'f': f}\n\n    def training_epoch_end(self, outputs):\n        loss_xy = torch.mean(torch.stack(\n            [output['loss_xy'] for output in outputs], dim=0))\n        loss_floor = torch.mean(torch.stack(\n            [output['loss_floor'] for output in outputs], dim=0))\n        loss = torch.mean(torch.stack([output['loss']\n                          for output in outputs], dim=0))\n\n    def validation_step(self, batch, batch_nb):\n        x, y, f = batch['x'].unsqueeze(\n            -1), batch['y'].unsqueeze(-1), batch['floor']\n\n        xy_label = torch.cat([x, y], dim=-1)\n\n        output = self(batch)\n        xy_hat = output[:, 0:2]\n        f_hat = output[:, 2]\n\n        return {'xy_label': xy_label, 'xy_hat': xy_hat, 'f_hat': f_hat, 'f': f}\n\n    def validation_epoch_end(self, outputs):\n        xy_label = torch.cat([output['xy_label'] for output in outputs], dim=0)\n        xy_hat = torch.cat([output['xy_hat'] for output in outputs], dim=0)\n        f_hat = torch.cat([output['f_hat']\n                           for output in outputs], dim=0)\n        f_hat = torch.squeeze(f_hat)\n        f = torch.cat([output['f'] for output in outputs], dim=0)\n\n        loss_xy = self.critertion_xy(xy_hat, xy_label)\n        loss_floor = self.criterion_floor(f_hat, f)\n        loss = loss_xy + loss_floor\n\n        self.log('val_loss', loss, prog_bar=True)\n        self.log('val_metric', loss, prog_bar=True)\n\n    def configure_optimizers(self):\n        optimizer = optim.Adam(self.model.parameters(), lr=self.lr)\n        scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=5)\n\n        return {'optimizer': optimizer, 'lr_scheduler': scheduler, 'monitor': 'val_loss', }","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport torch\nimport os\nimport logging\nimport warnings\nfrom pytorch_lightning.utilities.seed import seed_everything\nfrom pytorch_lightning import Trainer\nfrom pytorch_lightning.callbacks import ModelCheckpoint\nfrom pytorch_lightning.callbacks.early_stopping import EarlyStopping\n\ndef init_config(seed=42):\n    logging.basicConfig(level=logging.INFO)\n    warnings.filterwarnings(\"ignore\")\n    seed_everything(seed)\n\ndef load_data():\n    # Load data\n    train_data_dir = os.path.join(Config.DATA_DIR, 'train_all.csv')\n    test_data_dir = os.path.join(Config.DATA_DIR, 'test_all.csv')\n\n    train_data = pd.read_csv(train_data_dir)\n    test_data = pd.read_csv(test_data_dir)\n\n    # Init datamodule\n    idm = IndoorDataModule(train_data, test_data, kfold=True)\n    idm.prepare_data()\n    return idm\n\n\ndef train_model(idm: IndoorDataModule, fold: int):\n    # Set fold\n    idm.set_fold_num(fold)\n    idm.setup()\n    \n    # Init model\n    model = IndoorLocModel(SeqLSTM(\n        Config.num_wifi_feats, idm.wifi_bssids_size, idm.site_id_dim))\n\n    # Init callback\n    checkpoint_callback = ModelCheckpoint(\n        monitor='val_loss',\n        dirpath=os.path.join(Config.SAVE_DIR, f'{fold}'),\n        filename='{epoch:02d}-{val_loss:.2f}-{val_metric:.2f}.pth',\n        save_top_k=5,\n        mode='min',\n    )\n    early_stopping = EarlyStopping(\n        monitor='val_loss',\n        mode='min',\n        patience=10,\n    )\n\n    # Init trainer\n    trainer = Trainer(\n        gpus=1,\n        num_sanity_val_steps=-1,\n        deterministic=True,\n        max_epochs=Config.epochs,\n        callbacks=[checkpoint_callback, early_stopping],\n        # profiler=\"simple\",\n    )\n    # trainer.tune(model, idm)\n\n    # Train\n    trainer.fit(model, idm)\n\n\ndef main():\n    init_config()\n\n    idm = load_data()\n\n    for fold in range(Config.fold_num):\n        train_model(idm, fold)\n\n\nif __name__ == \"__main__\":\n    main()","metadata":{},"execution_count":null,"outputs":[]}]}