{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport sys\nimport glob\nimport time\nimport pickle\nimport random\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom collections import defaultdict, OrderedDict\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import KFold, StratifiedKFold, GroupKFold\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.preprocessing import MinMaxScaler\n\nimport torch\nimport torch.nn as nn\nfrom torch.autograd import Variable\nfrom torch.utils.data import DataLoader\nfrom torch.nn import CrossEntropyLoss, MSELoss\nfrom torch.nn.modules.loss import _WeightedLoss\nimport torch.nn.functional as F\n\nNFOLDS = 5\nBATCH_SIZE = 128\nLR = 1e-3\nEPOCHS = 50\nEARLYSTOP_NUM = 3\nSCHEDULE_DECAY = 2\n\nDEBUG = False\nif DEBUG:\n    NFOLDS = 2\n    EPOCHS = 2\n\nfeature_dir = \"../input/indoor-navigation-and-location-wifi-features\"\ntrain_files = sorted(glob.glob(os.path.join(feature_dir, '*_train.csv')))\nsubm = pd.read_csv('../input/indoor-location-navigation/sample_submission.csv')\n# print('subm:\\n', subm.head())\n\nsave_path = './'\nif not os.path.exists(save_path):\n    os.mkdir(save_path)\nif DEBUG:\n    save_path = f'{save_path}/debug'\n    if not os.path.exists(save_path):\n        os.mkdir(save_path)\n\n    train_files = train_files[:3]\n\ndef seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nclass AverageMeter:\n    \"\"\"\n    Computes and stores the average and current value\n    \"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\nclass EarlyStopping:\n    def __init__(self, patience=7, mode=\"max\", delta=0.001):\n        self.patience = patience\n        self.counter = 0\n        self.mode = mode\n        self.best_score = None\n        self.early_stop = False\n        self.delta = delta\n        if self.mode == \"min\":\n            self.val_score = np.Inf\n        else:\n            self.val_score = -np.Inf\n\n    def __call__(self, epoch_score, model, model_path):\n\n        if self.mode == \"min\":\n            score = -1.0 * epoch_score\n        else:\n            score = np.copy(epoch_score)\n\n        if self.best_score is None:\n            self.best_score = score\n            self.save_checkpoint(epoch_score, model, model_path)\n        elif score < self.best_score: #  + self.delta\n            self.counter += 1\n            # print('EarlyStopping counter: {} out of {}'.format(self.counter, self.patience))\n            if self.counter >= self.patience:\n                self.early_stop = True\n        else:\n            self.best_score = score\n            # ema.apply_shadow()\n            self.save_checkpoint(epoch_score, model, model_path)\n            # ema.restore()\n            self.counter = 0\n\n    def save_checkpoint(self, epoch_score, model, model_path):\n        if epoch_score not in [-np.inf, np.inf, -np.nan, np.nan]:\n            # print('Validation score improved ({} --> {}). Saving model!'.format(self.val_score, epoch_score))\n            # if not DEBUG:\n            torch.save(model.state_dict(), model_path)\n        self.val_score = epoch_score\n\n##### Model Fn\nclass IndoorDataset:\n    def __init__(self, feat, x, y, f):\n        self.feat = feat\n        self.label_x = x.reshape(-1, 1)\n        self.label_y = y.reshape(-1, 1)\n        self.label_f = f.reshape(-1, 1)\n\n    def __len__(self):\n        return len(self.feat)\n\n    def __getitem__(self, idx):\n        return {\n            'feat': torch.tensor(self.feat[idx], dtype=torch.float),\n\n            'label_x': torch.tensor(self.label_x[idx], dtype=torch.float),\n            'label_y': torch.tensor(self.label_y[idx], dtype=torch.float),\n            'label_f': torch.tensor(self.label_f[idx], dtype=torch.float),\n        }\n\nclass MeanPositionLoss(nn.Module):\n    def __init__(self):\n        super(MeanPositionLoss, self).__init__()\n    def forward(self, output_way_x, output_way_y, output_floor, way_x, way_y, floor):\n        diff_x = output_way_x - way_x\n        diff_y = output_way_y - way_y\n        diff_f = output_floor - floor\n\n        error = torch.sqrt(diff_x * diff_x + diff_y * diff_y) + 15 * torch.sqrt(diff_f * diff_f)\n        return torch.mean(error)\n\ndef MeanPositionScore(pred_x, pred_y, pred_f, label_x, label_y, label_f):\n    diff_f = pred_f - label_f\n    diff_x = pred_x - label_x\n    diff_y = pred_y - label_y\n    error = np.sqrt(diff_x * diff_x + diff_y * diff_y) + 15 * np.sqrt(diff_f * diff_f)\n    return np.mean(error)\n\ndef train_fn(data_loader, model, optimizer, device, scheduler=None):\n    model.train()\n    losses = AverageMeter()\n\n    # tk0 = tqdm(data_loader, total=len(data_loader), desc=\"Train\")\n    for bi, d in enumerate(data_loader):\n        torch.cuda.empty_cache()\n        feat = d['feat']\n\n        label_x = d[\"label_x\"]\n        label_y = d['label_y']\n        label_f = d['label_f']\n\n        feat = feat.to(device, dtype=torch.float)\n\n        label_x = label_x.to(device, dtype=torch.float)\n        label_y = label_y.to(device, dtype=torch.float)\n        label_f = label_f.to(device, dtype=torch.float)\n\n        optimizer.zero_grad()\n        output_x, output_y, output_f = model(\n            x=feat,\n        )\n        torch.cuda.empty_cache()\n        loss = loss_fn(output_x, output_y, output_f, label_x, label_y, label_f)\n        torch.cuda.empty_cache()\n        loss.backward()\n        optimizer.step()\n        # ema.update()\n        if scheduler:\n            scheduler.step()\n\n        torch.cuda.empty_cache()\n        losses.update(loss.item(), feat.size(0))\n        # tk0.set_postfix(loss=losses.avg)\n\ndef infer_fn(data_loader, model, device):\n    model.eval()\n    pred_x, pred_y, pred_f = [], [], []\n\n    # tk0 = tqdm(data_loader, total=len(data_loader), desc=\"Infer\")\n    with torch.no_grad():\n        for bi, d in enumerate(data_loader):\n            feat = d['feat']\n\n            feat = feat.to(device, dtype=torch.float)\n\n            output_x, output_y, output_f = model(\n                x=feat,\n            )\n\n            pred_x.extend(output_x.cpu().detach().numpy())\n            pred_y.extend(output_y.cpu().detach().numpy())\n            pred_f.extend(output_f.cpu().detach().numpy())\n    # print('pred_f: ', pred_f.shape)\n\n    return np.concatenate(pred_x), np.concatenate(pred_y), np.concatenate(pred_f)\n\nclass Model(nn.Module):\n    def __init__(self, input_dim):\n        super(Model, self).__init__()\n        self.batch_norm0 = nn.BatchNorm1d(input_dim)\n        self.dropout0 = nn.Dropout(0.2)\n\n        dropout_rate = 0.2\n        hidden_size = 256\n        self.dense1 = nn.Linear(input_dim, hidden_size)\n        self.batch_norm1 = nn.BatchNorm1d(hidden_size)\n        self.dropout1 = nn.Dropout(dropout_rate)\n\n        self.dense2 = nn.Linear(hidden_size, hidden_size)\n        self.batch_norm2 = nn.BatchNorm1d(hidden_size)\n        self.dropout2 = nn.Dropout(dropout_rate)\n\n        self.dense3 = nn.Linear(hidden_size, hidden_size)\n        self.batch_norm3 = nn.BatchNorm1d(hidden_size)\n        self.dropout3 = nn.Dropout(dropout_rate)\n\n        self.regressioner = nn.Linear(hidden_size, 3)\n\n        self.Relu = nn.ReLU(inplace=True)\n        self.PReLU = nn.PReLU()\n        self.LeakyReLU = nn.LeakyReLU(negative_slope=0.01, inplace=True)\n        # self.GeLU = nn.GELU()\n        self.RReLU = nn.RReLU()\n\n    def forward(self, x):\n        x = self.batch_norm0(x)\n        x = self.dropout0(x)\n\n        x1 = self.dense1(x)\n        x1 = self.batch_norm1(x1)\n        # x1 = F.relu(x1)\n        # x1 = self.PReLU(x1)\n        x1 = self.LeakyReLU(x1)\n        x1 = self.dropout1(x1)\n\n        x2 = self.dense2(x1)\n        x2 = self.batch_norm2(x2)\n        # x2 = F.relu(x2)\n        # x2 = self.PReLU(x2)\n        x2 = self.LeakyReLU(x2)\n        x2 = self.dropout2(x2)\n\n        x3 = self.dense3(x2)\n        x3 = self.batch_norm3(x3)\n        # x3 = F.relu(x3)\n        # x3 = self.PReLU(x3)\n        x3 = self.LeakyReLU(x3)\n        x3 = self.dropout3(x3)\n\n        output = self.regressioner(x3)\n        output_x, output_y, output_f = output.split(1, dim=-1)\n        # print('output_x: ', output_x.shape)\n        # print('output_y: ', output_y.shape)\n        # print('output_f: ', output_f.shape)\n        return output_x, output_y, output_f\n\nall_oof, all_pred = [], []\nstart_time = time.time()\nfor n_files, file in enumerate(train_files):\n    seed_everything(seed=42)\n\n    SiteID = file.split('/')[-1].split('_')[0]\n    train = pd.read_csv(file, index_col=0)\n    # print(train.head())\n\n    groups = train.iloc[:, -1].values\n    train_feat, train_x, train_y, train_f = train.iloc[:, :-4].values, train.iloc[:, -4].values, train.iloc[:, -3].values, train.iloc[:, -2].values\n    feat_dim = train_feat.shape[1]\n\n    train_feat[train_feat == -999] = -99\n    train_feat = train_feat / 100. + 1.\n\n    print(f'[{n_files+1:2}/{len(train_files)}]\\t{SiteID} Train: {len(train):5}\\tFeat Num: {feat_dim:5}')\n\n    oof_x, oof_y, oof_f = np.zeros(len(train_feat)), np.zeros(len(train_feat)), np.zeros(len(train_feat))\n    # kf = KFold(n_splits=NFOLDS, shuffle=True, random_state=42)\n    kf = GroupKFold(n_splits=NFOLDS)\n    for _fold, (trn_idx, val_idx) in enumerate(kf.split(train_feat, groups=groups)):\n        # if _fold != 4:\n        #     continue\n        train_set = IndoorDataset(\n            feat=train_feat[trn_idx],\n            x=train_x[trn_idx],\n            y=train_y[trn_idx],\n            f=train_f[trn_idx],\n        )\n        valid_set = IndoorDataset(\n            feat=train_feat[val_idx],\n            x=train_x[val_idx],\n            y=train_y[val_idx],\n            f=train_f[val_idx],\n        )\n        train_loader = DataLoader(train_set, batch_size=BATCH_SIZE, shuffle=True, drop_last=True, num_workers=4)\n        valid_loader = DataLoader(valid_set, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)\n\n        torch.cuda.empty_cache()\n        device = torch.device('cuda')\n        model = Model(input_dim=feat_dim)\n        model.to(device)\n\n        optimizer = torch.optim.Adam(model.parameters(), lr=LR)\n        # scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5,\n        #                                                        patience=SCHEDULE_DECAY, verbose=False)\n        scheduler = None\n\n        loss_fn = MeanPositionLoss()\n\n        best_mpe = 500.\n        model_weights = f\"{save_path}/{SiteID}_model{_fold}.pth\"\n        es = EarlyStopping(patience=EARLYSTOP_NUM, mode=\"min\")\n        for epoch in range(EPOCHS):\n            train_fn(train_loader, model, optimizer, device, scheduler=None)\n            val_pred_x, val_pred_y, val_pred_f = infer_fn(valid_loader, model, device)\n            val_mpe = MeanPositionScore(val_pred_x, val_pred_y, val_pred_f, train_x[val_idx], train_y[val_idx], train_f[val_idx])\n            if np.isnan(val_mpe):\n                break\n\n            if best_mpe >= val_mpe:\n                best_mpe = val_mpe\n            if scheduler:\n                scheduler.step(best_mpe)\n\n            # print(f\"\\tFOLD{_fold} EPOCH:{epoch:3} val_mpe={val_mpe:.5f} \"\n            #       f\"time: {(time.time() - start_time) / 60:.2f}min\")\n\n            es(val_mpe, model, model_path=model_weights)\n            if es.early_stop:\n                # print(\"Early stopping\")\n                break\n        # val&infer pred\n        model.load_state_dict(torch.load(model_weights))\n        oof_x[val_idx], oof_y[val_idx], oof_f[val_idx] = infer_fn(valid_loader, model, device)\n    oof_mpe = MeanPositionScore(oof_x, oof_y, oof_f, train_x, train_y, train_f)\n    print(f'\\t{SiteID} OOF MPE: {oof_mpe:.5f} time: {(time.time() - start_time) / 60:.2f}min')\n\n    train_pred_df = train[['path']].copy(deep=True)\n    train_pred_df['label_floor'] = train_f\n    train_pred_df['label_x'] = train_x\n    train_pred_df['label_y'] = train_y\n    train_pred_df['floor'] = oof_f\n    train_pred_df['x'] = oof_x\n    train_pred_df['y'] = oof_y\n    all_oof.append(train_pred_df)\n\nall_oof = pd.concat(all_oof).reset_index(drop=True)\nall_oof['floor'] = all_oof['floor'].apply(lambda x: int(x))\nall_oof_mpe = MeanPositionScore(all_oof.x.values, all_oof.y.values, all_oof.floor.values, all_oof.label_x.values, all_oof.label_y.values, all_oof.label_floor.values)\nprint(f'\\nAll OOF MPE: {all_oof_mpe:.4f}')\nall_oof.to_csv(f'{save_path}/all_oof.csv', index=False)\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}