{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This is my attempt at putting together one end to end notebook for using models on wifi features. Thanks @deviananzelmo,  @higepon and @jiweiliu for sharing your notebooks on wifi features\n\n1. setting up a config class to tune parameters easily\n2. Deterministic seed function for torch users\n3. LR range finder function\n4. This includes reading data, creating Pytorch Datasets and Dataloaders\n5. Pytorch model to fit (in this case a simple MLP model)\n6. Choosing cv strategy (group kfold seems correct for the competition)\n7. train models according to building wifi features Data\n8. Generate OOF predictions from each fold's validation data\n9. Predicting for Test set \n10. plotting training results \n\nIf your're new to OOF concept, try reading [chris doette's wonderful post on hill climbing method](https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/175614)\n\n\nAs of now I haven't added any post processing, but will try to add it here if time permits","metadata":{}},{"cell_type":"markdown","source":"## Library imports","metadata":{}},{"cell_type":"code","source":"!pip install pickle5","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# basic imports\nimport os\nimport gc\nimport math\nimport glob\nimport random\nimport itertools\nimport numpy as np\nimport pandas as pd\nimport pickle5 as pickle\nfrom tqdm.notebook import tqdm\n\n# DL library imports\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau\nfrom  torch.cuda.amp import autocast, GradScaler\n\n# metrics calculation\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import GroupKFold\n\n# basic plotting library\nimport matplotlib.pyplot as plt\n\n# interactive plots\nimport plotly.express as px\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\nfrom plotly.offline import iplot\n\nimport warnings  \nwarnings.filterwarnings('ignore')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Config parameters","metadata":{}},{"cell_type":"code","source":"class CFG:\n    # pipeline parameters\n    SEED        = 42\n    TRAIN       = True\n    LR_FIND     = False\n    TEST        = False\n    N_FOLDS     = 2 \n    N_EPOCHS    = 5\n    TEST_BATCH_SIZE  = 128\n    TRAIN_BATCH_SIZE = 64\n    NUM_WORKERS      = 4\n    DATA_FRAC        = 1.0\n    FOLD_TO_TRAIN    = [0, 1, 2, 3, 4] # \n\n    # model parameters\n    MODEL_ARCH  = 'MLP'\n    MODEL_NAME  = 'mlp_v1'\n    WGT_PATH    = ''\n    WGT_MODEL   = ''\n    PRINT_N_EPOCH = 2\n    \n    # scheduler variables\n    MAX_LR    = 1e-2\n    MIN_LR    = 1e-5\n    SCHEDULER = 'CosineAnnealingWarmRestarts'  # ['ReduceLROnPlateau', 'None', OneCycleLR','CosineAnnealingLR']\n    T_0       = 10     # CosineAnnealingWarmRestarts\n    T_MULT    = 2      # CosineAnnealingWarmRestarts\n    T_MAX     = 10      # CosineAnnealingLR\n\n    # optimizer variables\n    OPTIMIZER     = 'Adam'\n    WEIGHT_DECAY  = 1e-6\n    GRD_ACC_STEPS = 1\n    MAX_GRD_NORM  = 2\n\n    # features parameters\n    USE_FREQ_FEATS = False\n    USE_DT_FEATS   = False\n    \n    BUILDING_SITES_RANGE = [0,1]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"floor_map = {\"B2\": -2, \"B1\": -1, \"F1\": 0, \"F2\": 1, \"F3\": 2, \"F4\": 3, \"F5\": 4, \"F6\": 5, \"F7\": 6, \"F8\": 7, \"F9\": 8,\n             \"1F\": 0, \"2F\": 1, \"3F\": 2, \"4F\": 3, \"5F\": 4, \"6F\": 5, \"7F\": 6, \"8F\": 7, \"9F\": 8}\n\nminCount = 1\nfreqFillerValue = 0\nrssiFillerValue = -999.0\ndtFillerValue   = 1000.0\nmodelOutputDir = '.'\nwifiFeaturesDir_train = '../input/idln-mlp-wifi-features-dataset/wiFiFeatures/train'\nwifiFeaturesDir_test  = '../input/idln-mlp-wifi-features-dataset/wiFiFeatures/test'\nsampleCsvPath = '../input/indoor-location-navigation/sample_submission.csv'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Helper functions","metadata":{"papermill":{"duration":0.030885,"end_time":"2021-02-15T09:18:40.779692","exception":false,"start_time":"2021-02-15T09:18:40.748807","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def getBuildingName(buildingDataPath):\n    return buildingDataPath.split('/')[-1].split('_')[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find_no_of_trainable_params(model):\n    total_trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n    return total_trainable_params","metadata":{"papermill":{"duration":0.040621,"end_time":"2021-02-15T09:18:40.851226","exception":false,"start_time":"2021-02-15T09:18:40.810605","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nset_seed(CFG.SEED)","metadata":{"papermill":{"duration":0.045466,"end_time":"2021-02-15T09:18:40.927741","exception":false,"start_time":"2021-02-15T09:18:40.882275","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n#### rssi features is fixed, we can choose to use dt and freq features optionally\n# Incase freq signal is not needed, use rssi and dt features alone    \n# There are 5 columns for timestamp, y, pathNames values in csv, reamining are features\n# total features = 3 * [rssi, dt, freq]\n# hence unique wifi ids = totalFeatures / 3\n\"\"\"\ndef getBuildingFeatures(buildingData, dataType):\n    if dataType == 'train':\n        buildingData = buildingData.iloc[:,1:-4].values.astype(np.float16)\n    else:\n        buildingData = buildingData.iloc[:,1:-1].values.astype(np.float16)\n    \n    numBssids = int(buildingData.shape[1] / 3)\n    ## replace -999 with 99, 1000.0 with 50.0 and scale accordindly\n    buildingData[buildingData == -999.0] = -99.0\n    buildingData[buildingData == 1000.0] = 50.0\n    buildingData[:,0:numBssids]              = buildingData[:,0:numBssids] / 100.0\n    buildingData[:,numBssids: 2*numBssids]   = buildingData[:,numBssids: 2*numBssids] / 50.0\n    buildingData[:,2*numBssids: 3*numBssids] = buildingData[:,2*numBssids: 3*numBssids] / 1000.0\n\n    if CFG.USE_FREQ_FEATS == True:    \n        ## use all features\n        if CFG.USE_DT_FEATS == True:\n            X = buildingData\n        ## use rssi and freq features alone\n        else:\n            desiredFeatures = list(range(0, numBssids)) + list(range(2*numBssids, 3*numBssids))\n            X = buildingData[:, desiredFeatures]\n    else:\n        ## use only rssi features alone\n        if CFG.USE_DT_FEATS == True:\n            X = buildingData[:,0:2*numBssids]\n        ## use only rssi features alone\n        else:\n            X = buildingData[:,0:numBssids]\n    return X","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def getBuildingData(buildingDataPath):\n    # read building data \n    #### data = pd.read_pickle(buildingDataPath)\n    with open(buildingDataPath, 'rb') as inputFile:\n        data = pickle.load(inputFile)    \n    \n    # use fraction if needed\n    if CFG.DATA_FRAC < 1:\n        data = data.sample(frac=CFG.DATA_FRAC).reset_index(drop=True)\n\n    # first column is timestamp\n    timestamps = data.iloc[:,0].values   # np.expand_dims( , ,axis=1)\n    \n    # last column is pathFile name\n    groups = data.iloc[:,-1].values\n    \n    # target values are last but 3 columns\n    y = data.iloc[:,-4:-1].values\n    X = getBuildingFeatures(data,'train')\n    ## del data\n    ## gc.collect()\n    return timestamps,X,y,groups","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def getInputFeatureSize(featureFilesPath):\n    sampleData = np.load(f\"{npyWifiFeaturesDir}/{featureFilesPath[0]}\")\n    return len(sampleData)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def competitionMetric(preds, targets):\n    \"\"\" The metric used in this competition \"\"\"\n    # position error\n    meanPosPredictionError = torch.mean(torch.sqrt(\n                             torch.square(torch.subtract(preds[:,0], targets[:,0])) + \n                             torch.square(torch.subtract(preds[:,1], targets[:,1]))))\n    # error in floor prediction\n    meanFloorPredictionError = torch.mean(15 * torch.abs(preds[:,2] - targets[:,2]))\n    return meanPosPredictionError, meanFloorPredictionError","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def getOptimizer(model : nn.Module):    \n    if CFG.OPTIMIZER == 'Adam':\n        optimizer = optim.Adam(model.parameters(), weight_decay=CFG.WEIGHT_DECAY, lr=CFG.MAX_LR)\n    else:\n        optimizer = optim.SGD(model.parameters(), weight_decay=CFG.WEIGHT_DECAY, lr=CFG.MAX_LR, momentum=0.9)\n    return optimizer","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def getScheduler(optimizer, dataloader_train):\n    if CFG.SCHEDULER == 'OneCycleLR':\n        scheduler = optim.lr_scheduler.OneCycleLR(optimizer, max_lr= CFG.MAX_LR, epochs = CFG.N_EPOCHS, \n                          steps_per_epoch = len(dataloader_train), pct_start=0.25, div_factor=10, anneal_strategy='cos')\n    elif CFG.SCHEDULER == 'CosineAnnealingWarmRestarts':\n        scheduler = CosineAnnealingWarmRestarts(optimizer, T_0=CFG.T_0, T_mult=CFG.T_MULT, eta_min=CFG.MIN_LR, last_epoch=-1)\n    elif CFG.SCHEDULER == 'CosineAnnealingLR':\n        scheduler = CosineAnnealingLR(optimizer, T_max=CFG.T_MAX * len(dataloader_train), eta_min=CFG.MIN_LR, last_epoch=-1)\n    else:\n        scheduler = None\n    return scheduler","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def getDataLoader(dataset, datasetType : str):\n    if datasetType == 'train':\n        batchSize = CFG.TRAIN_BATCH_SIZE\n        shuffleDataset = True\n    else:\n        batchSize = CFG.TEST_BATCH_SIZE\n        shuffleDataset = False\n    \n    dataLoader = DataLoader(dataset, batch_size= batchSize, shuffle=shuffleDataset,\n                            num_workers=CFG.NUM_WORKERS, pin_memory=False, drop_last=False)\n    return dataLoader","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plotTrainingResults(resultsDf, buildingName):\n    # subplot to plot\n    fig = make_subplots(rows=1, cols=1)\n    colors = [ ('#d32f2f', '#ef5350'), ('#303f9f', '#5c6bc0'), ('#00796b', '#26a69a'),\n                ('#fbc02d', '#ffeb3b'), ('#5d4037', '#8d6e63')]\n\n    # find number of folds input df\n    numberOfFolds = resultsDf['fold'].nunique()\n    \n    # iterate through folds and plot\n    for i in range(numberOfFolds):\n        data = resultsDf[resultsDf['fold'] == i]\n        fig.add_trace(go.Scatter(x=data['epoch'].values, y=data['trainPosLoss'].values,\n                                mode='lines', visible='legendonly' if i > 0 else True,\n                                line=dict(color=colors[i][0], width=2),\n                                name='{}-trainPossLoss-Fold{}'.format(buildingName, i)),row=1, col=1)\n\n        fig.add_trace(go.Scatter(x=data['epoch'], y=data['valPosLoss'].values,\n                                 mode='lines+markers', visible='legendonly' if i > 0 else True,\n                                 line=dict(color=colors[i][1], width=2),\n                                 name='{}-valPosLoss-Fold{}'.format(buildingName,i)),row=1, col=1)\n    fig.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset class","metadata":{}},{"cell_type":"code","source":"class wiFiFeaturesDataset(Dataset):\n    def __init__(self, timeStamps, X_data, y_data, groups):\n        self.timeStamps = timeStamps \n        self.X_data = X_data\n        self.y_data = y_data\n        self.groups = groups\n        \n    def __getitem__(self, index):\n        x  = torch.from_numpy(self.X_data[index].astype(np.float32))\n        y  = torch.from_numpy(self.y_data[index].astype(np.float32))\n        ts = self.timeStamps[index].astype(np.int64)\n        group = self.groups[index]\n        return ts,x,y,group\n    \n    def __len__ (self):\n        return len(self.X_data)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class wiFiFeaturesDataset_test(Dataset):\n    def __init__(self, timeStamps, X_data, groups):\n        self.timeStamps = timeStamps \n        self.X_data = X_data\n        self.groups = groups\n        \n    def __getitem__(self, index):\n        x  = torch.from_numpy(self.X_data[index].astype(np.float32))\n        ts = self.timeStamps[index].astype(np.int64)\n        group = self.groups[index]\n        return ts,x,group\n    \n    def __len__ (self):\n        return len(self.X_data)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## MLP Model class","metadata":{}},{"cell_type":"code","source":"class wiFiFeaturesMLPModel(nn.Module):\n    def __init__(self, n_input, n_output):\n        super().__init__()\n        self.lin1 = nn.Linear(in_features=n_input, out_features=512)\n        self.lin2 = nn.Linear(in_features=512,     out_features=32)\n        self.lin3 = nn.Linear(in_features=32,      out_features=n_output)\n        self.bn1 = nn.BatchNorm1d(512)\n        self.bn2 = nn.BatchNorm1d(32)\n        self.drops = nn.Dropout(0.3)        \n\n    def forward(self, x):\n        numBatches = x.shape[0]\n        \n        x = F.relu(self.lin1(x))\n        x = self.drops(x)\n        \n        ## batchnorm doesnt work for batchsize of 1\n        if numBatches > 1:\n            x = self.bn1(x)\n            x = F.relu(self.lin2(x))\n            x = self.drops(x)\n            x = self.bn2(x)\n            x = self.lin3(x)\n        else:\n            x = F.relu(self.lin2(x))\n            x = self.drops(x)\n            x = self.lin3(x)\n        return x","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Lr range finder","metadata":{}},{"cell_type":"code","source":"def plot_lr_finder_results(lr_finder): \n    # Create subplot grid\n    fig = make_subplots(rows=1, cols=2)\n    # layout ={'title': 'Lr_finder_result'}\n    \n    # Create a line (trace) for the lr vs loss, gradient of loss\n    trace0 = go.Scatter(x=lr_finder['log_lr'], y=lr_finder['smooth_loss'],name='log_lr vs smooth_loss')\n    trace1 = go.Scatter(x=lr_finder['log_lr'], y=lr_finder['grad_loss'],name='log_lr vs loss gradient')\n\n    # Add subplot trace & assign to each grid\n    fig.add_trace(trace0, row=1, col=1);\n    fig.add_trace(trace1, row=1, col=2);\n    iplot(fig, show_link=False)\n    #fig.write_html(CFG.MODEL_NAME + '_lr_find.html');","metadata":{"papermill":{"duration":0.044791,"end_time":"2021-02-15T09:18:43.045246","exception":false,"start_time":"2021-02-15T09:18:43.000455","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find_lr(model, optimizer, data_loader, init_value = 1e-8, final_value=100.0, beta = 0.98, num_batches = 200):\n    assert(num_batches > 0)\n    mult = (final_value / init_value) ** (1/num_batches)\n    lr = init_value\n    optimizer.param_groups[0]['lr'] = lr\n    batch_num = 0\n    avg_loss = 0.0\n    best_loss = 0.0\n    smooth_losses = []\n    raw_losses = []\n    log_lrs = []\n    dataloader_it = iter(data_loader)\n    progress_bar = tqdm(range(num_batches))                \n        \n    for idx in progress_bar:\n        batch_num += 1\n        try:\n            _, inputs, targets = next(dataloader_it)\n            #print(images.shape)\n        except:\n            dataloader_it = iter(data_loader)\n            _, inputs, targets = next(dataloader_it)\n\n        # Move input and label tensors to the default device\n        inputs = inputs.to(device)\n        targets = targets.to(device)\n\n        # handle exception in criterion\n        try:\n            # Forward pass\n            y_preds = model(inputs)\n            posLoss, floorLoss = criterion(y_preds, targets)\n            loss = posLoss + floorLoss\n        except:\n            if len(smooth_losses) > 1:\n                grad_loss = np.gradient(smooth_losses)\n            else:\n                grad_loss = 0.0\n            lr_finder_results = {'log_lr':log_lrs, 'raw_loss':raw_losses, \n                                 'smooth_loss':smooth_losses, 'grad_loss': grad_loss}\n            return lr_finder_results \n                    \n        #Compute the smoothed loss\n        avg_loss = beta * avg_loss + (1-beta) *loss.item()\n        smoothed_loss = avg_loss / (1 - beta**batch_num)\n        \n        #Stop if the loss is exploding\n        if batch_num > 1 and smoothed_loss > 50 * best_loss:\n            if len(smooth_losses) > 1:\n                grad_loss = np.gradient(smooth_losses)\n            else:\n                grad_loss = 0.0\n            lr_finder_results = {'log_lr':log_lrs, 'raw_loss':raw_losses, \n                                 'smooth_loss':smooth_losses, 'grad_loss': grad_loss}\n            return lr_finder_results\n        \n        #Record the best loss\n        if smoothed_loss < best_loss or batch_num==1:\n            best_loss = smoothed_loss\n        \n        #Store the values\n        raw_losses.append(loss.item())\n        smooth_losses.append(smoothed_loss)\n        log_lrs.append(math.log10(lr))\n        \n        # Backward pass\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        \n        # print info\n        progress_bar.set_description(f\"loss:{loss.item()},smoothLoss: {smoothed_loss},lr:{lr}\")\n\n        #Update the lr for the next step\n        lr *= mult\n        optimizer.param_groups[0]['lr'] = lr\n    \n    grad_loss = np.gradient(smooth_losses)\n    lr_finder_results = {'log_lr':log_lrs, 'raw_loss':raw_losses, \n                         'smooth_loss':smooth_losses, 'grad_loss': grad_loss}\n    return lr_finder_results","metadata":{"papermill":{"duration":0.059229,"end_time":"2021-02-15T09:18:43.136586","exception":false,"start_time":"2021-02-15T09:18:43.077357","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CFG.LR_FIND == True:\n    # create dataset instance\n    tempTs, tempX, tempY,_ = getBuildingData(buildingCsvPath=buildingsList[0])\n    tempX = stdScaler.fit_transform(tempX)\n    tempTrainDataset = wiFiFeaturesDataset(tempTs, tempX, tempY)\n    tempTrainDataloader = DataLoader(tempTrainDataset, batch_size= CFG.TRAIN_BATCH_SIZE, shuffle=True,\n                          num_workers=CFG.NUM_WORKERS, pin_memory=False, drop_last=False)\n    \n    # create model instance   \n    model = wiFiFeaturesMLPModel(n_input=tempX.shape[1], n_output=3)\n    model.to(device);\n    \n    # optimizer function, lr schedulers and loss function\n    optimizer = getOptimizer(model)\n    lrFinderResults = find_lr(model, optimizer, tempTrainDataloader)\n    plot_lr_finder_results(lrFinderResults)\n    del tempX, tempY, tempTrainDataset, tempTrainDataloader, model, optimizer","metadata":{"papermill":{"duration":0.048229,"end_time":"2021-02-15T09:18:43.217286","exception":false,"start_time":"2021-02-15T09:18:43.169057","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train & Validate helper functions","metadata":{}},{"cell_type":"code","source":"def validateModel(model, validationDataloader):\n    # placeholders to store output\n    val_ts = []\n    val_preds = []\n    val_targets = []\n    val_groups = []\n\n    # set model to Validate mode\n    model.eval()\n    dataLoaderIterator = iter(validationDataloader)\n\n    for idx in range(len(validationDataloader)):\n        try:\n            ts, inputs, targets, valGroups = next(dataLoaderIterator)\n        except StopIteration:\n            dataLoaderIterator = iter(validationDataloader)\n            ts, inputs, targets, valGroups = next(dataLoaderIterator)\n\n        inputs = inputs.to(device)\n        targets = targets.to(device) \n\n        # forward prediction\n        with torch.no_grad():    \n            y_preds = model(inputs)\n\n        # store predictions and targets to compute metrics later\n        val_ts.append(ts)\n        val_preds.append(y_preds)\n        val_targets.append(targets)\n        val_groups.append(valGroups)\n\n    # concatenate to get as 1 2d array and find total loss  \n    val_preds = torch.cat(val_preds, 0)\n    val_targets = torch.cat(val_targets, 0)\n    valPosLoss, valFloorLoss = criterion(val_preds, val_targets)\n    valScore = valPosLoss + valFloorLoss\n\n    # np array concatenation\n    val_ts = np.concatenate(val_ts, axis=0)\n    val_groups = np.concatenate(val_groups, axis=0)\n    \n    # store results\n    validationResults = {'valPosLoss': valPosLoss.item() , 'valFloorLoss': valFloorLoss.item(),\\\n                         'val_ts': val_ts, 'val_groups': val_groups,\n                         'val_preds'  :val_preds.cpu().data.numpy(), \n                         'val_targets':val_targets.cpu().data.numpy(),\n                         }\n    return validationResults","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def trainValidateOneFold(buildingName, i_fold, model, optimizer, scheduler, dataloader_train, dataloader_valid):\n    trainFoldResults = []\n    bestValScore = np.inf\n    bestEpoch = 0\n\n    for epoch in range(CFG.N_EPOCHS):\n        #print('Epoch {}/{}'.format(epoch + 1, CFG.N_EPOCHS))\n        model.train()\n        trainPosLoss = 0.0\n        trainFloorLoss = 0.0\n\n        # training iterator\n        tr_iterator = iter(dataloader_train)\n\n        for idx in range(len(dataloader_train)):\n            try:\n                _, inputs, targets, _ = next(tr_iterator)\n            except StopIteration:\n                tr_iterator = iter(dataloader_train)\n                _, inputs, targets, _ = next(tr_iterator)\n\n            inputs = inputs.to(device)\n            targets = targets.to(device)  \n\n            # builtin package to handle automatic mixed precision\n            with autocast():\n                # Forward pass\n                y_preds = model(inputs)   \n                posLoss, floorLoss = criterion(y_preds, targets)\n                loss = posLoss + floorLoss\n\n                # Backward pass\n                scaler.scale(loss).backward()        \n                scaler.step(optimizer)\n                scaler.update()\n                optimizer.zero_grad() \n\n                # log the necessary losses\n                trainPosLoss   += posLoss.item()\n                trainFloorLoss += floorLoss.item()\n\n                if scheduler is not None: \n                    if CFG.SCHEDULER == 'CosineAnnealingWarmRestarts':\n                        scheduler.step(epoch + idx / len(dataloader_train)) \n                    # onecyle lr scheduler / CosineAnnealingLR scheduler\n                    else:\n                        scheduler.step()\n                    \n        # Validate\n        foldValidationResults = validateModel(model, dataloader_valid)\n         \n        # store results\n        trainFoldResults.append({ 'fold': i_fold, 'epoch': epoch, \n                                  'trainPosLoss': trainPosLoss / len(dataloader_train), \n                                  'trainFloorLoss': trainFloorLoss / len(dataloader_train), \n                                  'valPosLoss'  : foldValidationResults['valPosLoss'] , \n                                  'valFloorLoss': foldValidationResults['valFloorLoss']})\n        \n        valScore = foldValidationResults['valPosLoss'] # + foldVal['valFloorLoss']\n        # save best models        \n        if(valScore < bestValScore):\n            # reset variables\n            bestValScore = valScore\n            bestEpoch = epoch\n\n            # save model weights\n            torch.save({'model': model.state_dict(), 'val_ts' : foldValidationResults['val_ts'], \n                        'val_preds':foldValidationResults['val_preds'], \n                        'val_targets':foldValidationResults['val_targets'],\n                        'val_groups' : foldValidationResults['val_groups']}, \n                        f\"{modelOutputDir}/{buildingName}_{CFG.MODEL_NAME}_fold{i_fold}_best.pth\")\n\n    print(f\"For Fold {i_fold}, Best validation score of {bestValScore} was got at epoch {bestEpoch}\") \n    return trainFoldResults","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def trainValidateOneBuilding(buildingDataPath, modelToFit):\n    # placeholder to store results\n    buildingTrainResults = []\n    \n    buildingName = getBuildingName(buildingDataPath)\n    print(f\"Processing data for building - {buildingName}\")\n    timestamps, X, y, groups = getBuildingData(buildingDataPath)\n    print(f\"Building Data shapes : {timestamps.shape, X.shape, y.shape, groups.shape}\")\n\n    for i_fold, (trainIndex, validIndex) in enumerate(folds.split(X=X, y=y[:,0],groups=groups)):\n        if i_fold in CFG.FOLD_TO_TRAIN:\n            ## print(\"Fold {}/{}\".format(i_fold + 1, CFG.N_FOLDS))\n            \n            # splitting into train and validataion sets\n            trainTimeStamps, X_train, y_train, trainGroups = timestamps[trainIndex], X[trainIndex], y[trainIndex], groups[trainIndex]\n            validTimeStamps, X_valid, y_valid, validGroups = timestamps[validIndex], X[validIndex], y[validIndex], groups[validIndex] \n                        \n            # create torch Datasets and Dataloader for each fold's train and validation data\n            dataset_train = wiFiFeaturesDataset(trainTimeStamps, X_train, y_train, trainGroups)\n            dataset_valid = wiFiFeaturesDataset(validTimeStamps, X_valid, y_valid, validGroups)            \n            dataloader_train = getDataLoader(dataset_train, datasetType= 'train')\n            dataloader_valid = getDataLoader(dataset_valid, datasetType= 'valid')\n            \n            # supervised model instance and move to compute device\n            model = modelToFit(n_input=X.shape[1], n_output=3)\n            model.to(device);\n            ### print(f\"there are {find_no_of_trainable_params(model)} params in model\")\n\n            # optimizer function, lr schedulers and loss function\n            optimizer = getOptimizer(model)\n            scheduler = getScheduler(optimizer, dataloader_train)\n            # print(f\"optimizer={optimizer}, scheduler={scheduler}, loss_fn={criterion}\")\n\n            # train and validate single fold\n            foldResults = trainValidateOneFold(buildingName, i_fold, model, optimizer, scheduler,dataloader_train, dataloader_valid)\n            buildingTrainResults = buildingTrainResults + foldResults\n            \n            ## del trainTimeStamps, X_train, y_train, trainGroups\n            ## del validTimeStamps, X_valid, y_valid, validGroups\n            ## del dataloader_train, dataloader_valid, model, optimizer, scheduler\n            ## gc.collect()\n    \n    ## del timestamps, X, y, groups\n    ## gc.collect()\n    \n    buildingTrainResults = pd.DataFrame(buildingTrainResults)\n    buildingTrainResults['valTotalLoss'] = buildingTrainResults['valPosLoss'] + buildingTrainResults['valFloorLoss']\n    buildingTrainResults['trainTotalLoss'] = buildingTrainResults['trainPosLoss'] + buildingTrainResults['trainFloorLoss']\n    return buildingTrainResults","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def getFoldBestResultsDf(trainResults):\n    bestResults = []\n    numFolds = trainResults['fold'].nunique()\n    \n    for fold in range(numFolds):\n        foldDf = trainResults[trainResults['fold']== fold]\n        bestResults.append(foldDf.iloc[np.argmin(foldDf['valTotalLoss'].values),:])\n    \n    bestResults =pd.DataFrame(bestResults)\n    valPosLossBest = bestResults['valPosLoss'].values\n    print(f\"Best valPosLoss for all folds = {valPosLossBest}\")\n    print(f\"Mean, std ={valPosLossBest.mean()}, {valPosLossBest.std()}\")\n    return bestResults","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Generate OOF function","metadata":{}},{"cell_type":"code","source":"def generateOOF(modelSaveDir:str, buildingName:str, modelName:str):\n    oof_ts = []\n    oof_preds = []\n    oof_targets = []\n    oof_groups = []\n    oof_folds = []\n\n    modelPaths = sorted(glob.glob(f\"{modelSaveDir}/{buildingName}_{modelName}_fold*.pth\"))\n\n    for fold in range(len(modelPaths)):\n        # load building-model-fold checkpoint\n        checkPoint = torch.load(modelPaths[fold], map_location=torch.device(device))\n        numRows = len(checkPoint['val_ts'])\n\n        oof_ts.append(checkPoint['val_ts'])\n        oof_preds.append(checkPoint['val_preds'])\n        oof_targets.append(checkPoint['val_targets'])\n        oof_groups.append(checkPoint['val_groups'])\n        oof_folds.append([fold] * numRows)\n    \n    oof_ts = np.concatenate(oof_ts,axis=0)\n    oof_preds = np.concatenate(oof_preds,axis=0)\n    oof_targets = np.concatenate(oof_targets,axis=0)\n    oof_groups = np.concatenate(oof_groups,axis=0)\n    oof_folds = np.concatenate(oof_folds,axis=0)\n    \n    #print(oof_ts.shape, oof_preds.shape, oof_targets.shape, oof_groups.shape, oof_folds.shape)\n    oof_df = pd.DataFrame({'timestamp' : oof_ts, 'x_preds': oof_preds[:,0], 'y_preds': oof_preds[:,1],\n                       'floor_preds': oof_preds[:,2], 'x_tgt': oof_targets[:,0], 'y_tgt': oof_targets[:,1],\n                       'floor_tgt': oof_targets[:,2], 'path' : oof_groups, 'fold' : oof_folds\n                      })\n    print(f\"OOF prediction for {buildingName} site generated\")\n    return oof_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test set prediction function","metadata":{}},{"cell_type":"code","source":"def generateWiFiSubmission(modelToFit, modelSaveDir:str, buildingName:str, modelName:str):\n    modelPaths = sorted(glob.glob(f\"{modelSaveDir}/{buildingName}_{modelName}_fold*.pth\"))\n    buildingTestData = f\"{wifiFeaturesDir_test}/{buildingName}_test.pickle\"\n    with open(buildingTestData, 'rb') as inputFile:\n        testData = pickle.load(inputFile)    \n    \n    test_ts = []\n    test_fold = []\n    test_preds = []\n    test_groups = []\n\n    for fold in range(CFG.N_FOLDS):\n        ## print(f\"Fold {fold} processing\")\n        #print(f\"Before stdscaler : testX mean = {testX.mean()}, testData std = {testX.std()}\")\n        testGroups = testData.iloc[:,-1].values    \n        testTimestamps = testData.iloc[:,0].values        \n        testX = getBuildingFeatures(testData, 'test')  \n        \n        checkPoint = torch.load(modelPaths[fold], map_location=torch.device(device))\n        model = modelToFit(n_input=testX.shape[1], n_output=3)\n        model.to(device);\n        model.load_state_dict(checkPoint['model'])\n\n        # set model to Validate mode\n        model.eval()\n        ## test Dataset and data loaders\n        testDataset = wiFiFeaturesDataset_test(testTimestamps, testX, testGroups)\n        testDataloader = getDataLoader(testDataset, datasetType= 'test')\n\n        dataLoaderIterator = iter(testDataloader)\n        for idx in range(len(testDataloader)):\n            try:\n                ts, inputs, testGroups = next(dataLoaderIterator)\n            except StopIteration:\n                dataLoaderIterator = iter(testDataloader)\n                ts, inputs, testGroups = next(dataLoaderIterator)\n\n            inputs = inputs.to(device)\n            # forward prediction\n            with torch.no_grad():    \n                y_preds = model(inputs)\n\n            # store predictions and targets to compute metrics later\n            test_ts.append(ts)\n            test_preds.append(y_preds)\n            test_groups.append(testGroups)\n        \n        test_fold.append([fold] * len(testX))\n        ## del testDataloader\n        ## torch.cuda.empty_cache()\n        ## gc.collect()\n        \n    # concatenate to get as 1 2d array \n    test_preds = torch.cat(test_preds, 0).cpu().data.numpy() \n    test_ts = np.concatenate(test_ts, axis=0)\n    test_fold = np.concatenate(test_fold, axis=0)\n    test_groups = np.concatenate(test_groups, axis=0)\n    subm_wifi_df = pd.DataFrame({'timestamp' : test_ts, 'x_preds': test_preds[:,0], 'y_preds': test_preds[:,1],\n                                 'floor_preds': test_preds[:,2], 'path' : test_groups, 'fold' : test_fold})\n    subm_wifi_df.to_pickle(f\"{buildingName}_wifi_subm.pickle\")  \n    print(f\"Test data prediction for {buildingName} site generated\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Compute Device as CPU or GPU","metadata":{}},{"cell_type":"code","source":"## Device as cpu or tpu\ndevice = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device('cpu')\nprint(device)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocessing classes","metadata":{}},{"cell_type":"code","source":"# for cv\nfolds = GroupKFold(n_splits=CFG.N_FOLDS)\n\n# scaler to handle AMP\nscaler = GradScaler()   \n\ncriterion = competitionMetric\nmodelToFit = wiFiFeaturesMLPModel","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training & Validation main function","metadata":{}},{"cell_type":"code","source":"%%time\nif CFG.TRAIN == True:\n    \n    buildingPathList_train = sorted(glob.glob(f\"{wifiFeaturesDir_train}/*.pickle\"))\n    buildingPathList_train = buildingPathList_train[CFG.BUILDING_SITES_RANGE[0]: CFG.BUILDING_SITES_RANGE[1]]\n    print(f\"{len(buildingPathList_train)} sites are to be trained\")\n    ## print(buildingPathList_train)\n\n    for number,buildingPath_train in enumerate(buildingPathList_train):\n        \n        print(f'{number}----------------------------------')\n        ## get building name\n        buildingName = getBuildingName(buildingPath_train)\n        \n        ## train and validate for building data\n        buildingTrainResults = trainValidateOneBuilding(buildingPath_train, modelToFit)\n        bestResults = getFoldBestResultsDf(buildingTrainResults)\n        \n        ## generate OOF prediction for building-model combination\n        buildingOOF = generateOOF(modelOutputDir, buildingName, CFG.MODEL_NAME)\n        \n        ## prediction for test data too\n        generateWiFiSubmission(modelToFit, modelOutputDir, buildingName, CFG.MODEL_NAME)\n\n        ## save results to file\n        buildingOOF.to_pickle(f\"{modelOutputDir}/{buildingName}_{CFG.MODEL_NAME}_OOF.pickle\")\n        bestResults.to_pickle(f\"{modelOutputDir}/{buildingName}_{CFG.MODEL_NAME}_bestResults.pickle\")\n        buildingTrainResults.to_pickle(f\"{modelOutputDir}/{buildingName}_{CFG.MODEL_NAME}_trainResults.pickle\")\n        \n        ## plot building results\n        plotTrainingResults(buildingTrainResults, buildingName)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"```python\nbuildingPathList_train = sorted(glob.glob(f\"{wifiFeaturesDir_train}/*.pickle\"))\nbuildingPathList_test = sorted(glob.glob(f\"{wifiFeaturesDir_test}/*.pickle\"))\n\nfor idx in range(len(buildingPathList_train)):\n    print('-----------------------------')\n    trainFilePath = buildingPathList_train[idx]\n    testFilePath  = buildingPathList_test[idx]\n    print(f\"{idx}. {getBuildingName(trainFilePath)}\")\n    with open(trainFilePath, 'rb') as inputTrainFile:\n        trainData = pickle.load(inputTrainFile)\n    with open(testFilePath, 'rb') as inputTestFile:\n        testData = pickle.load(inputTestFile)\n    \n    with pd.option_context('display.max_rows', 1, 'display.max_columns', 12,\n                           'display.width', 500, 'display.precision', 3,\n                           'display.colheader_justify', 'left'):\n        display(trainData)\n        display(testData)\n```        ","metadata":{"_kg_hide-input":true}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}