{"cells":[{"metadata":{},"cell_type":"markdown","source":"# 1. Quantile Regression PyTorch (tabular data only)\nThis notebook generates baseline predictions using tabular data only. I created it to better understand the data.\n\nIt is a simplified version of [this great notebook](Osic-Multiple-Quantile-Regression-Starter) from [Ulrich GOUE\n](https://www.kaggle.com/ulrich07). It also builds on this great [tutorial about Quantile Regression for neural networks](https://medium.com/the-artificial-impostor/quantile-regression-part-2-6fdbc26b2629).\n\nChanges vs previous version:\n- Factored monitoring code out of training block\n- Implemented early stopping\n- Implemented learning rate decay scheduler","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"\n# 2. Imports and global variables","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import copy\nfrom datetime import datetime, timedelta\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nfrom pathlib import Path\nfrom sklearn.model_selection import GroupKFold\nfrom torch.utils.data import Dataset\nfrom tqdm import tqdm\nimport torch\nfrom torch.utils.data import DataLoader, Subset\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import Adam\nfrom torch.optim.lr_scheduler import StepLR\nfrom torch.utils.tensorboard import SummaryWriter\nfrom tqdm.notebook import trange\nfrom time import time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"root_dir = Path('/kaggle/input/osic-pulmonary-fibrosis-progression')\nmodel_dir = Path('/kaggle/working')\nnum_kfolds = 5\nbatch_size = 32\nlearning_rate = 3e-3\nnum_epochs = 1000\nes_patience = 20\nquantiles = (0.2, 0.5, 0.8)\nmodel_name ='descartes'\ntensorboard_dir = Path('/kaggle/working/runs')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3. Dataset interface","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"class ClinicalDataset(Dataset):\n    def __init__(self, root_dir, mode, transform=None):\n        self.transform = transform\n        self.mode = mode\n\n        tr = pd.read_csv(Path(root_dir)/\"train.csv\")\n        tr.drop_duplicates(keep=False, inplace=True, subset=['Patient', 'Weeks'])\n        chunk = pd.read_csv(Path(root_dir)/\"test.csv\")\n\n        sub = pd.read_csv(Path(root_dir)/\"sample_submission.csv\")\n        sub['Patient'] = sub['Patient_Week'].apply(lambda x: x.split('_')[0])\n        sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))\n        sub = sub[['Patient', 'Weeks', 'Confidence', 'Patient_Week']]\n        sub = sub.merge(chunk.drop('Weeks', axis=1), on=\"Patient\")\n\n        tr['WHERE'] = 'train'\n        chunk['WHERE'] = 'val'\n        sub['WHERE'] = 'test'\n        data = tr.append([chunk, sub])\n\n        data['min_week'] = data['Weeks']\n        data.loc[data.WHERE == 'test', 'min_week'] = np.nan\n        data['min_week'] = data.groupby('Patient')['min_week'].transform('min')\n\n        base = data.loc[data.Weeks == data.min_week]\n        base = base[['Patient', 'FVC']].copy()\n        base.columns = ['Patient', 'min_FVC']\n        base['nb'] = 1\n        base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')\n        base = base[base.nb == 1]\n        base.drop('nb', axis=1, inplace=True)\n\n        data = data.merge(base, on='Patient', how='left')\n        data['base_week'] = data['Weeks'] - data['min_week']\n        del base\n\n        COLS = ['Sex', 'SmokingStatus']\n        self.FE = []\n        for col in COLS:\n            for mod in data[col].unique():\n                self.FE.append(mod)\n                data[mod] = (data[col] == mod).astype(int)\n\n        data['age'] = (data['Age'] - data['Age'].min()) / \\\n                      (data['Age'].max() - data['Age'].min())\n        data['BASE'] = (data['min_FVC'] - data['min_FVC'].min()) / \\\n                       (data['min_FVC'].max() - data['min_FVC'].min())\n        data['week'] = (data['base_week'] - data['base_week'].min()) / \\\n                       (data['base_week'].max() - data['base_week'].min())\n        data['percent'] = (data['Percent'] - data['Percent'].min()) / \\\n                          (data['Percent'].max() - data['Percent'].min())\n        self.FE += ['age', 'percent', 'week', 'BASE']\n\n        self.raw = data.loc[data.WHERE == mode].reset_index()\n        del data\n\n    def __len__(self):\n        return len(self.raw)\n\n    def __getitem__(self, idx):\n        if torch.is_tensor(idx):\n            idx = idx.tolist()\n\n        sample = {\n            'patient_id': self.raw['Patient'].iloc[idx],\n            'features': self.raw[self.FE].iloc[idx].values,\n            'target': self.raw['FVC'].iloc[idx]\n        }\n        if self.transform:\n            sample = self.transform(sample)\n\n        return sample\n\n    def group_kfold(self, n_splits):\n        gkf = GroupKFold(n_splits=n_splits)\n        groups = self.raw['Patient']\n        for train_idx, val_idx in gkf.split(self.raw, self.raw, groups):\n            train = Subset(self, train_idx)\n            val = Subset(self, val_idx)\n            yield train, val\n\n    def group_split(self, test_size=0.2):\n        \"\"\"To test no-kfold\n        \"\"\"\n        gss = GroupShuffleSplit(n_splits=1, test_size=test_size)\n        groups = self.raw['Patient']\n        idx = list(gss.split(self.raw, self.raw, groups))\n        train = Subset(self, idx[0][0])\n        val = Subset(self, idx[0][1])\n        return train, val","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 4. Neural Net model and Quantile loss","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class QuantModel(nn.Module):\n    def __init__(self, in_tabular_features=9, out_quantiles=3):\n        super(QuantModel, self).__init__()\n        self.fc1 = nn.Linear(in_tabular_features, 100)\n        self.fc2 = nn.Linear(100, 100)\n        self.fc3 = nn.Linear(100, out_quantiles)\n\n    def forward(self, x):\n        x = F.relu(self.fc1(x))\n        x = F.relu(self.fc2(x))\n        x = self.fc3(x)\n        return x\n\n\ndef quantile_loss(preds, target, quantiles):\n    assert not target.requires_grad\n    assert preds.size(0) == target.size(0)\n    losses = []\n    for i, q in enumerate(quantiles):\n        errors = target - preds[:, i]\n        losses.append(torch.max((q - 1) * errors, q * errors).unsqueeze(1))\n    loss = torch.mean(torch.sum(torch.cat(losses, dim=1), dim=1))\n    return loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 5. Training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Helper class that monitors training\nclass Monitor:\n    def __init__(self, model, es_patience, experiment_name, tensorboard_dir,\n                 num_epochs, dataset_sizes, model_file):\n\n        self.model = model\n        self.model_file = model_file\n        self.es_patience = es_patience\n        self.tensorboard_dir = tensorboard_dir\n        self.dataset_sizes = dataset_sizes\n        date_time = datetime.now().strftime(\"%Y%m%d-%H%M\")\n        log_dir = tensorboard_dir / f'{experiment_name}-{date_time}'\n        self.w = SummaryWriter(log_dir)\n\n        self.bar = trange(num_epochs, desc=experiment_name)\n\n        self.epoch_loss = {'train': np.inf, 'val': np.inf}\n        self.epoch_metric = {'train': -np.inf, 'val': -np.inf}\n        self.best_loss = np.inf\n        self.best_model_wts = None\n\n        self.e = {'train': 0, 'val': 0}  # epoch counter\n        self.t = {'train': 0, 'val': 0}  # global time-step (never resets)\n        self.running_loss = 0.0\n        self.running_metric = 0.0\n        self.es_counter = 0\n\n    def reset_epoch(self):\n        self.running_loss = 0.0\n        self.running_metric = 0.0\n\n    def step(self, loss, inputs, preds, targets, phase):\n        self.running_loss += loss.item() * inputs.size(0)\n        self.running_metric += self.metric(preds, targets).sum()\n        self.t[phase] += 1\n\n    def log_epoch(self, phase):\n        self.epoch_loss[phase] = self.running_loss / self.dataset_sizes[phase]\n        self.epoch_metric[phase] = self.running_metric / self.dataset_sizes[phase]\n        self.bar.set_postfix(\n            a_train_loss=f'{self.epoch_loss[\"train\"]:0.1f}',\n            b_val_loss=f'{self.epoch_loss[\"val\"]:0.1f}',\n            c_train_metric=f'{self.epoch_metric[\"train\"]:0.4f}',\n            d_val_metric=f'{self.epoch_metric[\"val\"]:0.4f}',\n            es_counter=self.es_counter\n        )\n        self.w.add_scalar(\n            f'Loss/{phase}', self.epoch_loss[phase], self.e[phase])\n        self.w.add_scalar(\n            f'Accuracy/{phase}', self.epoch_metric[phase], self.e[phase])\n\n        self.e[phase] += 1\n\n        # Early stop and model backup\n        early_stop = False\n        if phase == 'val':\n            if self.epoch_loss['val'] < self.best_loss:\n                self.best_loss = self.epoch_loss['val']\n                self.best_model_wts = copy.deepcopy(self.model.state_dict())\n                torch.save(self.best_model_wts, self.model_file)\n                self.es_counter = 0\n            else:\n                self.es_counter += 1\n                if self.es_counter >= self.es_patience:\n                    early_stop = True\n                    self.bar.close()\n\n        return early_stop\n\n    @staticmethod\n    def metric(preds, targets):\n        sigma = preds[:, 2] - preds[:, 0]\n        sigma[sigma < 70] = 70\n        delta = (preds[:, 1] - targets).abs()\n        delta[delta > 1000] = 1000\n        return -np.sqrt(2) * delta / sigma - torch.log(np.sqrt(2) * sigma)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models = []\n\n# Load the data\ndata = ClinicalDataset(root_dir=root_dir, mode='train')\nfolds = data.group_kfold(num_kfolds)\nt0 = time()\n\nfor fold, (trainset, valset) in enumerate(folds):\n    # Prepare to save model weights\n    Path(model_dir).mkdir(parents=True, exist_ok=True)\n    now = datetime.now()\n    fname = f'{model_name}-{now.year}{now.month:02d}{now.day:02d}_{fold}.pth'\n    model_file = Path(model_dir) / fname\n\n    dataset_sizes = {'train': len(trainset), 'val': len(valset)}\n    dataloaders = {\n        'train': DataLoader(trainset, batch_size=batch_size,\n                            shuffle=True, num_workers=2),\n        'val': DataLoader(valset, batch_size=batch_size,\n                          shuffle=False, num_workers=2)\n    }\n\n    # Create the model and optimizer\n    device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n    model = QuantModel().to(device)\n    optimizer = Adam(model.parameters(), lr=learning_rate)\n    scheduler = StepLR(optimizer, step_size=20, gamma=0.5)\n    monitor = Monitor(\n        model=model,\n        es_patience=es_patience,\n        experiment_name=f'{model_name}_fold_{fold}',\n        tensorboard_dir=tensorboard_dir,\n        num_epochs=num_epochs,\n        dataset_sizes=dataset_sizes,\n        model_file=model_file\n    )\n\n    # Training loop\n    for epoch in monitor.bar:\n        for phase in ['train', 'val']:\n            if phase == 'train':\n                model.train()  # Set model to training mode\n            else:\n                model.eval()   # Set model to evaluate mode\n\n            monitor.reset_epoch()\n\n            # Iterate over data\n            for batch in dataloaders[phase]:\n                inputs = batch['features'].float().to(device)\n                targets = batch['target'].to(device)\n\n                # zero the parameter gradients\n                optimizer.zero_grad()\n                # forward\n                # track gradients if only in train\n                with torch.set_grad_enabled(phase == 'train'):\n                    preds = model(inputs)\n                    loss = quantile_loss(preds, targets, quantiles)\n                    # backward + optimize only if in training phase\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n\n                monitor.step(loss, inputs, preds, targets, phase)\n\n            # epoch statistics\n            early_stop = monitor.log_epoch(phase)\n\n        if early_stop:\n            break\n\n        # Updates the learning rate\n        scheduler.step()\n\n    # load best model weights\n    model.load_state_dict(monitor.best_model_wts)\n    models.append(model)\n\nprint(f'Training complete! Time: {timedelta(seconds=time() - t0)}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 6. Generating submission CSV","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"data = ClinicalDataset(root_dir, mode='test')\navg_preds = np.zeros((len(data), len(quantiles)))\n\nfor model in models:\n    dataloader = DataLoader(data, batch_size=batch_size,\n                            shuffle=False, num_workers=2)\n    preds = []\n    for batch in dataloader:\n        inputs = batch['features'].float()\n        with torch.no_grad():\n            x = model(inputs)\n            preds.append(x)\n\n    preds = torch.cat(preds, dim=0).numpy()\n    avg_preds += preds\n\navg_preds /= len(models)\ndf = pd.DataFrame(data=avg_preds, columns=list(quantiles))\ndf['Patient_Week'] = data.raw['Patient_Week']\ndf['FVC'] = df[quantiles[1]]\ndf['Confidence'] = df[quantiles[2]] - df[quantiles[0]]\ndf = df.drop(columns=list(quantiles))\ndf.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(df))\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}