{"cells":[{"metadata":{},"cell_type":"markdown","source":"# 1. Quantile Regression PyTorch (tabular data only)\nThis notebook generates baseline predictions using tabular data only. I created it to better understand the data.\n\nIt is a simplified version of [this great notebook](Osic-Multiple-Quantile-Regression-Starter) from [Ulrich GOUE\n](https://www.kaggle.com/ulrich07). It also builds on this great [tutorial about Quantile Regression for neural networks](https://medium.com/the-artificial-impostor/quantile-regression-part-2-6fdbc26b2629).\n\nChanges vs previous version:\n- Factored monitoring code out of training block\n- Implemented early stopping\n- Implemented learning rate decay scheduler","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Changed from previous version\n* Prediction Cuda error\n* Model change\n","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"\n# 2. Imports and global variables","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import copy\nfrom datetime import datetime, timedelta\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nfrom pathlib import Path\nfrom sklearn.model_selection import GroupKFold\nfrom torch.utils.data import Dataset\nfrom tqdm import tqdm\nimport torch\nfrom torch.utils.data import DataLoader, Subset\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import Adam\nfrom torch.optim.lr_scheduler import StepLR\nfrom torch.utils.tensorboard import SummaryWriter\nfrom tqdm.notebook import trange\nfrom time import time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"root_dir = Path('/kaggle/input/osic-pulmonary-fibrosis-progression')\nmodel_dir = '/kaggle/working/model_states'\nnum_kfolds = 5\nbatch_size = 32\nlearning_rate = 3e-3\nnum_epochs = 1000\nes_patience = 10\nquantiles = (0.2, 0.5, 0.8)\nmodel_name ='descartes'\ntensorboard_dir = Path('/kaggle/working/runs')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3. Dataset interface","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"class ClinicalDataset(Dataset):\n    def __init__(self, root_dir, mode, transform=None):\n        self.transform = transform\n        self.mode = mode\n\n        tr = pd.read_csv(Path(root_dir)/\"train.csv\")\n        tr.drop_duplicates(keep=False, inplace=True, subset=['Patient', 'Weeks'])\n        chunk = pd.read_csv(Path(root_dir)/\"test.csv\")\n\n        sub = pd.read_csv(Path(root_dir)/\"sample_submission.csv\")\n        sub['Patient'] = sub['Patient_Week'].apply(lambda x: x.split('_')[0])\n        sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))\n        sub = sub[['Patient', 'Weeks', 'Confidence', 'Patient_Week']]\n        sub = sub.merge(chunk.drop('Weeks', axis=1), on=\"Patient\")\n\n        tr['WHERE'] = 'train'\n        chunk['WHERE'] = 'val'\n        sub['WHERE'] = 'test'\n        data = tr.append([chunk, sub])\n\n        data['min_week'] = data['Weeks']\n        data.loc[data.WHERE == 'test', 'min_week'] = np.nan\n        data['min_week'] = data.groupby('Patient')['min_week'].transform('min')\n\n        base = data.loc[data.Weeks == data.min_week]\n        base = base[['Patient', 'FVC']].copy()\n        base.columns = ['Patient', 'min_FVC']\n        base['nb'] = 1\n        base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')\n        base = base[base.nb == 1]\n        base.drop('nb', axis=1, inplace=True)\n\n        data = data.merge(base, on='Patient', how='left')\n        data['base_week'] = data['Weeks'] - data['min_week']\n        del base\n\n        COLS = ['Sex', 'SmokingStatus']\n        self.FE = []\n        for col in COLS:\n            for mod in data[col].unique():\n                self.FE.append(mod)\n                data[mod] = (data[col] == mod).astype(int)\n\n        data['age'] = (data['Age'] - data['Age'].min()) / \\\n                      (data['Age'].max() - data['Age'].min())\n        data['BASE'] = (data['min_FVC'] - data['min_FVC'].min()) / \\\n                       (data['min_FVC'].max() - data['min_FVC'].min())\n        data['week'] = (data['base_week'] - data['base_week'].min()) / \\\n                       (data['base_week'].max() - data['base_week'].min())\n        data['percent'] = (data['Percent'] - data['Percent'].min()) / \\\n                          (data['Percent'].max() - data['Percent'].min())\n        self.FE += ['age', 'percent', 'week', 'BASE']\n\n        self.raw = data.loc[data.WHERE == mode].reset_index()\n        del data\n\n    def __len__(self):\n        return len(self.raw)\n\n    def __getitem__(self, idx):\n        if torch.is_tensor(idx):\n            idx = idx.tolist()\n\n        sample = {\n            'patient_id': self.raw['Patient'].iloc[idx],\n            'features': self.raw[self.FE].iloc[idx].values,\n            'target': self.raw['FVC'].iloc[idx]\n        }\n        if self.transform:\n            sample = self.transform(sample)\n\n        return sample\n\n    def group_kfold(self, n_splits):\n        gkf = GroupKFold(n_splits=n_splits)\n        groups = self.raw['Patient']\n        for train_idx, val_idx in gkf.split(self.raw, self.raw, groups):\n            train = Subset(self, train_idx)\n            val = Subset(self, val_idx)\n            yield train, val\n\n    def group_split(self, test_size=0.2):\n        \"\"\"To test no-kfold\n        \"\"\"\n        gss = GroupShuffleSplit(n_splits=1, test_size=test_size)\n        groups = self.raw['Patient']\n        idx = list(gss.split(self.raw, self.raw, groups))\n        train = Subset(self, idx[0][0])\n        val = Subset(self, idx[0][1])\n        return train, val","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 4. Neural Net model and Quantile loss","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class QuantModel(nn.Module):\n    def __init__(self, in_tabular_features=9, out_quantiles=3):\n        super(QuantModel, self).__init__()\n        self.fc1 = nn.Linear(in_tabular_features, 200)\n        self.fc2 = nn.Linear(200, 100)\n        self.fc3 = nn.Linear(100, out_quantiles)\n\n    def forward(self, x):\n        x = F.leaky_relu(self.fc1(x))\n        #x = self.bn1(x)\n        x = F.leaky_relu(self.fc2(x))\n        x = self.fc3(x)\n        return x\n\n\ndef quantile_loss(preds, target, quantiles):\n    #assert not target.requires_grad\n    assert len(preds) == len(target)\n    losses = []\n    for i, q in enumerate(quantiles):\n        errors = target - preds[:, i]\n        losses.append(torch.max((q - 1) * errors, q * errors).unsqueeze(1))\n    loss = torch.mean(torch.sum(torch.cat(losses, dim=1), dim=1))\n    return loss\n\ndef metric_loss(pred_fvc,true_fvc):\n        #Implementation of the metric in pytorch\n    sigma = pred_fvc[:, 2] - pred_fvc[:, 0]\n    true_fvc=torch.reshape(true_fvc,pred_fvc[:,1].shape)\n    sigma_clipped=torch.clamp(sigma,min=70)\n    delta=torch.clamp(torch.abs(pred_fvc[:,1]-true_fvc),max=1000)\n    metric=torch.div(-torch.sqrt(torch.tensor([2.0]))*delta,sigma_clipped)-torch.log(torch.sqrt(torch.tensor([2.0]))*sigma_clipped)\n    return metric","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 5. Training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"models = []\n\ntrain_loss = []\nval_lll = []\n# Load the data\ndata = ClinicalDataset(root_dir=root_dir, mode='train')\nfolds = data.group_kfold(num_kfolds)\n#t0 = time()\n#if len(testfiles) == 5:\n    #f= open(\"/kaggle/working/training.log\",\"w+\") \nfor fold, (trainset, valset) in enumerate(folds):\n    best_val = None\n    patience = es_patience\n    model = QuantModel().to(device)\n    params = [p for p in model.parameters() if p.requires_grad]\n    optimizer = Adam(params, lr=learning_rate)\n# lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.1)\n    lr_scheduler = StepLR(optimizer, step_size=20, gamma=0.5)\n\n\n    print(\"==\"*20+\"Fold \"+str(fold+1)+\"==\"*20)\n    Path(model_dir).mkdir(parents=True, exist_ok=True)\n    model_path = model_dir+f'/fold_{fold}.pth'\n    now = datetime.now()\n    dataset_sizes = {'train': len(trainset), 'val': len(valset)}\n    dataloaders = {\n            'train': DataLoader(trainset, batch_size=batch_size,\n                                shuffle=True, num_workers=2),\n            'val': DataLoader(valset, batch_size=batch_size,\n                              shuffle=False, num_workers=2)\n    }\n    train_loss_epoch = []\n    val_lll_epoch = []\n    for epoch in range(num_epochs):\n        start_time = time()\n        itr = 1\n        model.train()\n        train_losses =[]\n        for batch in dataloaders['train']:\n            inputs = batch['features'].float().to(device)\n            targets = batch['target'].to(device)\n            optimizer.zero_grad()\n            with torch.set_grad_enabled(True):\n                preds = model(inputs)\n                loss = quantile_loss(preds, targets, quantiles)\n                train_losses.append(loss.tolist())\n                loss.backward()\n                optimizer.step()\n           \n            if itr % 50 == 0:\n                print(f\"Epoch #{epoch+1} Iteration #{itr} loss: {loss}\")\n            itr += 1\n            \n        model.eval()\n        all_preds = []\n        all_targets = []\n        for batch in dataloaders['val']:\n            inputs = batch['features'].float().to(device)\n            targets = batch['target']\n            optimizer.zero_grad()\n            with torch.set_grad_enabled(False):\n                preds = model(inputs)\n                all_preds.extend(preds.detach().cpu().numpy().tolist())\n                all_targets.extend(targets.numpy().tolist()) # np.append(an_array, row_to_append, 0)\n        all_preds =torch.FloatTensor(all_preds)\n        all_targets =torch.FloatTensor(all_targets)\n        val_metric_loss = metric_loss(all_preds, all_targets)\n        val_metric_loss = torch.mean(val_metric_loss).tolist()\n\n        lr_scheduler.step()\n        print(f\"Epoch #{epoch+1}\",\"Training loss : {0:.4f}\".format(np.mean(train_losses)),\"Validation LLL : {0:.4f}\".format(val_metric_loss),\"Time taken :\",str(timedelta(seconds=time() - start_time))[:7])\n        train_loss_epoch.append(np.mean(train_losses))\n        val_lll_epoch.append(val_metric_loss)\n        if not best_val:\n            best_val = val_metric_loss  # So any validation roc_auc we have is the best one for now\n            print(\"Info : Saving model\")\n            torch.save(copy.deepcopy(model.state_dict()), model_path)  # Saving the model\n        if val_metric_loss > best_val:\n            print(\"Info : Saving model as Laplace Log Likelihood is increased from {0:.4f}\".format(best_val),\"to {0:.4f}\".format(val_metric_loss))\n            best_val = val_metric_loss\n            patience = es_patience  # Resetting patience since we have new best validation accuracy\n            torch.save(copy.deepcopy(model.state_dict()), model_path)  # Saving current best model torch.save(model.state_dict(), 'fasterrcnn_resnet50_fpn.pth')\n        else:\n            patience -= 1\n            if patience == 0:\n                print('Early stopping. Best Validation Laplace Log Likelihood: {:.3f}'.format(best_val))\n                break\n    model.load_state_dict(torch.load(model_path))\n    models.append(model)\n    train_loss.append(train_loss_epoch)\n    val_lll.append(val_lll_epoch)\nprint('Finished Training of BiLSTM Model')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 6. Generating submission CSV","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"data = ClinicalDataset(root_dir, mode='test')\n\navg_preds = np.zeros((len(data), len(quantiles)))\n\nfor model in models:\n    dataloader = DataLoader(data, batch_size=batch_size,\n                            shuffle=False, num_workers=2)\n    preds = []\n    for batch in dataloader:\n        inputs = batch['features'].float()\n        inputs = inputs.cuda()\n        with torch.no_grad():\n            x = model(inputs)\n            preds.append(x)\n\n    preds = torch.cat(preds, dim=0).cpu().numpy()\n    avg_preds += preds\n\navg_preds /= len(models)\ndf = pd.DataFrame(data=avg_preds, columns=list(quantiles))\ndf['Patient_Week'] = data.raw['Patient_Week']\ndf['FVC'] = df[quantiles[1]]\ndf['Confidence'] = df[quantiles[2]] - df[quantiles[0]]\ndf = df.drop(columns=list(quantiles))\ndf.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(df))\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.to_csv('submission.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!head 'submission.csv'","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}