{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Bayesian Neural Network for obtaining confidence by sampling","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Seemed like a nice problem for me to learn to use Bayesian Neural Networks with Pyro. I'll try to step through what's going on. With thanks to https://github.com/paraschopra/bayesian-neural-network-mnist/blob/master/bnn.ipynb**\nhttps://forum.pyro.ai/t/dealing-with-noise-in-bayesian-neural-network-regression/863\nhttps://github.com/Rachnog/Deep-Trading/blob/master/bayesian/Pyro%20-%20bayesian%20regression.ipynb","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"https://www.kaggle.com/raghaw/install-segmentation-model-offline-in-infer-kernel","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir -p /tmp/pip/cache/\n!cp ../input/pyroppl/pyro_ppl-1.3.1-py3-none-any.whl /tmp/pip/cache/\n!cp ../input/pyroapi/pyro_api-0.1.2-py3-none-any.whl /tmp/pip/cache/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install --no-index --find-links /tmp/pip/cache/ pyro-ppl==1.3.1","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n#Load the dependancies\nimport pyro\nfrom pyro.infer import EmpiricalMarginal, SVI, Trace_ELBO, TracePredictive\nfrom   torch.distributions import constraints\nfrom pyro.distributions import Normal, Categorical\nfrom pyro.infer import SVI, Trace_ELBO\nfrom pyro.optim import Adam\nfrom torch.autograd import Variable\nfrom tqdm.notebook import tqdm\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport torch\nimport pyro.distributions as dist\nimport torch.nn as nn\nfrom pyro.nn import PyroModule\nfrom torch import optim\nimport random\nfrom torchvision import models\nimport torch.multiprocessing as mp\nfrom pyro.nn import PyroSample\nfrom torch.nn import functional as F\nimport matplotlib.pyplot as plt\nplt.style.use('ggplot')\nimport pydicom.pixel_data_handlers.gdcm_handler as gdcm_handler \nimport cv2\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nimport pydicom\nimport os\nfrom torch.utils.data import DataLoader, Dataset\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n#    for filename in filenames:\n#        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs=100\nbatch_size=8\nnum_workers=3\nprec_alpha    = 3\nprec_beta     = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Net(torch.nn.Module):\n    def __init__(self, n_feature, n_hidden):\n        super(Net, self).__init__()\n        self.hidden1 = torch.nn.Linear(n_feature, n_hidden) \n        self.hidden2 = torch.nn.Linear(n_hidden, n_hidden)# hidden layer\n        self.predict = torch.nn.Linear(n_hidden, 1)   # output layer\n\n    def forward(self, x):\n        x = self.hidden1(x)\n        x = self.hidden2(x)\n        x = self.predict(x)\n        return x\n    \ndef model(X,y):\n    num_x       = X.shape[0]\n    mu = Variable(torch.zeros(n_hidden, n_features))\n    sigma = Variable(torch.ones(n_hidden, n_features))\n    bias_mu = Variable(torch.zeros(n_hidden))\n    bias_sigma = Variable(torch.ones(n_hidden))\n    w_prior, b_prior = Normal(mu, sigma), Normal(bias_mu, bias_sigma)\n    \n    mu2 = Variable(torch.zeros(n_hidden, n_hidden))\n    sigma2 = Variable(torch.ones(n_hidden, n_hidden))\n    bias_mu2 = Variable(torch.zeros(n_hidden))\n    bias_sigma2 = Variable(torch.ones(n_hidden))\n    w_prior2, b_prior2 = Normal(mu2, sigma2), Normal(bias_mu2, bias_sigma2) \n    \n    mu3 = Variable(torch.zeros(1, n_hidden))\n    sigma3 = Variable(torch.ones(1, n_hidden))\n    bias_mu3 = Variable(torch.zeros(1))\n    bias_sigma3 = Variable(torch.ones(1))\n    w_prior3, b_prior3 = Normal(mu3, sigma3), Normal(bias_mu3, bias_sigma3)   \n    \n    priors = {'hidden1.weight': w_prior, \n              'hidden1.bias': b_prior,\n              'hidden2.weight': w_prior2, \n              'hidden3.bias': b_prior2,\n              'predict.weight': w_prior3,\n              'predict.bias': b_prior3}\n    \n    # lift module parameters to random variables sampled from the priors\n    lifted_module = pyro.random_module(\"module\", regression_model, priors)\n    # sample a regressor (which also samples w and b)\n    lifted_reg_model = lifted_module()\n    precision   = pyro.sample(\"precision\", pyro.distributions.Gamma(prec_alpha, prec_beta))\n    noise_scale = 1 / precision.sqrt()\n\n    with pyro.plate(\"map\", len(X), subsample_size = min(num_x, batch_size)) as ind:\n            prediction_mean = lifted_reg_model(X[ind]).squeeze(-1)\n            pyro.sample(\"obs\", \n                    pyro.distributions.Normal(prediction_mean, Variable(torch.ones(X.shape[0])*noise_scale)), \n                    obs = y[ind])\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def guide(X,y):\n    alpha     = pyro.param(\"alpha\", torch.tensor(prec_alpha), constraint = constraints.positive)\n    beta      = pyro.param(\"beta\",  torch.tensor(prec_beta),  constraint = constraints.positive)\n    precision = pyro.sample(\"precision\", pyro.distributions.Gamma(alpha, beta))\n\n    w_mu = Variable(torch.randn(n_hidden, n_features), requires_grad=True)\n    w_log_sig = Variable((torch.ones(n_hidden, n_features) + 0.05 * torch.randn(n_hidden, n_features)), requires_grad=True)\n    b_mu = Variable(torch.randn(n_hidden), requires_grad=True)\n    b_log_sig = Variable((torch.ones(n_hidden) + 0.05 * torch.randn(n_hidden)), requires_grad=True)\n    \n    # register learnable params in the param store\n    mw_param = pyro.param(\"guide_mean_weight\", w_mu)\n    sw_param = softplus(pyro.param(\"guide_log_sigma_weight\", w_log_sig))\n    mb_param = pyro.param(\"guide_mean_bias\", b_mu)\n    sb_param = softplus(pyro.param(\"guide_log_sigma_bias\", b_log_sig))\n    \n    # gaussian guide distributions for w and b\n    w_dist = Normal(mw_param, sw_param)\n    b_dist = Normal(mb_param, sb_param)\n    \n#     w_mu2 = Variable(torch.randn(1, second_layer).type_as(data.data), requires_grad=True)\n#     w_log_sig2 = Variable(torch.randn(1, second_layer).type_as(data.data), requires_grad=True)\n#     b_mu2 = Variable(torch.randn(1).type_as(data.data), requires_grad=True)\n#     b_log_sig2 = Variable(torch.randn(1).type_as(data.data), requires_grad=True)\n    \n    w_mu2 = Variable(torch.randn(n_hidden, n_hidden), requires_grad=True)\n    w_log_sig2 = Variable((torch.ones(n_hidden, n_hidden) + 0.05 * torch.randn(n_hidden, n_hidden)), requires_grad=True)\n    b_mu2 = Variable(torch.randn(n_hidden), requires_grad=True)\n    b_log_sig2 = Variable((torch.ones(n_hidden) + 0.05 * torch.randn(n_hidden)), requires_grad=True)\n    \n    # register learnable params in the param store\n    mw_param2 = pyro.param(\"guide_mean_weight2\", w_mu2)\n    sw_param2 = softplus(pyro.param(\"guide_log_sigma_weight2\", w_log_sig2))\n    mb_param2 = pyro.param(\"guide_mean_bias2\", b_mu2)\n    sb_param2 = softplus(pyro.param(\"guide_log_sigma_bias2\", b_log_sig2))\n    \n    # gaussian guide distributions for w and b\n    w_dist2 = Normal(mw_param2, sw_param2)\n    b_dist2 = Normal(mb_param2, sb_param2)\n    \n    w_mu3 = Variable(torch.randn(1, n_hidden), requires_grad=True)\n    w_log_sig3 = Variable((torch.ones(1, n_hidden) + 0.05 * torch.randn(1, n_hidden)), requires_grad=True)\n    b_mu3 = Variable(torch.randn(1), requires_grad=True)\n    b_log_sig3 = Variable((torch.ones(1) + 0.05 * torch.randn(1)), requires_grad=True)\n    \n    # register learnable params in the param store\n    mw_param3 = pyro.param(\"guide_mean_weight3\", w_mu3)\n    sw_param3 = softplus(pyro.param(\"guide_log_sigma_weight3\", w_log_sig3))\n    mb_param3 = pyro.param(\"guide_mean_bias3\", b_mu3)\n    sb_param3 = softplus(pyro.param(\"guide_log_sigma_bias3\", b_log_sig3))\n    \n    # gaussian guide distributions for w and b\n    w_dist3 = Normal(mw_param3, sw_param3)\n    b_dist3 = Normal(mb_param3, sb_param3)\n      \n    dists = {'hidden1.weight': w_dist, \n              'hidden1.bias': b_dist,\n             'hidden2.weight': w_dist2, \n              'hidden2.bias': b_dist2,\n              'predict.weight': w_dist3,\n              'predict.bias': b_dist3}\n    \n    # overloading the parameters in the module with random samples from the guide distributions\n    lifted_module = pyro.random_module(\"module\", regression_model, dists)\n    # sample a regressor\n    return lifted_module()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Metrics\nWe will use these to measure the model performance in terms we understand but we will optimize for the Evidence Lower Bound (ELBO)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def metric_loss(pred_fvc,true_fvc,pred_sigma):\n    true_fvc=torch.reshape(true_fvc,pred_fvc.shape)\n    sigma_clipped=torch.clamp(pred_sigma,min=70)\n    delta=torch.clamp(torch.abs(pred_fvc-true_fvc),max=1000)\n    metric=torch.div(-torch.sqrt(torch.tensor([2.0]).to(device))*delta,sigma_clipped)-torch.log(torch.sqrt(torch.tensor([2.0]).to(device))*sigma_clipped)\n    return -metric\n\ndef fvc_loss(pred_fvc,true_fvc):\n    true_fvc=torch.reshape(true_fvc,pred_fvc.shape)\n    fvc_err=torch.abs(pred_fvc-true_fvc)\n    return fvc_err","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_training_loss(train, val,title='loss'):\n    plt.figure()\n    plt.plot(train, label='Train')\n    plt.plot(val, label='Val')\n    if title=='loss':\n        plt.title('Model Training Loss')\n    else:\n        plt.title('Model Metric Loss')\n    plt.xlabel('Epoch')\n    plt.ylabel('Loss')\n    plt.legend()\n    plt.savefig('training_loss')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prepare Data\nAll in the same way as my tabular pytorch model","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Load Dataframes","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train=pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv')\ntest=pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\nsubmission=pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Prepare Training Data (Tabular)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train['base_Weeks']=train.groupby(['Patient'])['Weeks'].transform('min')\nbase=train[train.Weeks==train.base_Weeks]\nbase = base.rename(columns={'FVC': 'base_FVC','Percent': 'base_Percent'})\nbase.drop_duplicates(subset=['Patient', 'Weeks'], keep='first',inplace=True)\ntrain=train.merge(base[['Patient','base_FVC','base_Percent']],on='Patient',how='left')\ntrain['Week_passed'] = train['Weeks'] - train['base_Weeks']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Prepare Test Data (tabular)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test = test.rename(columns={'Weeks': 'base_Weeks', 'FVC': 'base_FVC','Percent': 'base_Percent'})\n\n# Adding Sample Submission\nsubmission = pd.read_csv(\"../input/osic-pulmonary-fibrosis-progression/sample_submission.csv\")\n\n# In submisison file, format: ID_'week', using lambda to split the ID\nsubmission['Patient'] = submission['Patient_Week'].apply(lambda x:x.split('_')[0])\n\n# In submisison file, format: ID_'week', using lambda to split the Week\nsubmission['Weeks'] = submission['Patient_Week'].apply(lambda x:x.split('_')[1]).astype(int)\n\ntest = submission.drop(columns = [\"FVC\", \"Confidence\"]).merge(test, on = 'Patient')\n\ntest['Week_passed'] = test['Weeks'] - test['base_Weeks']\n\ntest=test[train.columns.drop(['FVC','Percent'])]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### OH Encode Sex and Smoking\nWith thanks to https://www.kaggle.com/ulrich07/osic-keras-starter-with-custom-metrics","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"COLS = ['Sex','SmokingStatus']\nfor col in COLS:\n    for mod in train[col].unique():\n        train[mod] = (train[col] == mod).astype(int)\n        \n        test[mod] = (test[col] == mod).astype(int)\n    train.drop(col,axis=1,inplace=True)\n    test.drop(col,axis=1,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Rescale based on train data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import preprocessing\nrobust_scaler = preprocessing.RobustScaler()\ntrain[train.columns.difference(['Patient','FVC','Percent','Weeks','base_Weeks'])]=robust_scaler.fit_transform(train[train.columns.difference(['Patient','FVC','Percent','Weeks','base_Weeks'])])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### I've used this to avoid the files dcmread can't load (future version with CNN)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import pickle\nwith open('../input/file-dictionary/good_files.pickle', 'rb') as handle:\n    good_file_dict = pickle.load(handle)\nwith open('../input/file-dictionary/bad_files.pickle', 'rb') as handle:\n    bad_file_dict = pickle.load(handle)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class OSIC(Dataset):\n    def __init__(self,patient_ids,df,file_dict,train=True, transform=None,nims=10):\n        self.df=df[df.Patient.isin(patient_ids)]\n        self.train=train\n        if self.train:\n            self.fvc=self.df['FVC'].values\n        else:\n            self.df[self.df.columns.difference(['Patient','FVC','Percent','Weeks','base_Weeks'])]=robust_scaler.transform(self.df[self.df.columns.difference(['Patient','FVC','Percent','Weeks','base_Weeks'])])\n    \n        self.data=self.df[self.df.columns.difference(['FVC','Patient','Percent'])].values\n        self.patients=self.df['Patient'].values\n        self.file_dict=file_dict\n        self.nims=nims\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        if self.train:\n            data = {'fvc': self.fvc[idx],\n                   'data': self.data[idx]}\n        else:\n            data = {'data': self.data[idx]}\n        return data","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Split training data into train and val by patient (80:20)\n'ID00011637202177653955184' has no images we can load. Therefore I'm going to drop.\n\nWe shuffle the train data","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Set up datasets and dataloaders","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"ids=train.Patient.unique()\nindex = np.argwhere(ids=='ID00011637202177653955184')\nids = list(np.delete(ids, index))\nrandom.shuffle(ids)\nids=np.array(ids)\n\ntrain_ids,val_ids=np.split(ids, [int(round(0.9 * len(ids), 0))])\n\ntrain_dataset = OSIC(train_ids,train,good_file_dict)  \ntrain_dataloader = DataLoader(train_dataset, batch_size=batch_size,shuffle=True,num_workers=num_workers)\n\nval_dataset = OSIC(val_ids,train,good_file_dict)  \nval_dataloader = DataLoader(val_dataset, batch_size=batch_size,shuffle=True,num_workers=num_workers)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Get the correct shaped model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"n_features = train_dataset.data.shape[1]\nn_hidden = 100\n\nsoftplus = nn.Softplus()\nregression_model = Net(n_features, n_hidden)\n\nprint('Number of parameters:')\nprint(sum(p.numel() for p in regression_model.parameters() if p.requires_grad))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Set up a prediction function that takes the average of a number of sampled models and returns the mean and standard deviation of the outputs","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict(x,num_samples):\n    sampled_models = [guide(None, None) for _ in range(num_samples)]\n    yhats = [sample_model(x) for sample_model in sampled_models]\n    mean = torch.mean(torch.stack(yhats), 0)\n    std = torch.std(torch.stack(yhats), 0)\n    return mean,std","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from pyro.infer import SVI, Trace_ELBO\n\n\nadam = pyro.optim.Adam({\"lr\": 0.03})\nsvi = SVI(model, guide, adam, loss=Trace_ELBO())\npyro.clear_param_store()\nepoch_val_metric=[]\nepoch_train_fvc=[]\nepoch_train_loss=[]\n#Start by training for fvc\nfor epoch in range(epochs):\n    train_loss=0\n    train_fvc=0\n    val_metric=0\n    val_fvc=0\n    for batch_idx, data in enumerate(train_dataloader):\n        svi_loss = svi.step(data['data'].float(), data['fvc'].float())\n        mean,std = predict(data['data'].float(),10)\n        train_fvc += fvc_loss(mean,data['fvc']).mean().item()\n        train_loss=svi_loss\n    print('====> Epoch: {} Average train loss ELBO: {:.4f}'.format(\n                        epoch, train_loss / len(train_dataloader)))\n    print('====> Epoch: {} Average train fvc absolute loss: {:.4f}'.format(\n                        epoch, train_fvc / len(train_dataloader)))\n    epoch_train_loss.append(train_loss/ len(train_dataloader))\n    epoch_train_fvc.append(train_fvc)\n    \n    for batch_idx, data in enumerate(val_dataloader):\n        mean,std = predict(data['data'].float(),10)\n        val_metric += metric_loss(mean,data['fvc'],std).mean().item()\n        val_fvc += fvc_loss(mean,data['fvc']).mean().item()\n    print('====> Epoch: {} Average val metric: {:.4f}'.format(\n                        epoch, val_metric / len(val_dataloader)))\n    print('====> Epoch: {} Average val fvc absolute loss: {:.4f}'.format(\n                        epoch, val_fvc / len(val_dataloader)))\n    epoch_val_metric.append(val_metric/ len(val_dataloader))\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for name, value in pyro.get_param_store().items():\n    print(name, pyro.param(name))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Plot training curves","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(epoch_train_loss)\nplt.title('ELBO loss on train data')\nplt.ylabel('Trace ELBO')\nplt.xlabel('Epochs')\nplt.yscale('log')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(epoch_val_metric)\nplt.title('Metric on Validation Data with ' + str(10) + \" samples\")\nplt.ylabel('Evaluation Metric')\nplt.xlabel('Epochs')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Test Data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"submission=pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ids=test.Patient.unique()\ntest_dataset = OSIC(test_ids,test,good_file_dict,train=False)  \ntest_dataloader = DataLoader(test_dataset, batch_size=batch_size,shuffle=False,num_workers=num_workers)\nfvc_pred = []\nsigma_pred = []\nwith torch.no_grad():\n    for batch_idx, data in enumerate(test_dataloader):\n        mean,std = predict(data['data'].float(),50)\n        fvc_pred.append(mean)\n        sigma_pred.append(std)\nfvc_pred=torch.cat(fvc_pred, dim=0)\nsigma_pred=torch.cat(sigma_pred, dim=0)\ntest['FVC']=fvc_pred.cpu().numpy()\ntest['Confidence']=sigma_pred.cpu().numpy()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test['Patient_Week']=test[\"Patient\"] + '_' + test['Weeks'].apply(str)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission=submission[['Patient_Week']].merge(test[['Patient_Week','FVC','Confidence']],on='Patient_Week')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False, float_format='%.1f')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Post-Match Analysis","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Test Predictions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.scatter(submission['FVC'],submission['Confidence'])\nplt.title('Test')\nplt.xlabel('FVC')\nplt.ylabel('Confidence')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fvc_pred = []\nsigma_pred = []\nwith torch.no_grad():\n    for batch_idx, data in enumerate(train_dataloader):\n        mean,std = predict(data['data'].float(),30)\n        fvc_pred.append(mean)\n        sigma_pred.append(std)\nfvc_pred_train=torch.cat(fvc_pred, dim=0)\nsigma_pred_train=torch.cat(sigma_pred, dim=0)\nplt.scatter(fvc_pred_train.cpu().numpy(),sigma_pred_train.cpu().numpy())\nplt.title('Train Confidence vs. FVC')\nplt.xlabel('FVC')\nplt.ylabel('Confidence')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.scatter(train_dataset.fvc,fvc_pred_train.cpu().numpy())\nplt.title('Train: predicted FVC vs true FVC')\nplt.xlabel('True FVC')\nplt.ylabel('Predicted FVC')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Val","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fvc_pred = []\nsigma_pred = []\nwith torch.no_grad():\n    for batch_idx, data in enumerate(val_dataloader):\n        mean,std = predict(data['data'].float(),30)\n        fvc_pred.append(mean)\n        sigma_pred.append(std)\nfvc_pred_val=torch.cat(fvc_pred, dim=0)\nsigma_pred_val=torch.cat(sigma_pred, dim=0)\nplt.scatter(fvc_pred_val.cpu().numpy(),sigma_pred_val.cpu().numpy())\nplt.title('Val')\nplt.xlabel('FVC')\nplt.ylabel('Confidence')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.scatter(val_dataset.fvc,fvc_pred_val.cpu().numpy())\nplt.title('Val: predicted FVC vs true FVC')\nplt.xlabel('True FVC')\nplt.ylabel('Predicted FVC')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## All","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(submission['FVC'], alpha=0.5,label='test')\nplt.hist(fvc_pred_train.cpu().numpy(), alpha=0.5,label='train')\nplt.hist(fvc_pred_val.cpu().numpy(), alpha=0.5,label='val')\nplt.legend()\nplt.title('Histogram of FVC predictions')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(submission['Confidence'], alpha=0.5,label='test')\nplt.hist(sigma_pred_train.cpu().numpy(), alpha=0.5,label='train')\nplt.hist(sigma_pred_val.cpu().numpy(), alpha=0.5,label='val')\nplt.legend()\nplt.title('Histogram of Confidence predictions')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}