{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Overview\nThis is a **Pytorch Reproducible** version of [Ulrich GOUE's Osic-Multiple-Quantile-Regression-Starter](https://www.kaggle.com/ulrich07/osic-multiple-quantile-regression-starter), almost everything follow Ulrich GOUE's notebook, partially modified below:\n- change loss function\n- save best model on valid score and then load best model to predict\n- add learning rate scheduler\n\nThe notebook can't reach Ulrich GOUE's score(-6.8322), I could not find the reason. If you find anything, please tell me. It is my first notebook, upvote if you like it.","execution_count":null},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.execute_input":"2020-09-04T12:17:06.619270Z","iopub.status.busy":"2020-09-04T12:17:06.610518Z","iopub.status.idle":"2020-09-04T12:17:09.129953Z","shell.execute_reply":"2020-09-04T12:17:09.129201Z"},"papermill":{"duration":2.536187,"end_time":"2020-09-04T12:17:09.130117","exception":false,"start_time":"2020-09-04T12:17:06.593930","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import KFold\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom glob import glob\nimport gc\nimport os\nimport math\nimport random\nfrom tqdm import tqdm\nfrom IPython.core.interactiveshell import InteractiveShell\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler\nfrom sklearn.metrics import mean_absolute_error\nimport warnings\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom torch.optim import Adam, AdamW\n%matplotlib inline\n\nwarnings.filterwarnings(\"ignore\")\nInteractiveShell.ast_node_interactivity = \"all\"","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.007036,"end_time":"2020-09-04T12:17:09.145191","exception":false,"start_time":"2020-09-04T12:17:09.138155","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Settings","execution_count":null},{"metadata":{"execution":{"iopub.execute_input":"2020-09-04T12:17:09.167971Z","iopub.status.busy":"2020-09-04T12:17:09.167306Z","iopub.status.idle":"2020-09-04T12:17:09.173893Z","shell.execute_reply":"2020-09-04T12:17:09.173367Z"},"papermill":{"duration":0.021932,"end_time":"2020-09-04T12:17:09.173992","exception":false,"start_time":"2020-09-04T12:17:09.152060","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"BATCH_SIZE = 128\nEPOCHS = 800\nLR = 1e-1\nFOLDER = 5\nSEED = 42\nDEVICE = 'cuda'\nQUANTILE = [0.2, 0.5, 0.8]\nSAVE_AND_LOAD_BEST_MODEL = True    # You can turn on this for higher score\nLR_Scheduler = False               # You can turn on this for higher score\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.007008,"end_time":"2020-09-04T12:17:09.189579","exception":false,"start_time":"2020-09-04T12:17:09.182571","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Data Preparing","execution_count":null},{"metadata":{"execution":{"iopub.execute_input":"2020-09-04T12:17:09.295445Z","iopub.status.busy":"2020-09-04T12:17:09.285246Z","iopub.status.idle":"2020-09-04T12:17:09.498844Z","shell.execute_reply":"2020-09-04T12:17:09.499285Z"},"papermill":{"duration":0.302722,"end_time":"2020-09-04T12:17:09.499441","exception":false,"start_time":"2020-09-04T12:17:09.196719","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"ROOT = \"../input/osic-pulmonary-fibrosis-progression\"\n\ntr = pd.read_csv(f\"{ROOT}/train.csv\")\ntr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])\nchunk = pd.read_csv(f\"{ROOT}/test.csv\")\n\nsub = pd.read_csv(f\"{ROOT}/sample_submission.csv\")\nsub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])\nsub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))\nsub =  sub[['Patient','Weeks','Confidence','Patient_Week']]\nsub = sub.merge(chunk.drop('Weeks', axis=1), on=\"Patient\", how='left')\n\ntr['WHERE'] = 'train'\nchunk['WHERE'] = 'val'\nsub['WHERE'] = 'test'\ndata = tr.append([chunk, sub])\n\nprint(\"Origin Shape: \")\nprint(tr.shape, chunk.shape, sub.shape, data.shape)\nprint(tr.Patient.nunique(), chunk.Patient.nunique(), sub.Patient.nunique(), data.Patient.nunique())\n\ndata['min_week'] = data['Weeks']\ndata.loc[data.WHERE=='test','min_week'] = np.nan\ndata['min_week'] = data.groupby('Patient')['min_week'].transform('min')\n\nbase = data.loc[data.Weeks == data.min_week]\nbase = base[['Patient','FVC']].copy()\nbase.columns = ['Patient','min_FVC']\nbase['nb'] = 1\nbase['nb'] = base.groupby('Patient')['nb'].transform('cumsum')\nbase = base[base.nb==1]\nbase.drop('nb', axis=1, inplace=True)\n\ndata = data.merge(base, on='Patient', how='left')\ndata['base_week'] = data['Weeks'] - data['min_week']\ndel base\n\nCOLS = ['Sex','SmokingStatus']\nFE = []\nfor col in COLS:\n    for mod in data[col].unique():\n        FE.append(mod)\n        data[mod] = (data[col] == mod).astype(int)\n\ndata['age'] = (data['Age'] - data['Age'].min() ) / ( data['Age'].max() - data['Age'].min() )\ndata['BASE'] = (data['min_FVC'] - data['min_FVC'].min() ) / ( data['min_FVC'].max() - data['min_FVC'].min() )\ndata['week'] = (data['base_week'] - data['base_week'].min() ) / ( data['base_week'].max() - data['base_week'].min() )\ndata['percent'] = (data['Percent'] - data['Percent'].min() ) / ( data['Percent'].max() - data['Percent'].min() )\nFE += ['age','percent','week','BASE']\nINPUT_FEATURES = len(FE)\nprint(\"\\nFE: \", FE, \n      \"\\nINPUT_FEATURES: \", INPUT_FEATURES)\n\ntr = data.loc[data.WHERE=='train']\nchunk = data.loc[data.WHERE=='val']\nsub = data.loc[data.WHERE=='test']\ndel data, chunk\n\nX_train = tr[FE].values\nY_train = tr['FVC'].values\nX_test = sub[FE].values\nprint(\"\\nAfter Shape: \")\nprint(X_train.shape, Y_train.shape, X_test.shape)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.00682,"end_time":"2020-09-04T12:17:09.513724","exception":false,"start_time":"2020-09-04T12:17:09.506904","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Necessary Class","execution_count":null},{"metadata":{"execution":{"iopub.execute_input":"2020-09-04T12:17:09.551415Z","iopub.status.busy":"2020-09-04T12:17:09.546240Z","iopub.status.idle":"2020-09-04T12:17:09.563002Z","shell.execute_reply":"2020-09-04T12:17:09.563545Z"},"papermill":{"duration":0.042911,"end_time":"2020-09-04T12:17:09.563705","exception":false,"start_time":"2020-09-04T12:17:09.520794","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"class DatasetRetriever(Dataset):\n    def __init__(self, x_data, y_data=None):\n        self.x_data = x_data\n        self.y_data = y_data\n    \n    def __len__(self):\n        return len(self.x_data)\n    \n    def __getitem__(self, idx):\n        if self.y_data is not None:\n            return torch.tensor(self.x_data[idx]), torch.tensor(self.y_data[idx])\n        else:\n            return torch.tensor(self.x_data[idx])\n\n\nclass QuantileRegression(nn.Module):\n    def __init__(self, input_features=INPUT_FEATURES):\n        super(QuantileRegression, self).__init__()\n        \n        self.nn1 = nn.Linear(input_features, 100)\n        self.nn2 = nn.Linear(100, 100)\n        self.nn3_1 = nn.Linear(100, 3)\n        self.nn3_2 = nn.Linear(100, 3)\n        torch.nn.init.xavier_uniform_(self.nn1.weight)\n        torch.nn.init.constant_(self.nn1.bias, 0)\n        torch.nn.init.xavier_uniform_(self.nn2.weight)\n        torch.nn.init.constant_(self.nn2.bias, 0)\n        torch.nn.init.xavier_uniform_(self.nn3_1.weight)\n        torch.nn.init.constant_(self.nn3_1.bias, 0)\n        torch.nn.init.xavier_uniform_(self.nn3_2.weight)\n        torch.nn.init.constant_(self.nn3_2.bias, 0)\n    \n    def forward(self, inputs):\n        X = F.relu(self.nn1(inputs))\n        X = F.relu(self.nn2(X))\n        X_1 = self.nn3_1(X)\n        X_2 = F.relu(self.nn3_2(X))\n        output = X_1 + torch.cumsum(X_2, dim=1)\n        return output\n\n\nclass LossMeter(object):\n    def __init__(self):\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n):\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\n\nclass ScoreMeter(object):\n    def __init__(self):\n        self.sum = 0\n        self.count = 0\n        self.avg = 0\n    \n    def compute_score(self, y_pred, y_true):\n        sigma = y_pred[:, 2] - y_pred[:, 0]\n        fvc_pred = y_pred[:, 1]\n        sigma_clip = np.maximum(sigma, 70.0)\n        delta = np.minimum(np.abs(y_true[:, 0]-fvc_pred), 1000.0)\n        metric = (delta / sigma_clip) * np.sqrt(2.0) + np.log(sigma_clip * np.sqrt(2.0))\n        return np.mean(metric)\n    \n    def update(self, preds, labels):\n        batch_size = preds.size(0)\n        preds = preds.data.cpu().numpy()\n        labels = labels.data.cpu().numpy()\n        val = self.compute_score(preds, labels)\n        self.sum += (val * batch_size)\n        self.count += batch_size\n        self.avg = self.sum / self.count\n\n\nclass QuantileRegressionLoss(nn.Module):\n    def __init__(self):\n        super(QuantileRegressionLoss, self).__init__()\n        self.quantile = torch.tensor(QUANTILE).to(DEVICE, dtype=torch.float)\n\n    def forward(self, preds, labels):\n        error = labels - preds\n        vector = torch.max(self.quantile*error, (self.quantile-1)*error)\n        return vector.mean()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.006966,"end_time":"2020-09-04T12:17:09.577750","exception":false,"start_time":"2020-09-04T12:17:09.570784","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Training","execution_count":null},{"metadata":{"execution":{"iopub.execute_input":"2020-09-04T12:17:09.620772Z","iopub.status.busy":"2020-09-04T12:17:09.599877Z","iopub.status.idle":"2020-09-04T12:17:09.627304Z","shell.execute_reply":"2020-09-04T12:17:09.626816Z"},"papermill":{"duration":0.042757,"end_time":"2020-09-04T12:17:09.627409","exception":false,"start_time":"2020-09-04T12:17:09.584652","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"class Fitter:\n    def __init__(self, model, device, fold):\n        self.model = model\n        self.device = device\n        self.fold = fold\n        self.optimizer = Adam(self.model.parameters(), lr=LR, weight_decay=0.01)  # default weight_decay=0\n        self.criterion = QuantileRegressionLoss()\n        if LR_Scheduler:\n            self.scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(self.optimizer, \n                                                                        mode='min', \n                                                                        factor=0.5, \n                                                                        patience=20,\n                                                                        min_lr=1e-4,\n                                                                        verbose=True)\n        print(f'Fitter prepared. Device is {self.device}')\n    \n    def fit(self, train_loader, valid_loader):\n        min_valid_score = 999\n        plot_rec = {\"train_loss\": [],\n                    \"train_score\":[],\n                    \"valid_loss\": [],\n                    \"valid_score\":[],\n                    \"best_epoch\": 0,\n                    \"final\": [],\n                    \"best\": []\n                    }\n        for epoch in range(EPOCHS):\n            train_loss, train_score = self.train_one_epoch(train_loader)\n#             print(f'Epoch: {epoch+1}, train_loss: {train_loss:.4f}, train_score: {train_score:.4f}', end='  ')\n            plot_rec[\"train_loss\"].append(train_loss)\n            plot_rec[\"train_score\"].append(train_score)\n            \n            valid_loss, valid_score = self.validation(valid_loader)\n#             print(f'valid_loss: {valid_loss:.4f}, valid_score: {valid_score:.4f}')\n            plot_rec[\"valid_loss\"].append(valid_loss)\n            plot_rec[\"valid_score\"].append(valid_score)\n            \n            if LR_Scheduler:\n                self.scheduler.step(valid_score)\n            \n            if SAVE_AND_LOAD_BEST_MODEL:\n                if valid_score < min_valid_score:\n                    min_valid_score = valid_score\n                    torch.save(self.model.state_dict(), f\"Folder-{self.fold}.bin\")\n                    plot_rec[\"best_epoch\"] = epoch + 1\n                    plot_rec[\"best\"] = [train_loss, train_score, valid_loss, valid_score]\n                    print(f'****************************** Epoch {epoch+1} Model Is Best ******************************')\n            \n        if SAVE_AND_LOAD_BEST_MODEL:\n            temp = plot_rec[\"best\"]\n            print(f\"\\nBest: \", \n                  f\"\\ntrain_loss: {temp[0]:.4f}, train_score: {temp[1]:.4f}, valid_loss: {temp[2]:.4f}, valid_score: {temp[3]:.4f}\")\n            \n        plot_rec[\"final\"] = [train_loss, train_score, valid_loss, valid_score]\n        print(f\"Final: \", \n              f\"\\ntrain_loss: {train_loss:.4f}, train_score: {train_score:.4f}, valid_loss: {valid_loss:.4f}, valid_score: {valid_score:.4f}\")\n        \n        return plot_rec\n    \n    def train_one_epoch(self, train_loader):\n        losses = LossMeter()\n        scores = ScoreMeter()\n        self.model.train()\n        for step, (ipt, lbl) in enumerate(train_loader):\n            ipt = ipt.to(self.device, dtype=torch.float)\n            lbl = lbl.to(self.device, dtype=torch.float).view(-1, 1)\n            self.optimizer.zero_grad()\n            opt = self.model(ipt)\n            loss = self.criterion(opt, lbl)\n            losses.update(loss.detach().item(), ipt.size(0))\n            scores.update(opt, lbl)\n            loss.backward()\n            self.optimizer.step()\n        return losses.avg, scores.avg\n    \n    def validation(self, validation_loader):\n        losses = LossMeter()\n        scores = ScoreMeter()\n        self.model.eval()\n        for step, (ipt, lbl) in enumerate(validation_loader):\n            with torch.no_grad():\n                ipt = ipt.to(self.device, dtype=torch.float)\n                lbl = lbl.to(self.device, dtype=torch.float).view(-1, 1)\n                opt = self.model(ipt)\n                loss = self.criterion(opt, lbl)\n                losses.update(loss.detach().item(), ipt.size(0))\n                scores.update(opt, lbl)\n        return losses.avg, scores.avg\n    \n    def run_inference(self, data_loader):\n        self.model.eval()\n        temp = np.empty((0, 3))\n        for step, ipt in enumerate(data_loader):\n            with torch.no_grad():\n                ipt = ipt.to(self.device, dtype=torch.float)\n                opt = self.model(ipt)\n                temp = np.append(temp, opt.cpu().detach().numpy(), axis=0)\n        return temp","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-04T12:17:09.659449Z","iopub.status.busy":"2020-09-04T12:17:09.657648Z","iopub.status.idle":"2020-09-04T12:17:09.660106Z","shell.execute_reply":"2020-09-04T12:17:09.660567Z"},"papermill":{"duration":0.025764,"end_time":"2020-09-04T12:17:09.660688","exception":false,"start_time":"2020-09-04T12:17:09.634924","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"def TrainAndPred(x_train, y_train, x_valid, y_valid, x_test, fold):\n    device = DEVICE\n    model = QuantileRegression()\n    model.to(device)\n    \n    train_data = DatasetRetriever(x_train, y_train)\n    train_data_for_pred = DatasetRetriever(x_train)\n    valid_data = DatasetRetriever(x_valid, y_valid)\n    valid_data_for_pred = DatasetRetriever(x_valid)\n    test_data = DatasetRetriever(x_test)\n    \n    train_data_loader = torch.utils.data.DataLoader(\n        train_data,\n        batch_size=BATCH_SIZE,\n        drop_last=False,\n        num_workers=0,\n        shuffle=True\n    )\n    \n    train_data_loader_for_pred = torch.utils.data.DataLoader(\n        train_data_for_pred,\n        batch_size=BATCH_SIZE,\n        drop_last=False,\n        num_workers=0,\n        shuffle=False\n    )\n    \n    valid_data_loader = torch.utils.data.DataLoader(\n        valid_data,\n        batch_size=BATCH_SIZE,\n        drop_last=False,\n        num_workers=0,\n        shuffle=False\n    )\n    \n    valid_data_loader_for_pred = torch.utils.data.DataLoader(\n        valid_data_for_pred,\n        batch_size=BATCH_SIZE,\n        drop_last=False,\n        num_workers=0,\n        shuffle=False\n    )\n    \n    test_data_loader = torch.utils.data.DataLoader(\n        test_data,\n        batch_size=BATCH_SIZE,\n        drop_last=False,\n        num_workers=0,\n        shuffle=False\n    )\n    \n    fitter = Fitter(model=model, device=device, fold=fold)\n    plot_rec = fitter.fit(train_data_loader, valid_data_loader)\n    \n    if SAVE_AND_LOAD_BEST_MODEL:\n        bestModel = QuantileRegression()\n        bestModel.load_state_dict(torch.load(f\"Folder-{fold}.bin\"))\n        bestModel.to(device)\n        fitter = Fitter(model=bestModel, device=device, fold=fold)\n        \n    pred_for_train = fitter.run_inference(train_data_loader_for_pred)\n    pred_for_valid = fitter.run_inference(valid_data_loader_for_pred)\n    pred_for_test = fitter.run_inference(test_data_loader)\n    \n    return pred_for_train, pred_for_valid, pred_for_test, plot_rec","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-04T12:17:09.689812Z","iopub.status.busy":"2020-09-04T12:17:09.689143Z","iopub.status.idle":"2020-09-04T12:21:26.715353Z","shell.execute_reply":"2020-09-04T12:21:26.716110Z"},"papermill":{"duration":257.048456,"end_time":"2020-09-04T12:21:26.716324","exception":false,"start_time":"2020-09-04T12:17:09.667868","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"for_train = np.zeros((len(X_train), 3))\nfor_valid = np.zeros((len(X_train), 3))\nfor_test = np.zeros((len(X_test), 3))\n\nplot_recs = []\nsplit_record = []\nkfold = KFold(FOLDER)\n\nfor fold, (xx, yy) in enumerate(kfold.split(X_train)):\n    split_record.append([fold, xx, yy])\n\nfor item in split_record:\n    fold, xx, yy = item\n    print(f\"\\n========================================fold:{fold+1}============================================\")\n#     seed_everything(SEED+fold)\n    temp_x_train = X_train[xx]\n    temp_y_train = Y_train[xx]\n    temp_x_valid = X_train[yy]\n    temp_y_valid = Y_train[yy]\n    print(\"Shape: \", temp_x_train.shape, temp_y_train.shape, temp_x_valid.shape, temp_y_valid.shape)\n    pred_for_train, pred_for_valid, pred_for_test, plot_rec = TrainAndPred(temp_x_train, temp_y_train, temp_x_valid, temp_y_valid, X_test, fold+1)\n    plot_recs.append(plot_rec)\n    for_train[xx] += pred_for_train / (FOLDER - 1)\n    for_valid[yy] = pred_for_valid\n    for_test[:] += pred_for_test / FOLDER\n\nnp.save(\"for_train\", for_train)\nnp.save(\"for_valid\", for_valid)\nnp.save(\"for_test\", for_test)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.00769,"end_time":"2020-09-04T12:21:26.732839","exception":false,"start_time":"2020-09-04T12:21:26.725149","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Drawing","execution_count":null},{"metadata":{"execution":{"iopub.execute_input":"2020-09-04T12:21:26.766207Z","iopub.status.busy":"2020-09-04T12:21:26.764353Z","iopub.status.idle":"2020-09-04T12:21:26.769169Z","shell.execute_reply":"2020-09-04T12:21:26.766940Z"},"papermill":{"duration":0.028123,"end_time":"2020-09-04T12:21:26.769331","exception":false,"start_time":"2020-09-04T12:21:26.741208","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"if SAVE_AND_LOAD_BEST_MODEL:\n    print(\"Best: \")\n    plot_recs_copy = np.array([plot_rec[\"best\"] for plot_rec in plot_recs]).mean(axis=0)\n    print(\"train_loss_avg: \", round(plot_recs_copy[0], 4), \"  train_score_avg: \", round(plot_recs_copy[1], 4))\n    print(\"valid_loss_avg: \", round(plot_recs_copy[2], 4), \"  valid_score_avg: \", round(plot_recs_copy[3], 4))\nelse:\n    print(\"Final: \")\n    plot_recs_copy = np.array([plot_rec[\"final\"] for plot_rec in plot_recs]).mean(axis=0)\n    print(\"train_loss_avg: \", round(plot_recs_copy[0], 4), \"  train_score_avg: \", round(plot_recs_copy[1], 4))\n    print(\"valid_loss_avg: \", round(plot_recs_copy[2], 4), \"  valid_score_avg: \", round(plot_recs_copy[3], 4))","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-04T12:21:26.806880Z","iopub.status.busy":"2020-09-04T12:21:26.805371Z","iopub.status.idle":"2020-09-04T12:21:26.808675Z","shell.execute_reply":"2020-09-04T12:21:26.808146Z"},"papermill":{"duration":0.029628,"end_time":"2020-09-04T12:21:26.808765","exception":false,"start_time":"2020-09-04T12:21:26.779137","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"# The place where the green line is is the epochs of the best model\nif SAVE_AND_LOAD_BEST_MODEL:\n    for fold, adict in enumerate(plot_recs):\n        best_epoch = adict[\"best_epoch\"]\n\n        best_epoch_train_loss = round(adict['best'][0],4)\n        best_epoch_train_score = round(adict['best'][1],4)\n        best_epoch_valid_loss = round(adict['best'][2],4)\n        best_epoch_valid_score = round(adict['best'][3],4)\n\n        min_loss = min(min(adict[\"train_loss\"]), min(adict[\"valid_loss\"]))\n        max_loss = max(max(adict[\"train_loss\"]), max(adict[\"valid_loss\"]))\n        min_score = min(min(adict[\"train_score\"]), min(adict[\"valid_score\"]))\n        max_score = max(max(adict[\"train_score\"]), max(adict[\"valid_score\"]))\n        fig, ax = plt.subplots(1, 2, figsize=(15, 6))\n        ax[0].plot(range(1, len(adict[\"train_loss\"])+1), adict[\"train_loss\"], label=\"train_loss\")\n        ax[0].plot(range(1, len(adict[\"train_loss\"])+1), adict[\"valid_loss\"], label=\"valid_loss\")\n        ax[0].plot([best_epoch, best_epoch], [min_loss, max_loss], label=\"best_epoch\")\n        ax[1].plot(range(1, len(adict[\"train_loss\"])+1), adict[\"train_score\"], label=\"train_score\")\n        ax[1].plot(range(1, len(adict[\"train_loss\"])+1), adict[\"valid_score\"], label=\"valid_score\")\n        ax[1].plot([best_epoch, best_epoch], [min_score, max_score], label=\"best_epoch\")\n        ax[0].legend()\n        ax[1].legend()\n        ax[0].grid()\n        ax[1].grid()\n        ax[0].set_title(f\"Fold{fold+1}-Loss-Epoch{best_epoch}-train{best_epoch_train_loss}-valid{best_epoch_valid_loss}\")\n        ax[1].set_title(f\"Fold{fold+1}-Score-Epoch{best_epoch}-train{best_epoch_train_score}-valid{best_epoch_valid_score}\")\n        ax[0].set_ylim(20, 100)\n        ax[1].set_ylim(6, 8)\n        plt.show()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-04T12:21:26.835409Z","iopub.status.busy":"2020-09-04T12:21:26.834628Z","iopub.status.idle":"2020-09-04T12:21:27.084199Z","shell.execute_reply":"2020-09-04T12:21:27.084726Z"},"papermill":{"duration":0.268787,"end_time":"2020-09-04T12:21:27.084869","exception":false,"start_time":"2020-09-04T12:21:26.816082","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"sigma_opt = mean_absolute_error(Y_train, for_valid[:, 1])\nunc = for_valid[:, 2] - for_valid[:, 0]\nsigma_mean = np.mean(unc)\nprint(sigma_opt, sigma_mean)\nprint(\"Min: \", unc.min(), \"Mean: \", unc.mean(), \"Max: \", unc.max(), \"Mean(>=0): \", (unc>=0).mean())\nplt.figure(figsize=(12, 6))\nplt.hist(unc)\nplt.title(\"uncertainty in prediction\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-04T12:21:27.116638Z","iopub.status.busy":"2020-09-04T12:21:27.115321Z","iopub.status.idle":"2020-09-04T12:21:27.377267Z","shell.execute_reply":"2020-09-04T12:21:27.377769Z"},"papermill":{"duration":0.283146,"end_time":"2020-09-04T12:21:27.377905","exception":false,"start_time":"2020-09-04T12:21:27.094759","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"plt.figure(figsize=(15, 8))\nidxs = np.random.randint(0, Y_train.shape[0], 100)\nplt.plot(Y_train[idxs], label=\"ground truth\")\nplt.plot(for_valid[idxs, 0], label=f\"q{int(QUANTILE[0]*100)}\")\nplt.plot(for_valid[idxs, 1], label=f\"q{int(QUANTILE[1]*100)}\")\nplt.plot(for_valid[idxs, 2], label=f\"q{int(QUANTILE[2]*100)}\")\nplt.legend(loc=\"best\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.010628,"end_time":"2020-09-04T12:21:27.399453","exception":false,"start_time":"2020-09-04T12:21:27.388825","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Submit","execution_count":null},{"metadata":{"execution":{"iopub.execute_input":"2020-09-04T12:21:27.432084Z","iopub.status.busy":"2020-09-04T12:21:27.431318Z","iopub.status.idle":"2020-09-04T12:21:27.435514Z","shell.execute_reply":"2020-09-04T12:21:27.435013Z"},"papermill":{"duration":0.024647,"end_time":"2020-09-04T12:21:27.435667","exception":false,"start_time":"2020-09-04T12:21:27.411020","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"sub['FVC1'] = 0.996 * for_test[:, 1]\nsub['Confidence1'] = for_test[:, 2] - for_test[:, 0]\nsubm = sub[['Patient_Week', 'FVC', 'Confidence', 'FVC1', 'Confidence1']].copy()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-04T12:21:27.471199Z","iopub.status.busy":"2020-09-04T12:21:27.467235Z","iopub.status.idle":"2020-09-04T12:21:27.500009Z","shell.execute_reply":"2020-09-04T12:21:27.500638Z"},"papermill":{"duration":0.054552,"end_time":"2020-09-04T12:21:27.500780","exception":false,"start_time":"2020-09-04T12:21:27.446228","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"subm.loc[~subm.FVC1.isnull(), 'FVC'] = subm.loc[~subm.FVC1.isnull(), 'FVC1']\nif sigma_mean < 70:\n    subm['Confidence'] = sigma_opt\nelse:\n    subm.loc[~subm.FVC1.isnull(), 'Confidence'] = subm.loc[~subm.FVC1.isnull(), 'Confidence1']\n\nsubm.describe().T","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-04T12:21:27.550877Z","iopub.status.busy":"2020-09-04T12:21:27.540445Z","iopub.status.idle":"2020-09-04T12:21:27.831808Z","shell.execute_reply":"2020-09-04T12:21:27.830113Z"},"papermill":{"duration":0.318592,"end_time":"2020-09-04T12:21:27.831938","exception":false,"start_time":"2020-09-04T12:21:27.513346","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"otest = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\nfor i in range(len(otest)):\n    subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]\n    subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1\n\nsubm[[\"Patient_Week\", \"FVC\", \"Confidence\"]].to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}