{"cells":[{"metadata":{},"cell_type":"markdown","source":"This notebook is based on @ulrich07 https://www.kaggle.com/ulrich07/osic-multiple-quantile-regression-starter and code with pytorch.If you are not familiar with tensorflow, refer to this notebook.\nbtw,Pytorch is a bit slow than tensorflow.","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pydicom\nimport os\nimport random\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\n\nfrom PIL import Image\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.model_selection import KFold\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.nn as nn\nimport torch\nfrom torch.utils.data.dataset import Dataset\nfrom torch.utils.data import DataLoader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed=2020):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed(seed)\n        torch.cuda.manual_seed_all(seed)#set all gpus seed\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = False#if input data type and channels' changes arent' large use it improve train efficient\n        torch.backends.cudnn.enabled = True\n    \nseed_everything(42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ROOT = \"../input/osic-pulmonary-fibrosis-progression\"\ndevice = torch.device('cuda')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr = pd.read_csv(f\"{ROOT}/train.csv\")\ntr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])\nchunk = pd.read_csv(f\"{ROOT}/test.csv\")\n\nprint(\"add infos\")\nsub = pd.read_csv(f\"{ROOT}/sample_submission.csv\")\nsub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])\nsub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))\nsub =  sub[['Patient','Weeks','Confidence','Patient_Week']]\nsub = sub.merge(chunk.drop('Weeks', axis=1), on=\"Patient\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr['WHERE'] = 'train'\nchunk['WHERE'] = 'val'\nsub['WHERE'] = 'test'\ndata = tr.append([chunk, sub])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.iloc[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(tr.shape, chunk.shape, sub.shape, data.shape)\nprint(tr.Patient.nunique(), chunk.Patient.nunique(), sub.Patient.nunique(), \n      data.Patient.nunique())\n#","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data['min_week'] = data['Weeks']\ndata.loc[data.WHERE=='test','min_week'] = np.nan\ndata['min_week'] = data.groupby('Patient')['min_week'].transform('min')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base = data.loc[data.Weeks == data.min_week]\nbase = base[['Patient','FVC']].copy()\nbase.columns = ['Patient','min_FVC']\nbase['nb'] = 1\nbase['nb'] = base.groupby('Patient')['nb'].transform('cumsum')\nbase = base[base.nb==1]\nbase.drop('nb', axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = data.merge(base, on='Patient', how='left')\ndata['base_week'] = data['Weeks'] - data['min_week']\ndel base","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"COLS = ['Sex','SmokingStatus'] #,'Age'\nFE = []\nfor col in COLS:\n    for mod in data[col].unique():\n        FE.append(mod)\n        data[mod] = (data[col] == mod).astype(int)\n#=================","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#\ndata['age'] = (data['Age'] - data['Age'].min() ) / ( data['Age'].max() - data['Age'].min() )\ndata['BASE'] = (data['min_FVC'] - data['min_FVC'].min() ) / ( data['min_FVC'].max() - data['min_FVC'].min() )\ndata['week'] = (data['base_week'] - data['base_week'].min() ) / ( data['base_week'].max() - data['base_week'].min() )\ndata['percent'] = (data['Percent'] - data['Percent'].min() ) / ( data['Percent'].max() - data['Percent'].min() )\nFE += ['age','percent','week','BASE']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr = data.loc[data.WHERE=='train']\nchunk = data.loc[data.WHERE=='val']\nsub = data.loc[data.WHERE=='test']\ndel data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr.shape, chunk.shape, sub.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### BASELINE NN ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.nn.functional as F\n\nclass MishFunction(torch.autograd.Function):\n    @staticmethod\n    def forward(ctx, x):\n        ctx.save_for_backward(x)\n        return x * torch.tanh(F.softplus(x))   # x * tanh(ln(1 + exp(x)))\n\n    @staticmethod\n    def backward(ctx, grad_output):\n        x = ctx.saved_variables[0]\n        sigmoid = torch.sigmoid(x)\n        tanh_sp = torch.tanh(F.softplus(x)) \n        return grad_output * (tanh_sp + x * sigmoid * (1 - tanh_sp * tanh_sp))\n\nclass Mish(nn.Module):\n    def forward(self, x):\n        return MishFunction.apply(x)\n\ndef to_Mish(model):\n    for child_name, child in model.named_children():\n        if isinstance(child, nn.ReLU):\n            setattr(model, child_name, Mish())\n        else:\n            to_Mish(child)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n#C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype=\"float32\")\nC1, C2 = torch.tensor(70,dtype=torch.float),torch.tensor(1000,dtype=torch.float)\nC1, C2 = C1.to(device),C2.to(device)\n#=============================#\ndef score(y_true, y_pred):\n    y_true = y_true.to(torch.float)\n    y_pred = y_pred.to(torch.float)\n    \n    sigma = y_pred[:,2] - y_pred[:,0]\n    fvc_pred = y_pred[:,1]\n    #sigma_clip = sigma + C1\n    sigma_clip = torch.max(sigma, C1)\n    delta = torch.abs(y_true[:,0] - fvc_pred)\n    delta = torch.min(delta, C2)\n    sq2 = torch.sqrt(torch.tensor(2, dtype=torch.float))\n    metric = (delta / sigma_clip)*sq2 + torch.log(sigma_clip* sq2)\n    return torch.mean(metric)\n\n#============================#\ndef qloss(y_true, y_pred):\n    # Pinball loss for multiple quantiles\n    device = y_true.device\n    qs = [0.2, 0.50, 0.8]\n    q = torch.tensor(np.array([qs]), dtype=torch.float32)\n    q = q.to(device)\n    e = y_true - y_pred\n    v = torch.max(q*e, (q-1)*e)\n    return torch.mean(v)\n#=============================#\ndef mloss(_lambda):\n    def loss(y_true, y_pred):\n        return _lambda * qloss(y_true, y_pred) + (1 - _lambda)*score(y_true, y_pred)\n    return loss\n#=================\nclass make_model(nn.Module):\n    def __init__(self, in_ch, out_ch=3):\n        super(make_model, self).__init__()\n        self.fc1 = nn.Sequential(\n            nn.Linear(in_ch, 100),\n            Mish()\n        )\n        self.fc2 = nn.Sequential(\n            nn.Linear(100, 100),\n            Mish()\n        )\n        self.fc3_p1 = nn.Linear(100, out_ch)\n        self.fc3_p2 = nn.Sequential(\n            nn.Linear(100, out_ch),\n            Mish()\n        )\n        \n    def forward(self, x):\n        x = self.fc1(x)\n        x = self.fc2(x)\n        x1 = self.fc3_p1(x)\n        x2 = self.fc3_p2(x)\n        x = x1 + torch.cumsum(x2,dim=1)\n        return x\n    \n    #x = L.Dense(100, activation=\"relu\", name=\"d1\")(z)\n    #x = L.Dense(100, activation=\"relu\", name=\"d2\")(x)\n    #x = L.Dense(100, activation=\"relu\", name=\"d3\")(x)\n    #p1 = L.Dense(3, activation=\"linear\", name=\"p1\")(x)\n    #p2 = L.Dense(3, activation=\"relu\", name=\"p2\")(x)\n    #preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1), \n    #                 name=\"preds\")([p1, p2])\n    \n    #model.compile(loss=qloss, optimizer=\"adam\", metrics=[score])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#test loss function\nMloss = mloss(0.8)\ny_true = (torch.tensor([2100,2300])).to(device).reshape(2,1)\npred = (torch.tensor([[1800,2100,2400],\n                     [2100,2300,2600]])).to(device).reshape(2,3)\nMloss(y_true,pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NFOLD = 5\nkf = KFold(n_splits=NFOLD)\npd_patient = pd.DataFrame({\"Patient\":tr[\"Patient\"].unique()})\n\nfor idx, (tr_idx, val_idx) in enumerate(kf.split(pd_patient)):\n    pd_patient.loc[val_idx,\"fold\"] = idx\npd_patient.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr['fold'] = -1\nfor i in range(len(pd_patient)):\n    tr.fold[tr.Patient==pd_patient.loc[i,\"Patient\"]] = pd_patient.loc[i,\"fold\"]\ntr.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ze = (sub[FE].values).astype(np.float32)\npe = np.zeros((ze.shape[0], 3))\nbatch = 128","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Data_Generate(Dataset):\n    def __init__(self,data,label=None):\n        self.data = data\n        self.label = label\n        \n    def __getitem__(self,index):\n        z_ = self.data[index]\n        if self.label is not None:\n            y_ = self.label[index]\n            y_ = y_[None,]\n            return z_,y_\n        else:\n            return z_\n         \n    def __len__(self):\n        return len(self.data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_db = Data_Generate(ze)\ntest_loader = DataLoader(test_db, batch_size=batch, shuffle=False, num_workers=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class EarlyStopping:\n    \"\"\"Early stops the training if validation loss doesn't improve after a given patience.\"\"\"\n    def __init__(self, patience=7, verbose=False, delta=0, path='checkpoint.pt'):\n        \"\"\"\n        Args:\n            patience (int): How long to wait after last time validation loss improved.\n                            Default: 7\n            verbose (bool): If True, prints a message for each validation loss improvement. \n                            Default: False\n            delta (float): Minimum change in the monitored quantity to qualify as an improvement.\n                            Default: 0\n            path (str): Path for the checkpoint to be saved to.\n                            Default: 'checkpoint.pt'\n        \"\"\"\n        self.patience = patience\n        self.verbose = verbose\n        self.counter = 0\n        self.best_score = None\n        self.early_stop = False\n        self.val_loss_min = np.Inf\n        self.delta = delta\n        self.path = path\n\n    def __call__(self, val_loss, model):\n\n        score = -val_loss\n\n        if self.best_score is None:\n            self.best_score = score\n            self.save_checkpoint(val_loss, model)\n        elif score < self.best_score + self.delta:\n            self.counter += 1\n            #print(f'EarlyStopping counter: {self.counter} out of {self.patience}')\n            if self.counter >= self.patience:\n                self.early_stop = True\n        else:\n            self.best_score = score\n            self.save_checkpoint(val_loss, model)\n            self.counter = 0\n\n    def save_checkpoint(self, val_loss, model):\n        '''Saves model when validation loss decrease.'''\n        if self.verbose:\n            print(f'Validation loss decreased ({self.val_loss_min:.6f} --> {val_loss:.6f}).  Saving model ...')\n        torch.save(model.state_dict(), self.path)\n        self.val_loss_min = val_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ncnt = 0\nEPOCHS = 1000\ncriterion = mloss(0.8)\nlist_train_loss,list_val_loss,list_train_score,list_val_score = [],[],[],[]\n\n#for tr_idx, val_idx in kf.split(z):\nfor fold in range(NFOLD):\n    \n    val_out = []\n    print(f\"FOLD {fold+1}\")\n    #==================load data kfold==========================#\n    tr_z = tr[FE][tr.fold!=fold].values.astype(np.float32)\n    tr_y = tr.FVC[tr.fold!=fold].values.astype(np.float32)\n    val_z = tr[FE][tr.fold==fold].values.astype(np.float32)\n    val_y = tr.FVC[tr.fold==fold].values.astype(np.float32)\n    train_db = Data_Generate(tr_z,tr_y)\n    train_loader = DataLoader(train_db, batch_size=batch, shuffle=True, num_workers=4)\n    val_db = Data_Generate(val_z,val_y)\n    val_loader = DataLoader(val_db, batch_size=batch, shuffle=False, num_workers=4)\n    #==================prepare model==========================#\n    tr_num_batch = len(train_loader)\n    val_num_batch = len(val_loader)\n    net = make_model(in_ch=len(FE)).to(device)\n    optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, net.parameters()), lr=0.001, weight_decay=5e-4)\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=20, min_lr=1e-8, verbose=False)\n    early_stopping = EarlyStopping(patience=80,path=f'Osic-NN-fold_{fold}.pth')\n\n    for epoch in tqdm(range(EPOCHS)):\n        train_loss,train_score,val_loss,val_score = 0,0,0,0\n        #==================train ==========================#\n        net.train()\n        for idx, sample in enumerate(train_loader):\n            data, label = sample\n            data, label = data.to(device), label.to(device)\n            out = net(data)\n            loss = criterion(label, out)\n            score_ = score(label ,out)\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n            train_loss += loss.item()/tr_num_batch\n            train_score += score_.item()/tr_num_batch\n        list_train_loss.append(train_loss)\n        list_train_score.append(train_score)\n        #==================val ==========================#\n        net.eval()   \n        for idx, sample in enumerate(val_loader):\n            data, label = sample\n            data, label = data.to(device), label.to(device)\n            with torch.no_grad():\n                out = net(data)\n            val_out.append(out.cpu().numpy())\n            loss = criterion(label, out)\n            score_ = score(label, out)\n            val_loss += loss.item()/val_num_batch\n            val_score += score_.item()/val_num_batch\n        list_val_loss.append(val_loss)\n        list_val_score.append(val_score)\n        early_stopping(val_loss, net)\n        if early_stopping.early_stop:\n            print(\"Early stopping\")\n            break\n        scheduler.step(val_loss)\n             \n    print(f\"train loss: {train_loss}  train score: {train_score}\\n \\\n          val loss: {val_loss} val score: {val_score}\\n \\\n          final lr: {optimizer.param_groups[0]['lr']}\"\n         )\n  ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### PREDICTION","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"predict test...\")\nfor k in range(NFOLD):\n    pred = []\n    net = make_model(in_ch=len(FE)).to(device)\n    net.load_state_dict(torch.load(f\"Osic-NN-fold_{k}.pth\"))\n    net.eval()\n    for idx, sample in enumerate(test_loader):\n        data = sample\n        data = data.to(device)\n        with torch.no_grad():\n            out = net(data)\n        out = (out.cpu().numpy()).astype(np.int)\n        pred.append(out)\n    pred = np.concatenate(pred)\n    pe += pred / NFOLD","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['FVC1'] = pe[:, 1]\nsub['Confidence1'] = abs(pe[:, 2] - pe[:, 0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm = sub[['Patient_Week','FVC1','Confidence1']].copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm.loc[~subm.FVC1.isnull()].head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm.describe().T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm.rename(columns={'FVC1':'FVC','Confidence1':'Confidence'},inplace=True) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm[[\"Patient_Week\",\"FVC\",\"Confidence\"]].to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}