{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pydicom\nimport os\nimport random\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom PIL import Image\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.model_selection import KFold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.models as M\nimport random\n'''\n\nimport torch\nimport torchvision","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed=2020):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    #tf.random.set_seed(seed)\n    \nseed_everything(42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ROOT = \"../input/osic-pulmonary-fibrosis-progression\"\nBATCH_SIZE=128","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr = pd.read_csv(f\"{ROOT}/train.csv\")\ntr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])\nchunk = pd.read_csv(f\"{ROOT}/test.csv\")\n\nprint(\"add infos\")\nsub = pd.read_csv(f\"{ROOT}/sample_submission.csv\")\nsub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])\nsub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))\nprint(sub.index.size)\nsub =  sub[['Patient','Weeks','Confidence','Patient_Week']]\nsub = sub.merge(chunk.drop('Weeks', axis=1), on=\"Patient\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub[sub.Patient=='ID00419637202311204720264']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr['WHERE'] = 'train'\nchunk['WHERE'] = 'val'\nsub['WHERE'] = 'test'\ndata = tr.append([chunk, sub])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(tr.shape, chunk.shape, sub.shape, data.shape)\nprint(tr.Patient.nunique(), chunk.Patient.nunique(), sub.Patient.nunique(), \n      data.Patient.nunique())\n#","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data['min_week'] = data['Weeks']\ndata.loc[data.WHERE=='test','min_week'] = np.nan\ndata['min_week'] = data.groupby('Patient')['min_week'].transform('min')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" data.loc[data.Weeks == data.min_week]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base = data.loc[data.Weeks == data.min_week]\nbase = base[['Patient','FVC']].copy()\nbase.columns = ['Patient','min_FVC']\nbase['nb'] = 1\nbase['nb'] = base.groupby('Patient')['nb'].transform('cumsum')\nbase = base[base.nb==1]\nbase.drop('nb', axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base[base.Patient=='ID00419637202311204720264']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = data.merge(base, on='Patient', how='left')\ndata['base_week'] = data['Weeks'] - data['min_week']\n#del base\ndata['diff_fvc_prev']=data['FVC'].diff(1)/data['FVC'].shift(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data[data.Patient=='ID00007637202177411956430']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"COLS = ['Sex','SmokingStatus'] #,'Age'\nFE = []\nfor col in COLS:\n    for mod in data[col].unique():\n        FE.append(mod)\n        data[mod] = (data[col] == mod).astype(int)\n#=================","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#\ndata['age'] = (data['Age'] - data['Age'].min() ) / ( data['Age'].max() - data['Age'].min() )\ndata['BASE'] = (data['min_FVC'] - data['min_FVC'].min() ) / ( data['min_FVC'].max() - data['min_FVC'].min() )\ndata['week'] = (data['base_week'] - data['base_week'].min() ) / ( data['base_week'].max() - data['base_week'].min() )\ndata['percent'] = (data['Percent'] - data['Percent'].min() ) / ( data['Percent'].max() - data['Percent'].min() )\n\nFE += ['age','percent','week','BASE']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#data.rename_columns({'base_week':'diff_from'})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#data['FVC'].diff(1)/data['FVC'].shift(1)*100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data[FE]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.head()\n#pd.options.display.max_rows=70\n#pd.options.display.max_columns=40\n\ndata=data.fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr[tr['Patient']=='ID00419637202311204720264'] ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr = data.loc[data.WHERE=='train']\nchunk = data.loc[data.WHERE=='val']\nsub = data.loc[data.WHERE=='test']\n#del data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr.shape, chunk.shape, sub.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#qloss_func(torch.tensor([1800]),torch.tensor([2000])),qloss()(torch.tensor([1800]),torch.tensor([2000]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### BASELINE NN ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#np.cumsum([1, 2,3])  # => [a, a + b, a + b + c]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = tr['FVC'].values\nz = tr[FE].values\nze = sub[FE].values\nnh = z.shape[1]\npe = np.zeros((ze.shape[0], 3))\npred = np.zeros((z.shape[0], 3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub[FE][sub.index==0].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DatasetRetriever(Dataset):\n\n    def __init__(self,   train_arrays,targets,df=None, transforms=None, test=False):\n        super().__init__()\n\n        self.image_ids = train_arrays\n        #self.df=df\n        self.test=test\n        if test:\n            self.targets=torch.ones(train_arrays.shape[0])\n        else:\n            self.targets=targets\n        \n    def __getitem__(self, index: int):\n        train_input = self.image_ids[index] \n        \n        target=self.targets[index]\n        '''\n        if self.test:\n            \n            #patient_id=self.df[index].image_id[0]\n            train_input =self.df[FE][sub.index==index].values[0]\n            return train_input, target \n        '''\n        return train_input, target \n    def __len__(self) -> int:\n        return self.image_ids.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NFOLD = 5\nkf = KFold(n_splits=NFOLD)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#len(train_dataset)\n#train_dataset[0][0].shape ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class osic_model(torch.nn.Module):\n    \n    def __init__(self, n_inputs=32):\n        super(osic_model, self).__init__()\n        self.layer1 = torch.nn.Linear(9, 32)\n        #self.batchnorm1= torch.nn.BatchNorm1d(self.layer1.out_features) \n        self.relu=torch.nn.ReLU(inplace=True)\n        \n        self.fc=torch.nn.Linear(self.layer1.out_features,3)\n        \n    def forward(self,input ):\n        x=self.layer1(input)\n       \n        x=self.relu(x)\n        \n        x=self.fc(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model=osic_model()\nmodel","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#tmp_x,tmp_y=next(iter(train_loader))\n#train_dataset[0][0].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"optimizer = torch.optim.AdamW(params=model.parameters(), lr=1e-2)\n#criterion=","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class qloss(torch.nn.Module):\n    def __init__(self):\n        super(qloss, self).__init__()\n        self.w=torch.nn.Parameter( torch.tensor([0.2,0.5,0.8]))\n         \n        \n    def forward(self,y_pred,y_true):\n        #q = torch.tensor(qs)\n    #tf.constant(np.array([qs]), dtype=tf.float32)\n        e = y_true.unsqueeze(-1) - y_pred\n    #print(q*e,(q-1)*e)\n        #print(self.w.unsqueeze(0).size(),e.size())\n        #v = torch.max(torch.cat([self.w.unsqueeze(0)*e, (self.w-1.).unsqueeze(0)*e],dim=1),dim=1)[0]#160,-640 1200,2000\n        #print([self.w.unsqueeze(0)*e, (self.w-1.).unsqueeze(0)*e])\n        v = torch.max(torch.stack([self.w.unsqueeze(0)*e, (self.w-1.).unsqueeze(0)*e],dim=1),dim=1)[0]\n        #print(v.size(),torch.stack([self.w.unsqueeze(0)*e, (self.w-1.).unsqueeze(0)*e],dim=1).size())\n        #print(v.size())\n        #print(v)\n        return torch.mean(torch.sum(v,dim=-1))\n        \n        \n\ndef qloss_func(y_true, y_pred):\n    # Pinball loss for multiple quantiles\n    qs = [0.2, 0.50, 0.8]\n    q = torch.tensor(qs)\n    #tf.constant(np.array([qs]), dtype=tf.float32)\n    e = y_true - y_pred\n    #print(q*e,(q-1)*e)\n    v = torch.max(q*e, (q-1.)*e)[0]#160,-640 1200,2000\n    return torch.mean(v)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def quantile_loss(preds, target, quantiles = [0.2, 0.50, 0.8]):\n    #assert not target.requires_grad\n    assert len(preds) == len(target)\n    losses = []\n    for i, q in enumerate(quantiles):\n        errors = target - preds[:, i]\n        print('q-1',q-1,(q - 1) * errors,'q',q, q * errors)\n        print('max',torch.max((q - 1) * errors, q * errors).unsqueeze(1))\n        losses.append(torch.max((q - 1) * errors, q * errors).unsqueeze(1))\n    #print( torch.sum(torch.cat(losses, dim=1),dim=1).size(),losses )\n    loss = torch.mean(torch.sum(torch.cat(losses, dim=1), dim=1))\n    #print(torch.sum(torch.cat(losses, dim=1), dim=1).size()) 4\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#quantile_loss (model(torch.randn(4,9)),gt[:4])\n#a=torch.randn(4,9)\n#qloss() (model(a),gt[:4]) ,quantile_loss(model(a),gt[:4])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm.notebook import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#https://www.kaggle.com/havinath/eda-observations-visualizations-pytorch/output\ndef metric_loss(pred_fvc,true_fvc):\n        #Implementation of the metric in pytorch\n    sigma = pred_fvc[:, 2] - pred_fvc[:, 0]\n    true_fvc=torch.reshape(true_fvc,pred_fvc[:,1].shape)\n    sigma_clipped=torch.clamp(sigma,min=70)\n    delta=torch.clamp(torch.abs(pred_fvc[:,1]-true_fvc),max=1000)\n    metric=torch.div(-torch.sqrt(torch.tensor([2.0]).to('cpu'))*delta,sigma_clipped)-torch.log(torch.sqrt(torch.tensor([2.0]).to('cpu'))*sigma_clipped)\n    return metric.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fold=0\ncriterion=qloss()\ntmp_loss= -100000\n#model.load_state_dict(torch.load('bestmodel.pth'))\nfor tr_idx, val_idx in tqdm(kf.split(z)):\n\n    train_dataset = DatasetRetriever(\n        train_arrays=z[tr_idx],#.index.values,\n        targets= y[tr_idx]\n\n    )\n\n    valid_dataset = DatasetRetriever(\n        train_arrays=z[val_idx],#.index.values,\n        targets= y[val_idx]\n\n    )\n    \n    \n    train_loader = torch.utils.data.DataLoader(\n        train_dataset,\n        batch_size=128,\n        sampler=RandomSampler(train_dataset),\n        pin_memory=False,\n        drop_last=True,\n        num_workers=4,\n        #collate_fn=collate_fn,\n    )\n    valid_loader = torch.utils.data.DataLoader(\n        valid_dataset,\n        batch_size=128,\n        #sampler=RandomSampler(train_dataset),\n        pin_memory=False,\n        drop_last=False,\n        num_workers=4,\n        #collate_fn=collate_fn,\n    )\n    print(f'fold{fold}',end='\\r')\n    \n    for epoch in (range(650)):\n        model.train()\n        \n        #print(f' epoch {epoch}',end='\\r')\n        \n    \n        for input,gt in  (train_loader):\n            pred=model(input.float())\n            #print(gt.size(),input.size())\n            loss=qloss()(pred,gt)\n            \n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n            \n        #print(f' epoch {epoch} :summary loss : {loss.item()}',end='\\r')\n        \n        with torch.no_grad() :\n            model.eval()\n            val_loss=0.\n            ll=0.\n            \n        \n            for input,gt in (valid_loader):\n                pred=model(input.float())\n                #print(gt.size(),input.size())\n                loss=criterion(pred,gt)\n                val_loss+=loss.item()\n                ll+=metric_loss(pred ,gt).item()\n        if epoch%200==0:\n            \n            print(f'train loss summary loss : {loss.item()} val_loss epoch {epoch} {val_loss/len(valid_loader)}')\n            print(f'll :  {epoch} {ll/len(valid_loader)}')\n            if ll/len(valid_loader)>tmp_loss:\n                print('saving best weights at',ll/len(valid_loader))\n                tmp_loss= ll/len(valid_loader)\n                torch.save(model.state_dict(),f'{fold}_bestmodel.pth')\n                 \n    fold=fold+1    \n    tmp_loss=-100000","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ze[0:2],sub[0:2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub=sub.reset_index(drop=True)\ntest_dataset = DatasetRetriever(\n        train_arrays=ze,#.index.values,\n        #df=sub,\n        targets= None,\n        test=True\n\n    )\ntest_loader = torch.utils.data.DataLoader(\n        test_dataset,\n        batch_size=128,\n        sampler=RandomSampler(test_dataset),\n        pin_memory=False,\n        drop_last=False,\n        num_workers=4,\n    shuffle=False\n        #collate_fn=collate_fn,\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset[0],FE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n'''\ncnt = 0\nEPOCHS = 650\nfor tr_idx, val_idx in kf.split(z):\n    cnt += 1\n    print(f\"FOLD {cnt}\")\n    net = make_model(nh)\n    net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS, \n            validation_data=(z[val_idx], y[val_idx]), verbose=0) #\n    print(\"train\", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))\n    print(\"val\", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))\n    print(\"predict val...\")\n    pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)\n    print(\"predict test...\")\n    pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0) / NFOLD\n#==============\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_preds=[]\nfor fold in range(5):\n    \n    model.eval()\n    model.load_state_dict(torch.load(f'{fold}_bestmodel.pth'))\n\n    test_preds=[]\n    for x_test,y_test in tqdm(test_loader):\n        preds=model(x_test.float())\n        test_preds.append(preds)\n    model_preds.append(torch.cat(test_preds))\n\npreds_numpy=torch.stack(model_preds,dim=0).mean(0).detach().numpy()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#preds_numpy=torch.cat(test_preds).detach().numpy()#","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"quantiles = (0.2, 0.5, 0.8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unc = preds_numpy[:,2] - preds_numpy[:, 0]\nsigma_mean = np.mean(unc)\nprint( sigma_mean)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#test_dataset[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#df = pd.DataFrame(data=preds_numpy, columns=list(quantiles))\ndf=pd.DataFrame({'Patient_Week':[]})\ndf['Patient_Week'] = sub['Patient_Week']\n#df['FVC'] = df[quantiles[1]]\ndf['FVC'] = preds_numpy[:,1]\n#df['Confidence'] = df[quantiles[2]] - df[quantiles[0]]\ndf['Confidence'] = preds_numpy[:,2] - preds_numpy[:,0]\n#df = df.drop(columns=list(quantiles))\ndf=df.reset_index(drop=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"otest = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\nfor i in range(len(otest)):\n    df.loc[df['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]\n    df.loc[df['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.describe().T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#df[(df.Patient_Week.str.contains('ID00419637202311204720264')) & (df.Confidence==0.1)]\n#df\ndf","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Please let me know if u find any bugs. \nBase kernel https://www.kaggle.com/ulrich07/osic-multiple-quantile-regression-starter/output","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}