{"cells":[{"metadata":{},"cell_type":"markdown","source":"## EfficientNet + Quantile Regression Model in Pytorch\n\nThis notebook generates predictions using Images and tabular data. \n\n### Acknowledgements \n\n* efficientnets-quantile-regression-inference: https://www.kaggle.com/leoisleo1/efficientnets-quantile-regression-inference\n\n* Training EfficientNet with pytorch: https://www.kaggle.com/noelmat/training-efficientnet-with-pytorch\n\n* melanoma-pytorch-starter-efficientnet: https://www.kaggle.com/nroman/melanoma-pytorch-starter-efficientnet","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Imports","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport torch\nfrom torchvision import models\nfrom pathlib import Path\nPath.ls = lambda x: list(x.iterdir())\n\nimport cv2 \nimport pydicom\nfrom tqdm import tqdm\nfrom matplotlib import pyplot as plt\nfrom torchvision import transforms\n\nfrom torch import nn\n# from efficientnet_pytorch import EfficientNet\n# from efficientnet_pytorch.utils import MemoryEfficientSwish\nimport warnings\n\nimport random\nfrom torch.optim import Adam\nfrom torch.optim.lr_scheduler import OneCycleLR, ReduceLROnPlateau\nimport pydicom\nfrom pathlib import Path\nPath.ls = lambda x: list(x.iterdir())\nimport sys\n\nfrom sklearn.model_selection import GroupKFold\nfrom torch.utils.data import DataLoader, Subset\nfrom torch.optim.lr_scheduler import StepLR\nfrom datetime import datetime, timedelta\nfrom time import time\nimport torch.nn.functional as F\nimport copy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"package_path = '../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master'\nsys.path.append(package_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from efficientnet_pytorch import EfficientNet","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Config","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"warnings.simplefilter('ignore')\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark =True\n    \nseed_everything(42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Config:\n    def __init__(self):\n        self.FOLDS = 2\n        self.EPOCHS = 40\n        self.DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else 'cpu')\n        self.TRAIN_BS = 32\n        self.VALID_BS = 128\n        self.model_type = 'efficientnet-b3'\n        self.loss_fn = nn.L1Loss()\n        \nconfig = Config()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = Path('/kaggle/input/osic-pulmonary-fibrosis-progression/')\npath.ls()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Load dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(path/'train.csv')\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df.drop(np.nonzero(np.array(train_df['Patient'] == 'ID00011637202177653955184',dtype=float))[0], axis=0).reset_index(drop=True)\ntrain_df = train_df.drop(np.nonzero(np.array(train_df['Patient'] == 'ID00052637202186188008618',dtype=float))[0], axis=0).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Preprocessing","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_tab(df):\n    vector = [(df.Weeks.values[0] - 30 )/30]\n    \n    if df.Sex.values[0] == 'male':\n       vector.append(0)\n    else:\n       vector.append(1)\n    \n    if df.SmokingStatus.values[0] == 'Never smoked':\n        vector.extend([0,0])\n    elif df.SmokingStatus.values[0] == 'Ex-smoker':\n        vector.extend([1,1])\n    elif df.SmokingStatus.values[0] == 'Currently smokes':\n        vector.extend([0,1])\n    else:\n        vector.extend([1,0])\n    return np.array(vector) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TAB = {}\nTARGET = {}\nPerson = []\n\nfor i, p in tqdm(enumerate(train_df.Patient.unique())):\n    sub = train_df.loc[train_df.Patient == p]\n    fvc = sub.FVC.values\n    weeks = sub.Weeks.values\n    c = np.vstack([weeks, np.ones(len(weeks))]).T\n    a, b = np.linalg.lstsq(c, fvc)[0]\n    \n    TARGET[p] = a\n    TAB[p] = get_tab(sub)\n    Person.append(p)\n\nPerson = np.array(Person)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Read dicom image","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def get_img(path):\n    d = pydicom.dcmread(path)\n    return cv2.resize(d.pixel_array / 2**11, (512, 512))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Dataset class","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class Dataset:\n    def __init__(self, path, df, tabular, targets, mode , folder = 'train' ):\n        self.df = df\n        self.tabular = tabular\n        self.targets = targets\n        self.folder = folder\n        self.mode = mode\n        self.path = path\n        self.transform = transforms.Compose([\n            transforms.ToTensor()\n        ])\n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self,idx):\n        row = self.df.loc[idx,:]\n        pid = row['Patient']\n        # Path to record\n        record = self.path/self.folder/pid\n        # select image id\n        try: \n            \n            img_id =  np.random.choice(len(record.ls()))\n            \n            img = get_img(record.ls()[img_id])\n            img = self.transform(img)\n            tab = torch.from_numpy(self.tabular[pid]).float()\n            if self.mode == 'train':\n                target = torch.tensor(self.targets[pid])\n                return (img,tab), target\n            else:\n                return (img,tab)\n        except Exception as e:\n            print(e)\n            print(pid, img_id)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def collate_fn(b):\n    xs, ys = zip(*b)\n    imgs, tabs = zip(*xs)\n    return (torch.stack(imgs).float(),torch.stack(tabs).float()),torch.stack(ys).float()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Model Architecture","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"pretrained_model = {\n    'efficientnet-b0': '../input/efficientnet-pytorch/efficientnet-b0-08094119.pth',\n    'efficientnet-b3': '../input/efficientnet-pytorch/efficientnet-b3-c8376fa2.pth'\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class OSIC_Model(nn.Module):\n    def __init__(self,eff_name='efficienet-b0'):\n        super().__init__()\n        self.input = nn.Conv2d(1,3,kernel_size=3,padding=1,stride=2)\n        self.bn = nn.BatchNorm2d(3)\n        #self.model = EfficientNet.from_pretrained(f'efficientnet-{eff_name}-c8376fa2.pth')\n        self.model = EfficientNet.from_name(eff_name)\n        self.model.load_state_dict(torch.load(pretrained_model[eff_name]))\n        self.model._fc = nn.Linear(1536, 500, bias=True)\n        self.meta = nn.Sequential(nn.Linear(4, 500),\n                                  nn.BatchNorm1d(500),\n                                  nn.ReLU(),\n                                  nn.Dropout(p=0.2),\n                                  nn.Linear(500,250),\n                                  nn.BatchNorm1d(250),\n                                  nn.ReLU(),\n                                  nn.Dropout(p=0.2))\n        self.output = nn.Linear(500+250, 1)\n        self.relu = nn.ReLU()\n    \n    def forward(self, x,tab):\n        x = self.relu(self.bn(self.input(x)))\n        x = self.model(x)\n        tab = self.meta(tab)\n        x = torch.cat([x, tab],dim=1)\n        return self.output(x)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Kfold splits","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import KFold\n\ndef get_split_idxs(n_folds=5):\n    kv = KFold(n_splits=n_folds)\n    splits = []\n    for i,(train_idx, valid_idx) in enumerate(kv.split(Person)):\n        splits.append((train_idx, valid_idx))\n        \n    return splits","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"splits = get_split_idxs(n_folds=config.FOLDS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_loop(model, dl, opt, sched, device, loss_fn):\n    model.train()\n    for X,y in dl:\n        imgs = X[0].to(device)\n        tabs = X[1].to(device)\n        y = y.to(device)\n        outputs = model(imgs, tabs)\n        loss = loss_fn(outputs.squeeze(), y)\n        opt.zero_grad()\n        loss.backward()\n        opt.step()\n        if sched is not None:\n            sched.step()\n            \n\ndef eval_loop(model, dl, device, loss_fn):\n    model.eval()\n    final_outputs = []\n    final_loss = []\n    with torch.no_grad():\n        for X,y in dl:\n            imgs = X[0].to(device)\n            tabs = X[1].to(device)\n            y=y.to(device)\n\n            outputs = model(imgs, tabs)\n            loss = loss_fn(outputs.squeeze(), y)\n\n            final_outputs.extend(outputs.detach().cpu().numpy().tolist())\n            final_loss.append(loss.detach().cpu().numpy())\n        \n    return final_outputs, final_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from functools import partial\n\ndef apply_mod(m,f):\n    f(m)\n    for l in m.children(): apply_mod(l,f)\n\ndef set_grad(m,b):\n    if isinstance(m, (nn.Linear, nn.BatchNorm2d)): return \n    if hasattr(m, 'weight'):\n        for p in m.parameters(): p.requires_grad_(b)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models = {}\nfor i in range(config.FOLDS):\n    models[i] = OSIC_Model(config.model_type)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for k,v in models.items():\n    apply_mod(v.model, partial(set_grad, b=False))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### View some training Images","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train_df.loc[train_df['Patient'].isin(Person[:21])].reset_index(drop=True)\ntrain_ds = Dataset(path, train, TAB, TARGET, mode='train')\ntrain_dl = torch.utils.data.DataLoader(\n    dataset=train_ds,\n    batch_size=config.TRAIN_BS,\n    shuffle=True,\n    collate_fn=collate_fn        \n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig=plt.figure(figsize=(8, 8))\ncolumns = 4\nrows = 4\ni=1\nfor X,y in train_dl:\n    pass\nj=0\nfor i in range(1, columns*rows +1):\n    img = np.array(X[0][j].permute(1,2,0))\n    img = cv2.cvtColor(img ,cv2.COLOR_GRAY2RGB)\n    fig.add_subplot(rows, columns, i)\n    plt.imshow(img)\n    j += 1\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"history = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i, (train_idx, valid_idx) in enumerate(splits):\n    print(f\"===================Fold : {i} ================\")\n\n    train = train_df.loc[train_df['Patient'].isin(Person[train_idx])].reset_index(drop=True)\n    valid = train_df.loc[train_df['Patient'].isin(Person[valid_idx])].reset_index(drop=True)\n\n\n    train_ds = Dataset(path, train, TAB, TARGET, mode= 'train')\n    train_dl = torch.utils.data.DataLoader(\n        dataset=train_ds,\n        batch_size=config.TRAIN_BS,\n        shuffle=True,\n        collate_fn=collate_fn        \n    )\n\n    valid_ds = Dataset(path, valid, TAB, TARGET, mode='train')\n    valid_dl = torch.utils.data.DataLoader(\n        dataset=valid_ds,\n        batch_size=config.VALID_BS,\n        shuffle=False,\n        collate_fn=collate_fn\n    )\n\n    model = models[i]\n    model.to(config.DEVICE)\n    lr=1e-3\n    momentum = 0.9\n    \n    num_steps = len(train_dl)\n    optimizer = Adam(model.parameters(), lr=lr,weight_decay=0.1)\n    scheduler = OneCycleLR(optimizer, \n                           max_lr=lr,\n                           epochs=config.EPOCHS,\n                           steps_per_epoch=num_steps\n                           )\n    sched = ReduceLROnPlateau(optimizer,\n                              verbose=True,\n                              factor=0.1)\n    losses = []\n    for epoch in range(config.EPOCHS):\n        print(f\"=================EPOCHS {epoch+1}================\")\n        train_loop(model, train_dl, optimizer, scheduler, config.DEVICE,config.loss_fn)\n        metrics = eval_loop(model, valid_dl,config.DEVICE,config.loss_fn)\n        total_loss = np.array(metrics[1]).mean()\n        losses.append(total_loss)\n        print(\"Loss ::\\t\", total_loss)\n        sched.step(total_loss)\n        \n    model.to('cpu')\n    history.append(losses)\n    \n    \n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for k, m in models.items():\n    torch.save(m.state_dict(), f'fold_{k}.pth')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Prediction & Submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\nsub = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data= []\nfor i in range(len(test_df)):\n    for j in range(-12, 134):\n        test_data.append([test_df['Patient'][i],j,test_df['Age'][i],test_df['Sex'][i],test_df['SmokingStatus'][i], test_df['FVC'][i],test_df['Percent'][i\n        ],str(test_df['Patient'][i])+'_'+str(j)])\n\ntest_data = pd.DataFrame(test_data, columns=['Patient','Weeks','Age','Sex','SmokingStatus','FVC','Percent','Patient_Week'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TAB_test = {}\n\nPerson_test = []\n\nfor i, p in tqdm(enumerate(test_data.Patient.unique())):\n    sub = test_data.loc[test_data.Patient == p]\n\n    weeks = sub.Weeks.values\n    c = np.vstack([weeks, np.ones(len(weeks))]).T\n\n    TAB_test[p] = get_tab(sub)\n    Person_test.append(p)\n\nPerson_test = np.array(Person_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def collate_fn_test(b):\n    imgs, tabs = zip(*b)\n    return (torch.stack(imgs).float(),torch.stack(tabs).float())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TARGET = {}\ntest = test_data\ntest_ds = Dataset(path, test_data, TAB_test,TARGET, mode= 'test')\ntest_dl = torch.utils.data.DataLoader(\n    dataset=test_ds,\n    batch_size=128,\n    shuffle=True,\n    collate_fn=collate_fn_test        \n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"avg_predictions= np.zeros((730,1))\n\nfor i in range(len(models)):\n    \n    predictions = []\n    model = models[i]\n    model = model.to(config.DEVICE)\n    model.load_state_dict(torch.load('./fold_' +str(i)+'.pth'))\n    model.eval()\n    with torch.no_grad():\n        for X in test_dl:\n            imgs = X[0].to(config.DEVICE)\n            tabs = X[1].to(config.DEVICE)\n\n            pred = model(imgs, tabs)\n\n            predictions.extend(pred.detach().cpu().numpy().tolist())\n    avg_predictions += predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = avg_predictions / len(models)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fvc = []\nconf = []\nfor i in range(len(test_data)):\n    p =test_data['Patient'][i]\n    B_test = predictions[i][0] * test_df.Weeks.values[test_df.Patient == p][0]\n    fvc.append(predictions[i][0] * test_data['Weeks'][i] + test_data['FVC'][i] - B_test)\n    conf.append(test_data['Percent'][i] + abs(predictions[i][0]) * abs(test_df.Weeks.values[test_df.Patient == p][0] - test_data['Weeks'][i]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nsubmission = test_data[['Patient_Week']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv')\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm ={}\nfor i in range(len(submission)):\n    subm[submission['Patient_Week'][i]]=[float(fvc[i]),float(conf[i])]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['FVC'] = sub['FVC'].astype(float)\nsub['Confidence'] = sub['Confidence'].astype(float)\nfor i in range(len(sub)):\n    id = sub['Patient_Week'][i]\n    sub['FVC'][i]= float(subm[id][0])\n    sub['Confidence'][i] = float(subm[id][1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.to_csv('submission_img.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Regression Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"root_dir = Path('/kaggle/input/osic-pulmonary-fibrosis-progression')\nmodel_dir = '/kaggle/working/model_states'\nnum_kfolds = 5\nbatch_size = 32\nlearning_rate = 3e-3\nnum_epochs = 1000\nes_patience = 10\nquantiles = (0.2, 0.5, 0.8)\nmodel_name ='descartes'\ntensorboard_dir = Path('/kaggle/working/runs')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Inference","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class ClinicalDataset(Dataset):\n    def __init__(self, root_dir, mode, transform=None):\n        self.transform = transform\n        self.mode = mode\n\n        tr = pd.read_csv(Path(root_dir)/\"train.csv\")\n        tr.drop_duplicates(keep=False, inplace=True, subset=['Patient', 'Weeks'])\n        chunk = pd.read_csv(Path(root_dir)/\"test.csv\")\n\n        sub = pd.read_csv(Path(root_dir)/\"sample_submission.csv\")\n        sub['Patient'] = sub['Patient_Week'].apply(lambda x: x.split('_')[0])\n        sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))\n        sub = sub[['Patient', 'Weeks', 'Confidence', 'Patient_Week']]\n        sub = sub.merge(chunk.drop('Weeks', axis=1), on=\"Patient\")\n\n        tr['WHERE'] = 'train'\n        chunk['WHERE'] = 'val'\n        sub['WHERE'] = 'test'\n        data = tr.append([chunk, sub])\n\n        data['min_week'] = data['Weeks']\n        data.loc[data.WHERE == 'test', 'min_week'] = np.nan\n        data['min_week'] = data.groupby('Patient')['min_week'].transform('min')\n\n        base = data.loc[data.Weeks == data.min_week]\n        base = base[['Patient', 'FVC']].copy()\n        base.columns = ['Patient', 'min_FVC']\n        base['nb'] = 1\n        base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')\n        base = base[base.nb == 1]\n        base.drop('nb', axis=1, inplace=True)\n\n        data = data.merge(base, on='Patient', how='left')\n        data['base_week'] = data['Weeks'] - data['min_week']\n        del base\n\n        COLS = ['Sex', 'SmokingStatus']\n        self.FE = []\n        for col in COLS:\n            for mod in data[col].unique():\n                self.FE.append(mod)\n                data[mod] = (data[col] == mod).astype(int)\n\n        data['age'] = (data['Age'] - data['Age'].min()) / \\\n                      (data['Age'].max() - data['Age'].min())\n        data['BASE'] = (data['min_FVC'] - data['min_FVC'].min()) / \\\n                       (data['min_FVC'].max() - data['min_FVC'].min())\n        data['week'] = (data['base_week'] - data['base_week'].min()) / \\\n                       (data['base_week'].max() - data['base_week'].min())\n        data['percent'] = (data['Percent'] - data['Percent'].min()) / \\\n                          (data['Percent'].max() - data['Percent'].min())\n        self.FE += ['age', 'percent', 'week', 'BASE']\n\n        self.raw = data.loc[data.WHERE == mode].reset_index()\n        del data\n\n    def __len__(self):\n        return len(self.raw)\n\n    def __getitem__(self, idx):\n        if torch.is_tensor(idx):\n            idx = idx.tolist()\n\n        sample = {\n            'patient_id': self.raw['Patient'].iloc[idx],\n            'features': self.raw[self.FE].iloc[idx].values,\n            'target': self.raw['FVC'].iloc[idx]\n        }\n        if self.transform:\n            sample = self.transform(sample)\n\n        return sample\n\n    def group_kfold(self, n_splits):\n        gkf = GroupKFold(n_splits=n_splits)\n        groups = self.raw['Patient']\n        for train_idx, val_idx in gkf.split(self.raw, self.raw, groups):\n            train = Subset(self, train_idx)\n            val = Subset(self, val_idx)\n            yield train, val\n\n    def group_split(self, test_size=0.2):\n        \"\"\"To test no-kfold\n        \"\"\"\n        gss = GroupShuffleSplit(n_splits=1, test_size=test_size)\n        groups = self.raw['Patient']\n        idx = list(gss.split(self.raw, self.raw, groups))\n        train = Subset(self, idx[0][0])\n        val = Subset(self, idx[0][1])\n        return train, val","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## NeuralNet model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class QuantModel(nn.Module):\n    def __init__(self, in_tabular_features=9, out_quantiles=3):\n        super(QuantModel, self).__init__()\n        self.fc1 = nn.Linear(in_tabular_features, 200)\n        self.fc2 = nn.Linear(200, 100)\n        self.fc3 = nn.Linear(100, out_quantiles)\n\n    def forward(self, x):\n        x = F.leaky_relu(self.fc1(x))\n        #x = self.bn1(x)\n        x = F.leaky_relu(self.fc2(x))\n        x = self.fc3(x)\n        return x\n\n\ndef quantile_loss(preds, target, quantiles):\n    #assert not target.requires_grad\n    assert len(preds) == len(target)\n    losses = []\n    for i, q in enumerate(quantiles):\n        errors = target - preds[:, i]\n        losses.append(torch.max((q - 1) * errors, q * errors).unsqueeze(1))\n    loss = torch.mean(torch.sum(torch.cat(losses, dim=1), dim=1))\n    return loss\n\ndef metric_loss(pred_fvc,true_fvc):\n        #Implementation of the metric in pytorch\n    sigma = pred_fvc[:, 2] - pred_fvc[:, 0]\n    true_fvc=torch.reshape(true_fvc,pred_fvc[:,1].shape)\n    sigma_clipped=torch.clamp(sigma,min=70)\n    delta=torch.clamp(torch.abs(pred_fvc[:,1]-true_fvc),max=1000)\n    metric=torch.div(-torch.sqrt(torch.tensor([2.0]))*delta,sigma_clipped)-torch.log(torch.sqrt(torch.tensor([2.0]))*sigma_clipped)\n    return metric","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"models = []\n\ntrain_loss = []\nval_lll = []\n# Load the data\ndata = ClinicalDataset(root_dir=root_dir, mode='train')\nfolds = data.group_kfold(num_kfolds)\n#t0 = time()\n#if len(testfiles) == 5:\n    #f= open(\"/kaggle/working/training.log\",\"w+\") \nfor fold, (trainset, valset) in enumerate(folds):\n    best_val = None\n    patience = es_patience\n    model = QuantModel().to(device)\n    params = [p for p in model.parameters() if p.requires_grad]\n    optimizer = Adam(params, lr=learning_rate)\n# lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.1)\n    lr_scheduler = StepLR(optimizer, step_size=20, gamma=0.5)\n\n\n    print(\"==\"*20+\"Fold \"+str(fold+1)+\"==\"*20)\n    Path(model_dir).mkdir(parents=True, exist_ok=True)\n    model_path = model_dir+f'/fold_{fold}.pth'\n    now = datetime.now()\n    dataset_sizes = {'train': len(trainset), 'val': len(valset)}\n    dataloaders = {\n            'train': DataLoader(trainset, batch_size=batch_size,\n                                shuffle=True, num_workers=2),\n            'val': DataLoader(valset, batch_size=batch_size,\n                              shuffle=False, num_workers=2)\n    }\n    train_loss_epoch = []\n    val_lll_epoch = []\n    for epoch in range(num_epochs):\n        start_time = time()\n        itr = 1\n        model.train()\n        train_losses =[]\n        for batch in dataloaders['train']:\n            inputs = batch['features'].float().to(device)\n            targets = batch['target'].to(device)\n            optimizer.zero_grad()\n            with torch.set_grad_enabled(True):\n                preds = model(inputs)\n                loss = quantile_loss(preds, targets, quantiles)\n                train_losses.append(loss.tolist())\n                loss.backward()\n                optimizer.step()\n           \n            if itr % 50 == 0:\n                print(f\"Epoch #{epoch+1} Iteration #{itr} loss: {loss}\")\n            itr += 1\n            \n        model.eval()\n        all_preds = []\n        all_targets = []\n        for batch in dataloaders['val']:\n            inputs = batch['features'].float().to(device)\n            targets = batch['target']\n            optimizer.zero_grad()\n            with torch.set_grad_enabled(False):\n                preds = model(inputs)\n                all_preds.extend(preds.detach().cpu().numpy().tolist())\n                all_targets.extend(targets.numpy().tolist()) # np.append(an_array, row_to_append, 0)\n        all_preds =torch.FloatTensor(all_preds)\n        all_targets =torch.FloatTensor(all_targets)\n        val_metric_loss = metric_loss(all_preds, all_targets)\n        val_metric_loss = torch.mean(val_metric_loss).tolist()\n\n        lr_scheduler.step()\n        print(f\"Epoch #{epoch+1}\",\"Training loss : {0:.4f}\".format(np.mean(train_losses)),\"Validation LLL : {0:.4f}\".format(val_metric_loss),\"Time taken :\",str(timedelta(seconds=time() - start_time))[:7])\n        train_loss_epoch.append(np.mean(train_losses))\n        val_lll_epoch.append(val_metric_loss)\n        if not best_val:\n            best_val = val_metric_loss  # So any validation roc_auc we have is the best one for now\n            print(\"Info : Saving model\")\n            torch.save(copy.deepcopy(model.state_dict()), model_path)  # Saving the model\n        if val_metric_loss > best_val:\n            print(\"Info : Saving model as Laplace Log Likelihood is increased from {0:.4f}\".format(best_val),\"to {0:.4f}\".format(val_metric_loss))\n            best_val = val_metric_loss\n            patience = es_patience  # Resetting patience since we have new best validation accuracy\n            torch.save(copy.deepcopy(model.state_dict()), model_path)  # Saving current best model torch.save(model.state_dict(), 'fasterrcnn_resnet50_fpn.pth')\n        else:\n            patience -= 1\n            if patience == 0:\n                print('Early stopping. Best Validation Laplace Log Likelihood: {:.3f}'.format(best_val))\n                break\n    model.load_state_dict(torch.load(model_path))\n    models.append(model)\n    train_loss.append(train_loss_epoch)\n    val_lll.append(val_lll_epoch)\nprint('Finished Training of BiLSTM Model')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Generating submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"data = ClinicalDataset(root_dir, mode='test')\n\navg_preds = np.zeros((len(data), len(quantiles)))\n\nfor model in models:\n    dataloader = DataLoader(data, batch_size=batch_size,\n                            shuffle=False, num_workers=2)\n    preds = []\n    for batch in dataloader:\n        inputs = batch['features'].float()\n        inputs = inputs.cuda()\n        with torch.no_grad():\n            x = model(inputs)\n            preds.append(x)\n\n    preds = torch.cat(preds, dim=0).cpu().numpy()\n    avg_preds += preds\n\navg_preds /= len(models)\ndf = pd.DataFrame(data=avg_preds, columns=list(quantiles))\ndf['Patient_Week'] = data.raw['Patient_Week']\ndf['FVC'] = df[quantiles[1]]\ndf['Confidence'] = df[quantiles[2]] - df[quantiles[0]]\ndf = df.drop(columns=list(quantiles))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(df))\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.to_csv('submission_reg.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Ensemble (Simple Blend)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_img = pd.read_csv('./submission_img.csv')\nsub_reg = pd.read_csv('./submission_reg.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_img.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_reg.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(len(sub_img)):\n    sub_img['FVC'][i] = 0.25*sub_img['FVC'][i] + 0.75*sub_reg.loc[sub_reg.Patient_Week == sub_img['Patient_Week'][i]]['FVC']\n    sub_img['Confidence'][i] = 0.26*sub_img['Confidence'][i] + 0.74*sub_reg.loc[sub_reg.Patient_Week == sub_img['Patient_Week'][i]]['Confidence']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = sub_img\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.to_csv('submission.csv',index= False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}