{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Pytoch CNN+Tabler+Pixel_Stats\n\nCompetition URL: https://www.kaggle.com/c/osic-pulmonary-fibrosis-progression\n\n\n\n#### Reference\n\n- effcient net の参考になるやつ\n    - https://www.kaggle.com/nroman/melanoma-pytorch-starter-efficientnet\n- quantile_loss を使っているkernel\n    - https://www.kaggle.com/carlossouza/quantile-regression-pytorch-tabular-data-only","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Import Libraries","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path = [\n    '/kaggle/input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master',\n] + sys.path","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport sys\nimport cv2\nimport time\nimport random\nimport pickle\nimport inspect\nimport datetime\nfrom PIL import Image\nfrom tqdm import tqdm\nfrom copy import deepcopy\nfrom importlib import machinery\n\n\nimport seaborn as sns\nsns.set_style(\"darkgrid\")\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\n\nimport numpy as np\nimport pandas as pd\n\nimport torch\nimport torch.nn as nn\nfrom torch import optim\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\n\nfrom efficientnet_pytorch import EfficientNet\n\nfrom sklearn import model_selection\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler\n\nimport warnings\nwarnings.simplefilter('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if os.path.exists('/kaggle/input'):\n    DATA_DIR = '/kaggle/input/osic-pulmonary-fibrosis-progression/'\n    PROCESSING_DIR = '/kaggle/input/osic-preprocessing-data/'\n    IM_FOLDER = '/kaggle/input/osic-average-images/'\nelse:\n    DATA_DIR = '../data/raw/'\n    PROCESSING_DIR = '../data/processing/'\n    IM_FOLDER = '../data/processing/average_image/'\n\nSEED = 55\n\nNUM_FOLD = 5\nIMG_SIZE = 256\nBATCH_SIZE = 64\nNUM_EPOCH = 20\n\nEFF_NET = 1\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed=1234):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    \nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load Data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocessing(data: pd.DataFrame, is_test: bool = True) -> pd.DataFrame:\n    # Create Common Features.\n    features = pd.DataFrame()\n    for patient, u_data in data.groupby('Patient'):\n        feature = pd.DataFrame({\n            'current_FVC': u_data['FVC'],\n            'current_Percent': u_data['Percent'],\n            'current_Age': u_data['Age'],\n            'current_Week': u_data['Weeks'],\n            'Patient': u_data['Patient'],\n            'Sex': u_data['Sex'].map({'Female': 0, 'Male': 1}),\n            'SmokingStatus': u_data['SmokingStatus']\n#             'SmokingStatus': u_data['SmokingStatus'].map({'Currently smokes': 0, 'Never smoked': 1, 'Ex-smoker': 2}),\n        })\n        features = pd.concat([features, feature])\n    features = pd.get_dummies(features, columns=['SmokingStatus'])\n    # Create Label Data.\n    if is_test:\n        label = pd.read_csv(os.path.join(DATA_DIR, 'sample_submission.csv'), usecols=['Patient_Week'])\n        label['Patient'] = label['Patient_Week'].apply(lambda x: x.split('_')[0])\n        label['pred_Weeks'] = label['Patient_Week'].apply(lambda x: x.split('_')[1]).astype(int)\n        label['FVC'] = np.nan\n\n        dst_data = pd.merge(label, features, how='left', on='Patient')\n    else:\n        label = pd.DataFrame({\n            'Patient_Week': data['Patient'].astype(str) + '_' + data['Weeks'].astype(str),\n            'Patient': data['Patient'],\n            'pred_Weeks': data['Weeks'],\n            'FVC': data['FVC']\n        })\n\n        dst_data = pd.merge(label, features, how='outer', on='Patient')\n        dst_data = dst_data.query('pred_Weeks!=current_Week')\n\n    dst_data = dst_data.assign(passed_Weeks=dst_data['pred_Weeks'] - dst_data['current_Week'])\n    return dst_data\n\n# Train Data.\ntrain = pd.read_csv(os.path.join(DATA_DIR, 'train.csv'))\ntrain = preprocessing(train, is_test=False)\ntrain.drop(['SmokingStatus_Currently smokes'], axis=1, inplace=True)\n\ntrain_PiXelStats_path = os.path.join(PROCESSING_DIR, 'train_pixel_stats.csv')\ntrain_picxel_stats = pd.read_csv(train_PiXelStats_path)\ntrain = train.merge(train_picxel_stats, how='left', on='Patient')\ntrain.dropna(axis=0, inplace=True)\n\n# Test Data.\ntest = pd.read_csv(os.path.join(DATA_DIR, 'test.csv'))\ntest = preprocessing(test, is_test=True)\n\ntest_PiXelStats_path = os.path.join(PROCESSING_DIR, 'test_pixel_stats.csv')\ntest_picxel_stats = pd.read_csv(test_PiXelStats_path)\ntest = test.merge(test_picxel_stats, how='left', on='Patient')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.shape)\ndisplay(train.head())\n\nprint(test.shape)\ndisplay(test.head())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Ref: https://www.kaggle.com/nroman/melanoma-pytorch-starter-efficientnet\n\nclass OSICDataset(Dataset):\n    \n    def __init__(self, df: pd.DataFrame, target: str, imfolder: str, train: bool = True, transforms = None, meta_features: list = None):\n        \"\"\"\n        OSIC Dataset for pytorch. \n        \n        Parameters\n        ----------\n        df : pd.Dataframe\n            DataFrame with data description.\n        target : str\n            target column.\n        imfolder : str\n            folder with images.\n        train : bool\n            flag  of whether train or test dataset.\n        transformers : torchvision.transforms\n            image transformation method to be applid\n        meta_features : list\n            list of features with meta information, such as sex and age.\n        \"\"\"\n        self.df = df\n        self.target = target\n        self.imfolder = imfolder\n        self.transforms = transforms\n        self.train = train\n        self.meta_features = meta_features\n        \n    def __getitem__(self, index):\n        patient = self.df.iloc[index]['Patient']\n        im_path = os.path.join(self.imfolder, patient + '.pt')\n        \n        x = torch.load(im_path)\n        meta = self.df.iloc[index][self.meta_features].to_numpy(dtype=np.float32)\n        \n        if self.transforms:\n            x = self.transforms(x)\n            \n        if self.train:\n            y = self.df.iloc[index][self.target]\n            return (x, meta), y\n        else:\n            return (x, meta)\n        \n    def __len__(self):\n        return len(self.df)\n        \n    \nclass Net(nn.Module):\n    def __init__(self, arch, n_meta_features: int):\n        super(Net, self).__init__()\n        self.arch = arch\n        self.arch._fc = nn.Linear(in_features=1280, out_features=512, bias=True)\n        self.meta = nn.Sequential(\n            nn.Linear(n_meta_features, 100),\n            nn.BatchNorm1d(100),\n            nn.ReLU(),\n            nn.Dropout(p=0.2),\n            nn.Linear(100, 50),  # FC layer output will have 250 features\n            nn.BatchNorm1d(50),\n            nn.ReLU(),\n            nn.Dropout(p=0.2)\n        )\n        self.ouput = nn.Linear(512 + 50, 3)\n        \n    def forward(self, inputs):\n        x, meta = inputs\n        cnn_features = self.arch(x)\n        meta_features = self.meta(meta)\n        features = torch.cat((cnn_features, meta_features), dim=1)\n        output = self.ouput(features)\n        return output\n\n\ndef osic_loss(target, pred, sigma):   \n    n_sqrt = torch.sqrt(torch.tensor(2.0))\n    delta = torch.abs(target - pred)\n\n    sigma[sigma<70] = 70.0\n    delta[delta>1000] = 1000.0\n    \n    metric = - (n_sqrt * delta / sigma) - torch.log(n_sqrt * sigma)\n    loss = torch.mean(metric)\n    return loss\n    \n\ndef quantile_loss(preds, target, quantiles):\n    assert not target.requires_grad\n    assert preds.size(0) == target.size(0)\n    losses = []\n    for i, q in enumerate(quantiles):\n        errors = target - preds[:, i]\n        losses.append(torch.max((q - 1) * errors, q * errors).unsqueeze(1))\n    loss = torch.mean(torch.sum(torch.cat(losses, dim=1), dim=1))\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CosineAnnealingWarmUpRestarts(_LRScheduler):\n    def __init__(self, optimizer, T_0, T_mult=1, eta_max=0.1, T_up=0, gamma=1., last_epoch=-1):\n        if T_0 <= 0 or not isinstance(T_0, int):\n            raise ValueError(\"Expected positive integer T_0, but got {}\".format(T_0))\n        if T_mult < 1 or not isinstance(T_mult, int):\n            raise ValueError(\"Expected integer T_mult >= 1, but got {}\".format(T_mult))\n        if T_up < 0 or not isinstance(T_up, int):\n            raise ValueError(\"Expected positive integer T_up, but got {}\".format(T_up))\n        self.T_0 = T_0\n        self.T_mult = T_mult\n        self.base_eta_max = eta_max\n        self.eta_max = eta_max\n        self.T_up = T_up\n        self.T_i = T_0\n        self.gamma = gamma\n        self.cycle = 0\n        super(CosineAnnealingWarmUpRestarts, self).__init__(optimizer, last_epoch)\n        self.T_cur = last_epoch\n    \n    def get_lr(self):\n        if self.T_cur == -1:\n            return self.base_lrs\n        elif self.T_cur < self.T_up:\n            return [(self.eta_max - base_lr)*self.T_cur / self.T_up + base_lr for base_lr in self.base_lrs]\n        else:\n            return [base_lr + (self.eta_max - base_lr) * (1 + math.cos(math.pi * (self.T_cur-self.T_up) / (self.T_i - self.T_up))) / 2\n                    for base_lr in self.base_lrs]\n\n    def step(self, epoch=None):\n        if epoch is None:\n            epoch = self.last_epoch + 1\n            self.T_cur = self.T_cur + 1\n            if self.T_cur >= self.T_i:\n                self.cycle += 1\n                self.T_cur = self.T_cur - self.T_i\n                self.T_i = (self.T_i - self.T_up) * self.T_mult + self.T_up\n        else:\n            if epoch >= self.T_0:\n                if self.T_mult == 1:\n                    self.T_cur = epoch % self.T_0\n                    self.cycle = epoch // self.T_0\n                else:\n                    n = int(math.log((epoch / self.T_0 * (self.T_mult - 1) + 1), self.T_mult))\n                    self.cycle = n\n                    self.T_cur = epoch - self.T_0 * (self.T_mult ** n - 1) / (self.T_mult - 1)\n                    self.T_i = self.T_0 * self.T_mult ** (n)\n            else:\n                self.T_i = self.T_0\n                self.T_cur = epoch\n                \n        self.eta_max = self.base_eta_max * (self.gamma**self.cycle)\n        self.last_epoch = math.floor(epoch)\n        for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):\n            param_group['lr'] = lr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Resize:\n    def __init__(self, image_size):\n        self.size = image_size\n    \n    def __call__(self, image):\n        image = cv2.resize(image, (self.size, self.size))\n        return image\n\ntransform = transforms.Compose([\n    Resize(IMG_SIZE),\n    transforms.ToTensor(),\n    transforms.Normalize([0.5], [0.5]),\n    transforms.Lambda(lambda x: torch.cat([x, x, x], 0))\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_metric_loss(history: dict, fold: int):\n    \"\"\"\n    Parameters\n    ----------\n    history : dict\n        history keys is train_metric, valid_metric, train_loss, valid_loss.\n    epochs : int\n        number of fold.\n    \"\"\"\n    data_size = len(history['train_metric'])\n    \n    plt.figure(figsize=(15,5))\n    # train and valid metric line\n    plt.plot(np.arange(data_size), history['train_metric'], '-o', label='Train Metric', color='#ff7f0e')\n    plt.plot(np.arange(data_size), history['valid_metric'], '-o', label='Valid Metric', color='#1f77b4')\n    # point best metric epoch\n    x = np.argmax(history['valid_metric'])\n    y = np.max(history['valid_metric'])\n    xdist = plt.xlim()[1] - plt.xlim()[0]\n    ydist = plt.ylim()[1] - plt.ylim()[0]\n    plt.scatter(x,y,s=200,color='#1f77b4')\n    plt.text(x-0.03*xdist, y-0.13*ydist, f'max valid_metric\\n{y:.2f}', size=14)\n    # Set Label\n    plt.ylabel('Metric',size=14)\n    plt.xlabel('Epoch',size=14)\n    plt.legend(loc=2)\n\n    plt2 = plt.gca().twinx()\n    # train and valid loss line\n    plt2.plot(np.arange(data_size), history['train_loss'], '-o', label='Train Loss', color='#2ca02c')\n    plt2.plot(np.arange(data_size), history['valid_loss'], '-o', label='Valid Loss', color='#d62728')\n    # point best loss epoch\n    x = np.argmin(history['valid_loss'])\n    y = np.min(history['valid_loss'])\n    ydist = plt.ylim()[1] - plt.ylim()[0]\n    plt.scatter(x, y, s=200, color='#d62728')\n    plt.text(x-0.03*xdist, y+0.05*ydist, 'min loss', size=14)\n    plt.ylabel('Loss', size=14)\n    plt.title(f'FOLD {fold} - Image Size {IMG_SIZE}')\n    plt.legend(loc=3)\n    plt.show()\n\n# MEMO: plot test\n# history = {name: [] for name in ['train_metric', 'valid_metric', 'train_loss', 'valid_loss']}\n\n# history['train_metric'] = np.arange(NUM_EPOCH)\n# history['valid_metric'] = np.arange(NUM_EPOCH) - 0.5\n\n# history['train_loss'] = np.random.rand(NUM_EPOCH)\n# history['valid_loss'] = np.random.rand(NUM_EPOCH)\n    \n# plot_metric_loss(history, 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"effcient_pretrained_path = [\n    '../input/efficientnet-pytorch/efficientnet-b0-08094119.pth',\n    '../input/efficientnet-pytorch/efficientnet-b1-dbc7070a.pth',\n    '../input/efficientnet-pytorch/efficientnet-b2-27687264.pth',\n    '../input/efficientnet-pytorch/efficientnet-b3-c8376fa2.pth',\n    '../input/efficientnet-pytorch/efficientnet-b4-e116e8b3.pth',\n    '../input/efficientnet-pytorch/efficientnet-b5-586e6cc6.pth',\n    '../input/efficientnet-pytorch/efficientnet-b6-c76e70fd.pth',\n    '../input/efficientnet-pytorch/efficientnet-b7-dcc49843.pth'\n]\n\nif os.path.exists('/kaggle/input'):\n    arch = EfficientNet.from_name(f'efficientnet-b{EFF_NET}')\n    arch.load_state_dict(torch.load(effcient_pretrained_path[EFF_NET]))\nelse:\n    arch = EfficientNet.from_pretrained(f'efficientnet-b{EFF_NET}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"drop_cols = ['Patient', 'Patient_Week', 'FVC']\nmeta_features = [c for c in train.columns.tolist() if c not in drop_cols]\n\nscaler = MinMaxScaler()\nscaler.fit(train[meta_features])\ntrain.loc[:, meta_features] = scaler.transform(train[meta_features])\ntest.loc[:, meta_features] = scaler.transform(test[meta_features])\n\n# ===== Group KFold ======\noof = np.zeros((len(train), 3))\ngkf = model_selection.GroupShuffleSplit(n_splits=NUM_FOLD, random_state=SEED)\nfor fold, (train_idx, valid_idx) in enumerate(gkf.split(X=train, y=train['FVC'], groups=train['Patient'])):\n    print('\\n' + '#'*20)\n    print('#'*5, f' Fold {fold+1}')\n    print('#'*20 + '\\n')\n    print(f'Train Size: {len(train_idx)}')\n    print(f'Valid Size: {len(valid_idx)}', '\\n')\n    # Model, Optimizer\n    model = Net(arch=arch, n_meta_features=len(meta_features)).to(device)\n    optimizer = optim.Adam(model.parameters(), lr=0.0001)\n    # scheduler = CosineAnnealingWarmUpRestarts(optimizer, T_0=150, T_mult=1, eta_max=0.1,  T_up=10, gamma=0.5)\n    # Dataset and Dataloader.\n    trainset = OSICDataset(\n        df=train.iloc[train_idx].reset_index(drop=True),\n        target='FVC',\n        imfolder=IM_FOLDER,\n        train=True,\n        transforms=transform,\n        meta_features=meta_features\n    )\n    validset = OSICDataset(\n        df=train.iloc[valid_idx].reset_index(drop=True),\n        target='FVC',\n        imfolder=IM_FOLDER,\n        train=True,\n        transforms=transform,\n        meta_features=meta_features\n    )\n    train_loader = DataLoader(dataset=trainset, batch_size=BATCH_SIZE, shuffle=True, num_workers=2)\n    val_loader = DataLoader(dataset=validset, batch_size=BATCH_SIZE, shuffle=False, num_workers=2)\n    # Define Fold Initial Variables.\n    updated = False\n    patience_cnt = 0\n    patience_es = 5\n    best_metrc = -999\n    history = {name: [] for name in ['train_metric', 'valid_metric', 'train_loss', 'valid_loss']}\n    for epoch in range(NUM_EPOCH):\n        start_time = time.time()\n        # Train Loader\n        model.train()\n        train_preds = torch.zeros((len(train_idx), 3), dtype=torch.float32, device=device)\n        for j, (x, y) in enumerate(train_loader):\n            x[0] = torch.tensor(x[0], device=device, dtype=torch.float32)\n            x[1] = torch.tensor(x[1], device=device, dtype=torch.float32)\n            y = torch.tensor(y, device=device, dtype=torch.float32)\n\n            z = model(x)\n            loss = quantile_loss(z, y, (0.2, 0.5, 0.8))\n            # Zero gradients, perform a backward pass, and update the weights.\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n            # Store train predict values.\n            start_idx = j * train_loader.batch_size\n            end_idx = start_idx + x[0].shape[0]\n            train_preds[start_idx:end_idx] = z\n\n        target = torch.tensor(trainset.df[trainset.target], device=device, dtype=torch.float32)\n        pred = train_preds[:, 1]\n        sigma = train_preds[:, 2] - train_preds[:, 0]\n        train_metric = osic_loss(target, pred, sigma).to('cpu').item()\n        history['train_metric'].append(train_metric)\n        \n        train_loss = quantile_loss(train_preds, target, (0.2, 0.5, 0.8)).to('cpu').item()\n        history['train_loss'].append(train_loss)\n\n        # Valid Loader\n        model.eval()\n        valid_preds = torch.zeros((len(valid_idx), 3), dtype=torch.float32, device=device)\n        with torch.no_grad():  # Do not calculate gradient since we are only predicting\n            for j, (x_val, y_val) in enumerate(val_loader):\n                x_val[0] = torch.tensor(x_val[0], device=device, dtype=torch.float32).clone().detach()\n                x_val[1] = torch.tensor(x_val[1], device=device, dtype=torch.float32).clone().detach()\n                y_val = torch.tensor(y_val, device=device, dtype=torch.float32, requires_grad=True).clone().detach()\n\n                start_idx = j * val_loader.batch_size\n                end_idx = start_idx + x_val[0].shape[0]\n                valid_preds[start_idx:end_idx] = model(x_val)\n\n        target = torch.tensor(validset.df[validset.target], device=device, dtype=torch.float32)\n        pred = valid_preds[:, 1]\n        sigma = valid_preds[:, 2] - valid_preds[:, 0]\n        valid_metric = osic_loss(target, pred, sigma).to('cpu').item()\n        history['valid_metric'].append(valid_metric)\n        \n        valid_loss = quantile_loss(valid_preds, target, (0.2, 0.5, 0.8)).to('cpu').item()\n        history['valid_loss'].append(valid_loss)\n        \n        # Log.\n        train_time = str(datetime.timedelta(seconds=time.time() - start_time))[:7]\n        print(f'Epoch {epoch:03}: | Train Loss: {train_loss:.02f} | Valid Loss: {valid_loss:.02f} | Train Metric: {train_metric:.02f} | Valid Metric: {valid_metric:.02f} | Training Time: {train_time}')\n\n        # Early Stopping\n        if valid_metric > best_metrc:\n            updated = True\n            # update OOf values.\n            oof[valid_idx] = valid_preds.to('cpu').detach().numpy().copy()\n            # update best metric.\n            best_metrc = valid_metric\n            torch.save(model.state_dict(), f'best_model_{fold+1}_fold.pt')\n        else:\n            patience_cnt += 1\n            \n        if patience_cnt >= patience_es:\n            # If it has never been updated\n            if not updated:\n                # update OOf values.\n                oof[valid_idx] = valid_preds.to('cpu').detach().numpy().copy()\n                # update best metric.\n                best_metrc = valid_metric\n                torch.save(model.state_dict(), f'best_model_{fold+1}_fold.pt')\n            \n            print(f\"Early stopping: Best Valid Metric is {best_metrc:.02f}\")\n            break\n\n    plot_metric_loss(history, fold + 1)\n\n\n# Export OOF.\noof_df = pd.DataFrame(oof, columns=['20', '50', '80'])\noof_df.to_csv('oof.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Evaluation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"target = torch.tensor(train['FVC'], device=device, dtype=torch.float32)\npred = torch.tensor(oof[:, 1], device=device, dtype=torch.float32)\nsigma = torch.tensor(oof[:, 2] - oof[:, 0], device=device, dtype=torch.float32)\noof_metric = osic_loss(target, pred, sigma)\n\nprint(f'OOF Metric Score: {oof_metric:.2f}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"testset = OSICDataset(\n    df=test.reset_index(drop=True),\n    target='FVC',\n    imfolder=IM_FOLDER,\n    train=False,\n    transforms=transform,\n    meta_features=meta_features\n)\n\n\npreds = np.zeros((len(test), 3))\nfor fold in range(NUM_FOLD):\n    test_loader = DataLoader(dataset=testset, batch_size=64, shuffle=True, num_workers=2)\n    \n    model = Net(arch=arch, n_meta_features=len(meta_features)).to(device)\n    model.load_state_dict(torch.load(f'/kaggle/working/best_model_{fold+1}_fold.pt'))\n    \n    model.eval()\n    with torch.no_grad():  # Do not calculate gradient since we are only predicting\n        for j, x in enumerate(test_loader):\n            x[0] = torch.tensor(x[0], device=device, dtype=torch.float32).clone().detach()\n            x[1] = torch.tensor(x[1], device=device, dtype=torch.float32).clone().detach()\n\n            start_idx = j * test_loader.batch_size\n            end_idx = start_idx + x[0].shape[0]\n            preds[start_idx:end_idx] += model(x).to('cpu').detach().numpy().copy()\n        \npreds = preds / NUM_FOLD","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_df = pd.DataFrame({\n    'Patient_Week': test['Patient_Week'].values,\n    'FVC': preds[:, 1],\n    'Confidence': preds[:, 2] - preds[:, 0]\n})\n\nsubmission = pd.read_csv(os.path.join(DATA_DIR, 'sample_submission.csv'), usecols=['Patient_Week'])\nsubmission = submission.merge(pred_df, how='left', on='Patient_Week')\n\n\nif os.path.exists('/kaggle/input'):\n    submission.to_csv('submission.csv', index=False)\n\nprint(submission.shape)\nsubmission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}