{"cells":[{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"!pip install efficientnet_pytorch","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"A lot of code in this kernel is directly inspired and taken from https://www.kaggle.com/khoongweihao/k-fold-tf-efficientnet-models-training/notebook . It would have been so easy to get this up and running. The attributed kernel is implemented in TensorFlow. I have implemented it in pytorch.","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"scrolled":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport torch\nfrom torchvision import models\nfrom pathlib import Path\nPath.ls = lambda x: list(x.iterdir())\n\nimport cv2 \nimport pydicom\nfrom tqdm import tqdm\nfrom matplotlib import pyplot as plt\nfrom torchvision import transforms\n\nfrom torch import nn\nfrom efficientnet_pytorch import EfficientNet\nfrom efficientnet_pytorch.utils import MemoryEfficientSwish\n\nfrom torch.optim import Adam\nfrom torch.optim.lr_scheduler import OneCycleLR, ReduceLROnPlateau","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Configurations","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"class Config:\n    def __init__(self):\n        self.FOLDS = 2\n        self.EPOCHS = 1\n        self.DEVICE = 'cuda'\n        self.TRAIN_BS = 64\n        self.VALID_BS = 128\n        self.model_type = 'b3'\n        self.loss_fn = nn.L1Loss()\n        \nconfig = Config()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"path = Path('/kaggle/input/osic-pulmonary-fibrosis-progression/')\npath.ls()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"train_df = pd.read_csv(path/'train.csv')\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"These IDS raise gdcm import error. Couldn't figure it out yet. So decided to skip them for now.","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"train_df = train_df.drop(np.nonzero(np.array(train_df['Patient'] == 'ID00011637202177653955184',dtype=float))[0], axis=0).reset_index(drop=True)\ntrain_df = train_df.drop(np.nonzero(np.array(train_df['Patient'] == 'ID00052637202186188008618',dtype=float))[0], axis=0).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preprocessing the meta features","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"def get_tab(df):\n    vector = [(df['Weeks'].values[0] - 30 )/30]\n    \n    if df.Sex.values[0] == 'Male':\n        vector.append(0)\n    else: \n        vector.append(1)\n    \n    if df['SmokingStatus'].values[0] == 'Never smoked':\n        vector.extend([0,0])\n    elif df['SmokingStatus'].values[0] == 'Currently smokes':\n        vector.extend([0,1])\n    elif df['SmokingStatus'].values[0] == 'Ex-smoker':\n        vector.extend([1,0])\n    else :\n        vector.extend([1,1])\n    return np.array(vector)\n ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"TAB = {}\nTARGET = {}\nPerson = []\n\nfor i, p in enumerate(train_df.Patient.unique()):\n    sub = train_df.loc[train_df.Patient == p]\n    fvc = sub.FVC.values\n    weeks = sub.Weeks.values\n    c = np.vstack([weeks, np.ones(len(weeks))]).T\n    a, b = np.linalg.lstsq(c, fvc)[0]\n    \n    TARGET[p] = a\n    TAB[p] = get_tab(sub)\n    Person.append(p)\n\nPerson = np.array(Person)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"def get_img(path):\n    d = pydicom.dcmread(path)\n    return cv2.resize((d.pixel_array - d.RescaleIntercept) / (d.RescaleSlope * 1000), (512,512))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"class Dataset:\n    def __init__(self, path, df, tabular, targets, folder = 'train'):\n        self.df = df\n        self.tabular = tabular\n        self.targets = targets\n        self.folder = folder\n        self.path = path\n        self.transform = transforms.Compose([\n            transforms.ToTensor()\n        ])\n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self,idx):\n        row = self.df.loc[idx,:]\n        pid = row['Patient']\n        # Path to record\n        record = self.path/self.folder/pid\n        # select image id\n        try: \n            \n            img_id =  np.random.choice(len(record.ls()))\n            \n            img = get_img(record.ls()[img_id])\n            img = self.transform(img)\n            tab = torch.from_numpy(self.tabular[pid]).float()\n            target = torch.tensor(self.targets[pid])\n            \n            return (img,tab), target\n        except Exception as e:\n            print(e)\n            print(pid, img_id)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Collate function","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"def collate_fn(b):\n    xs, ys = zip(*b)\n    imgs, tabs = zip(*xs)\n    return (torch.stack(imgs).float(),torch.stack(tabs).float()),torch.stack(ys).float()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I learnt how to use efficientnet from here : https://www.kaggle.com/nroman/melanoma-pytorch-starter-efficientnet","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Model","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self,eff_name='b0'):\n        super().__init__()\n        self.input = nn.Conv2d(1,3,kernel_size=3,padding=1,stride=2)\n        self.bn = nn.BatchNorm2d(3)\n        self.model = EfficientNet.from_pretrained(f'efficientnet-{eff_name}')\n        self.model._fc = nn.Linear(1536, 500, bias=True)\n        self.meta = nn.Sequential(nn.Linear(4, 500),\n                                  nn.BatchNorm1d(500),\n                                  nn.ReLU(),\n                                  nn.Dropout(p=0.2),\n                                  nn.Linear(500,250),\n                                  nn.BatchNorm1d(250),\n                                  nn.ReLU(),\n                                  nn.Dropout(p=0.2))\n        self.output = nn.Linear(500+250, 1)\n        self.relu = nn.ReLU()\n    \n    def forward(self, x,tab):\n        x = self.relu(self.bn(self.input(x)))\n        x = self.model(x)\n        tab = self.meta(tab)\n        x = torch.cat([x, tab],dim=1)\n        return self.output(x)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Generating the splits","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"from sklearn.model_selection import KFold\n\ndef get_split_idxs(n_folds=5):\n    kv = KFold(n_splits=n_folds)\n    splits = []\n    for i,(train_idx, valid_idx) in enumerate(kv.split(Person)):\n        splits.append((train_idx, valid_idx))\n        \n    return splits","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"splits = get_split_idxs(n_folds=config.FOLDS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"def train_loop(model, dl, opt, sched, device, loss_fn):\n    model.train()\n    for X,y in dl:\n        imgs = X[0].to(device)\n        tabs = X[1].to(device)\n        y = y.to(device)\n        outputs = model(imgs, tabs)\n        loss = loss_fn(outputs.squeeze(), y)\n        opt.zero_grad()\n        loss.backward()\n        opt.step()\n        if sched is not None:\n            sched.step()\n            \n\ndef eval_loop(model, dl, device, loss_fn):\n    model.eval()\n    final_outputs = []\n    final_loss = []\n    with torch.no_grad():\n        for X,y in dl:\n            imgs = X[0].to(device)\n            tabs = X[1].to(device)\n            y=y.to(device)\n\n            outputs = model(imgs, tabs)\n            loss = loss_fn(outputs.squeeze(), y)\n\n            final_outputs.extend(outputs.detach().cpu().numpy().tolist())\n            final_loss.append(loss.detach().cpu().numpy())\n        \n    return final_outputs, final_loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This is something I learnt from fast.ai. Check out this notebook if interested: https://github.com/fastai/course-v3/blob/master/nbs/dl2/11a_transfer_learning.ipynb","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"from functools import partial\n\ndef apply_mod(m,f):\n    f(m)\n    for l in m.children(): apply_mod(l,f)\n\ndef set_grad(m,b):\n    if isinstance(m, (nn.Linear, nn.BatchNorm2d)): return \n    if hasattr(m, 'weight'):\n        for p in m.parameters(): p.requires_grad_(b)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models = {}\nfor i in range(config.FOLDS):\n    models[i] = Model(config.model_type)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for k,v in models.items():\n    apply_mod(v.model, partial(set_grad, b=False))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i, (train_idx, valid_idx) in enumerate(splits):\n    print(f\"===================Fold : {i} ================\")\n\n    train = train_df.loc[train_df['Patient'].isin(Person[train_idx])].reset_index(drop=True)\n    valid = train_df.loc[train_df['Patient'].isin(Person[valid_idx])].reset_index(drop=True)\n\n\n    train_ds = Dataset(path, train, TAB, TARGET)\n    train_dl = torch.utils.data.DataLoader(\n        dataset=train_ds,\n        batch_size=config.TRAIN_BS,\n        shuffle=True,\n        collate_fn=collate_fn        \n    )\n\n    valid_ds = Dataset(path, valid, TAB, TARGET)\n    valid_dl = torch.utils.data.DataLoader(\n        dataset=valid_ds,\n        batch_size=config.VALID_BS,\n        shuffle=False,\n        collate_fn=collate_fn\n    )\n\n    model = models[i]\n    model.to(config.DEVICE)\n    lr=1e-3\n    momentum = 0.9\n    \n    num_steps = len(train_dl)\n    optimizer = Adam(model.parameters(), lr=lr,weight_decay=0.1)\n    scheduler = OneCycleLR(optimizer, \n                           max_lr=lr,\n                           epochs=config.EPOCHS,\n                           steps_per_epoch=num_steps\n                           )\n    sched = ReduceLROnPlateau(optimizer,\n                              verbose=True,\n                              factor=0.1)\n    losses = []\n    for epoch in range(config.EPOCHS):\n        print(f\"=================EPOCHS {epoch+1}================\")\n        train_loop(model, train_dl, optimizer, scheduler, config.DEVICE,config.loss_fn)\n        metrics = eval_loop(model, valid_dl,config.DEVICE,config.loss_fn)\n        total_loss = np.array(metrics[1]).mean()\n        losses.append(total_loss)\n        print(\"Loss ::\\t\", total_loss)\n        sched.step(total_loss)\n        \n    model.to('cpu')\n    history.append(losses)\n    \n    \n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for k, m in models.items():\n    torch.save(m.state_dict(), f'fold_{k}.pth')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}