{"cells":[{"metadata":{},"cell_type":"markdown","source":"Installing efficientnet_pytorch"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install efficientnet_pytorch","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom pathlib import Path\nPath.ls = lambda x: list(x.iterdir())\nimport os\n\nfrom pydicom import dcmread\nimport cv2\n\nfrom tqdm import tqdm\nfrom matplotlib import pyplot as plt\n\nimport torch\nfrom torch import nn\nfrom torch.optim import Adam\nfrom torch.optim.lr_scheduler import OneCycleLR, ReduceLROnPlateau\nfrom torchvision import transforms\n\nfrom efficientnet_pytorch import EfficientNet\nfrom efficientnet_pytorch.utils import MemoryEfficientSwish\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"path = Path('/kaggle/input/osic-pulmonary-fibrosis-progression/')\npath.ls()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(path/'train.csv')\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"err_patients = ['ID00011637202177653955184','ID00052637202186188008618']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df.drop(np.nonzero(np.array(train_df['Patient'] == 'ID00011637202177653955184',dtype=float))[0], axis=0).reset_index(drop=True)\ntrain_df = train_df.drop(np.nonzero(np.array(train_df['Patient'] == 'ID00052637202186188008618',dtype=float))[0], axis=0).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_tab(df):\n    vector = [(df['Age'].values[0] - 30 )/30]\n    \n    if df.Sex.values[0] == 'Male':\n        vector.append(0)\n    else: \n        vector.append(1)\n    \n    if df['SmokingStatus'].values[0] == 'Never smoked':\n        vector.extend([0,0])\n    elif df['SmokingStatus'].values[0] == 'Currently smokes':\n        vector.extend([0,1])\n    elif df['SmokingStatus'].values[0] == 'Ex-smoker':\n        vector.extend([1,0])\n    else :\n        vector.extend([1,1])\n    return np.array(vector)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TAB = {}\nTARGET = {}\nPerson = []\n\nfor i, p in enumerate(train_df.Patient.unique()):\n    sub = train_df.loc[train_df.Patient == p]\n    fvc = sub.FVC.values\n    weeks = sub.Weeks.values\n    c = np.vstack([weeks, np.ones(len(weeks))]).T\n    a, b = np.linalg.lstsq(c, fvc)[0]\n    \n    TARGET[p] = a\n    TAB[p] = get_tab(sub)\n    Person.append(p)\n\nPerson = np.array(Person)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for patient in err_patients:\n    if patient in Person:\n        Person.remove(patient)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_img(path,size = 512):\n    d = dcmread(path)\n    return cv2.resize((d.pixel_array - d.RescaleIntercept) / (d.RescaleSlope * 1000), (size,size))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_files(path,p_ids):\n    path = Path(path)    \n    return _get_files(path,p_ids)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _get_files(p, ds, extensions=['.dcm']):\n    p = Path(p)\n    paths = [p/d for d in ds]\n    files = [Path(file.path) for path in paths for file in os.scandir(path) if file is not None and Path(file.path).suffix in extensions]\n    return files","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(get_files(path/'train',Person.tolist()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Dataset:\n    def __init__(self,path,tabular,targets,p_ids, size=224, folder='train'):\n        self.path = Path(path)\n        self.tabular = tabular\n        self.targets = targets\n        self.folder = folder\n        self.size = size\n        self.transforms = transforms.Compose([\n            transforms.ToTensor()\n        ])\n        self.p_ids = p_ids\n        self.files = get_files(self.path/folder,self.p_ids)\n    \n    def __len__(self):\n        return len(self.files)\n    \n    def __getitem__(self,idx):\n        img_path = self.files[idx]\n        pid = img_path.parent.name\n        img = get_img(img_path,self.size)\n        img = self.transforms(img)\n        tab = torch.from_numpy(self.tabular[pid]).float()\n        target = torch.tensor(self.targets[pid])\n        return (img,tab), target","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def collate_fn(b):\n    xs, ys = zip(*b)\n    imgs, tabs = zip(*xs)\n    return (torch.stack(imgs).float(),torch.stack(tabs).float()),torch.stack(ys).float()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = Dataset(path,TAB,TARGET, Person)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dl = torch.utils.data.DataLoader(data,shuffle=True,batch_size=8,collate_fn=collate_fn)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self,eff_name='b0'):\n        super().__init__()\n        self.input = nn.Conv2d(1,3,kernel_size=3,padding=1,stride=2)\n        self.bn = nn.BatchNorm2d(3)\n        self.model = EfficientNet.from_pretrained(f'efficientnet-{eff_name}')\n        self.model._fc = nn.Linear(1536, 500, bias=True)\n        self.meta = nn.Sequential(nn.Linear(4, 500),\n                                  nn.BatchNorm1d(500),\n                                  nn.ReLU(),\n                                  nn.Dropout(p=0.2),\n                                  nn.Linear(500,250),\n                                  nn.BatchNorm1d(250),\n                                  nn.ReLU(),\n                                  nn.Dropout(p=0.2))\n        self.output = nn.Linear(500+250, 1)\n        self.relu = nn.ReLU()\n    \n    def forward(self, x,tab):\n        x = self.relu(self.bn(self.input(x)))\n        x = self.model(x)\n        tab = self.meta(tab)\n        x = torch.cat([x, tab],dim=1)\n        return self.output(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import KFold\n\ndef get_split_idxs(n_folds=5):\n    kv = KFold(n_splits=n_folds)\n    splits = []\n    for i,(train_idx, valid_idx) in enumerate(kv.split(Person)):\n        splits.append((train_idx, valid_idx))\n        \n    return splits","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_loop(model, dl, opt, sched, device, loss_fn):\n    model.train()\n    for X,y in tqdm(dl,total=len(dl)):\n        imgs = X[0].to(device)\n        tabs = X[1].to(device)\n        y = y.to(device)\n        outputs = model(imgs, tabs)\n        loss = loss_fn(outputs.squeeze(), y)\n        opt.zero_grad()\n        loss.backward()\n        opt.step()\n        if sched is not None:\n            sched.step()\n            \n\ndef eval_loop(model, dl, device, loss_fn):\n    model.eval()\n    final_outputs = []\n    final_loss = []\n    with torch.no_grad():\n        for X,y in tqdm(dl,total=len(dl)):\n            imgs = X[0].to(device)\n            tabs = X[1].to(device)\n            y=y.to(device)\n\n            outputs = model(imgs, tabs)\n            loss = loss_fn(outputs.squeeze(), y)\n\n            final_outputs.extend(outputs.detach().cpu().numpy().tolist())\n            final_loss.append(loss.detach().cpu().numpy())\n        \n    return final_outputs, final_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from functools import partial\n\ndef apply_mod(m,f):\n    f(m)\n    for l in m.children(): apply_mod(l,f)\n\ndef set_grad(m,b):\n    if isinstance(m, (nn.Linear, nn.BatchNorm2d)): return \n    if hasattr(m, 'weight'):\n        for p in m.parameters(): p.requires_grad_(b)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Config:\n    def __init__(self,lr, epochs = 2, model_type='b3',device='cuda',bs=64):\n        self.FOLDS = 5\n        self.EPOCHS = epochs\n        self.DEVICE = device\n        self.TRAIN_BS = bs\n        self.VALID_BS = bs*2\n        self.model_type = model_type\n        self.loss_fn = nn.L1Loss()\n        self.lr = lr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ModelUtils:\n    def __init__(self,config):\n        self.models = {}\n        self.config = config\n        \n        self.init_models()\n        \n    def __getattr__(self,name):\n        return getattr(self.config, name)\n        \n    def init_models(self):\n        for i in range(self.FOLDS):\n            self.models[i] = Model(self.model_type)\n    \n    def freeze(self):\n        for k,v in self.models.items():\n            apply_mod(v.model, partial(set_grad, b=False))\n    \n    def unfreeze(self):\n        for k,v in self.models.items():\n            apply_mod(v.model, partial(set_grad, b=True))\n    \n    def save_model(self,model,fold,config):\n        torch.save(model.state_dict(),f\"eff_{config.model_type}_fold_{fold}.pth\")\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = {}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fit(config, utils, save_weights = True):\n    for i, (train_idx, valid_idx) in enumerate(splits):\n        print(f\"===================Fold : {i} ================\")\n\n#         train = train_df.loc[train_df['Patient'].isin(Person[train_idx])].reset_index(drop=True)\n#         valid = train_df.loc[train_df['Patient'].isin(Person[valid_idx])].reset_index(drop=True)\n        train_ids = Person[train_idx]\n        valid_ids = Person[valid_idx]\n\n        train_ds = Dataset(path, TAB, TARGET,train_ids,size=224)\n        train_dl = torch.utils.data.DataLoader(\n            dataset=train_ds,\n            batch_size=config.TRAIN_BS,\n            shuffle=True,\n            collate_fn=collate_fn,\n            num_workers = 4\n        )\n\n        valid_ds = Dataset(path, TAB, TARGET,valid_ids,size=224)\n        valid_dl = torch.utils.data.DataLoader(\n            dataset=valid_ds,\n            batch_size=config.VALID_BS,\n            shuffle=False,\n            collate_fn=collate_fn,\n            num_workers = 4\n        )\n\n        model = utils.models[i]\n        model.to(config.DEVICE)\n        lr=config.lr\n        momentum = 0.9\n\n        num_steps = len(train_dl)\n        optimizer = Adam(model.parameters(), lr=lr,weight_decay=0.1)\n        scheduler = OneCycleLR(optimizer, \n                               max_lr=lr,\n                               epochs=config.EPOCHS,\n                               steps_per_epoch=num_steps\n                               )\n        sched = ReduceLROnPlateau(optimizer,\n                                  verbose=True,\n                                  factor=0.1,\n                                  patience=3                                \n                                 )\n        losses = []\n        best_loss = 999\n        for epoch in range(config.EPOCHS):\n            print(f\"=================EPOCHS {epoch+1}================\")\n            train_loop(model, train_dl, optimizer, scheduler, config.DEVICE,config.loss_fn)\n            metrics = eval_loop(model, valid_dl,config.DEVICE,config.loss_fn)\n            mean_loss = np.array(metrics[1]).mean()\n            losses.append(mean_loss)\n            print(\"Loss ::\\t\", mean_loss)\n            sched.step(mean_loss)\n            if mean_loss < best_loss:\n                best_loss = mean_loss\n                if save_weights:\n                    print('saving')\n                    utils.save_model(model, i, config)\n\n        model.to('cpu')\n        history[i] = losses\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"config = Config(lr=1e-3,bs=256,)\nmodel_utils = ModelUtils(config)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"splits = get_split_idxs(n_folds=config.FOLDS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_utils.freeze()\nfit(config, model_utils)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_utils.save_model(model_utils.models[0],1,config)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_utils.unfreeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"config.EPOCHS = 5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fit(config, model_utils)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}