{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!cp /kaggle/input/gdcm-conda-install/gdcm.tar .\n!tar -xvzf gdcm.tar\n!conda install --offline ./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2\n!rm -rf ./gdcm.tar\n# !git clone https://github.com/lessw2020/Ranger-Deep-Learning-Optimizer.git\nimport sys\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\n# sys.path.append('./Ranger-Deep-Learning-Optimizer/ranger')","metadata":{"execution":{"iopub.execute_input":"2021-07-03T19:24:48.434162Z","iopub.status.busy":"2021-07-03T19:24:48.433624Z","iopub.status.idle":"2021-07-03T19:25:17.338667Z","shell.execute_reply":"2021-07-03T19:25:17.339197Z","shell.execute_reply.started":"2021-07-03T18:28:32.019539Z"},"papermill":{"duration":28.933094,"end_time":"2021-07-03T19:25:17.339490","exception":false,"start_time":"2021-07-03T19:24:48.406396","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nfrom tqdm.autonotebook import tqdm\nfrom pprint import pprint\nfrom datetime import datetime\nimport os, sys, cv2, glob, random ,ast, warnings, time\nwarnings.filterwarnings('ignore')\n\nimport timm\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision.transforms as T\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torch.optim.lr_scheduler import CosineAnnealingLR, CosineAnnealingWarmRestarts\nfrom torch.optim import Adam, SGD, AdamW\n\n# import pytorch_lightning as pl\n# import torchmetrics\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\n# from ranger import Ranger  # this is from ranger.py\n# from ranger913A import RangerVA  # this is from ranger913A.py\n# from rangerqh import RangerQH  # this is from rangerqh.py\n\nfrom sklearn.metrics import roc_auc_score, accuracy_score\nfrom sklearn.model_selection import StratifiedKFold\nsys.path.append(\"../input/iterative-stratification/iterative-stratification-master\")\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold","metadata":{"execution":{"iopub.execute_input":"2021-07-03T19:25:26.134337Z","iopub.status.busy":"2021-07-03T19:25:26.133598Z","iopub.status.idle":"2021-07-03T19:25:31.516559Z","shell.execute_reply":"2021-07-03T19:25:31.516009Z","shell.execute_reply.started":"2021-07-03T18:29:07.828821Z"},"papermill":{"duration":5.413092,"end_time":"2021-07-03T19:25:31.516689","exception":false,"start_time":"2021-07-03T19:25:26.103597","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_names = timm.list_models()\n# pprint(model_names)","metadata":{"_kg_hide-output":true,"execution":{"iopub.execute_input":"2021-07-03T19:25:31.570904Z","iopub.status.busy":"2021-07-03T19:25:31.570247Z","iopub.status.idle":"2021-07-03T19:25:31.573326Z","shell.execute_reply":"2021-07-03T19:25:31.572837Z","shell.execute_reply.started":"2021-07-03T18:29:13.205308Z"},"papermill":{"duration":0.03229,"end_time":"2021-07-03T19:25:31.573461","exception":false,"start_time":"2021-07-03T19:25:31.541171","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CFG","metadata":{"papermill":{"duration":0.022669,"end_time":"2021-07-03T19:25:31.618769","exception":false,"start_time":"2021-07-03T19:25:31.596100","status":"completed"},"tags":[]}},{"cell_type":"code","source":"BATCH_SIZE = 16\nVAL_BATCH_SIZE = 32\nEPOCHS = 5 \nIMG_SIZE = 512\nif BATCH_SIZE == 8:\n    ITER_FREQ = 300\nelse:\n    ITER_FREQ = 90\nNUM_WORKERS = 8\nMEAN = [0.485, 0.456, 0.406]\nSTD = [0.229, 0.224, 0.225]\nSEED = 1111\nN_FOLDS = 5\nSTART_FOLD = 3\n\nMODEL_PATH = None\nMODEL_ARCH = 'tf_efficientnet_b5_ns' # tf_efficientnet_b4_ns, tf_efficientnet_b6_ns, resnext50_32x4d, seresnet152d\n\nLR = 1e-4\nMIN_LR = 1e-6 # CosineAnnealingWarmRestarts\nWEIGHT_DECAY = 1e-6\nMOMENTUM = 0.9\nT_0 = EPOCHS # CosineAnnealingWarmRestarts\nMAX_NORM = 1000\nT_MAX = 5\nITERS_TO_ACCUMULATE = 1\n\nBASE_OPTIMIZER = SGD #for Ranger\nOPTIMIZER = 'Adam' # Ranger, AdamW, AdamP, SGD\n\nSCHEDULER = 'CosineAnnealingWarmRestarts' # ReduceLROnPlateau, CosineAnnealingLR, CosineAnnealingWarmRestarts, OneCycleLR\nSCHEDULER_UPDATE = 'epoch' # batch\n\nTR_CRITERION = 'BCE'\nVL_CRITERION = 'BCE' # CrossEntropyLoss\nDEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# avg_prec = torchmetrics.AveragePrecision(num_classes = 4)\nLABELS = ['Negative for Pneumonia','Typical Appearance', 'Indeterminate Appearance', 'Atypical Appearance']","metadata":{"execution":{"iopub.execute_input":"2021-07-03T19:25:31.713463Z","iopub.status.busy":"2021-07-03T19:25:31.712758Z","iopub.status.idle":"2021-07-03T19:25:31.716354Z","shell.execute_reply":"2021-07-03T19:25:31.715939Z","shell.execute_reply.started":"2021-07-03T18:29:13.217717Z"},"papermill":{"duration":0.07498,"end_time":"2021-07-03T19:25:31.716469","exception":false,"start_time":"2021-07-03T19:25:31.641489","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AverageMeter(object):\n    def __init__(self):\n        self.reset()\n        \n    def reset(self):\n        self.val = 0\n        self.sum = 0\n        self.avg = 0\n        self.count = 0\n        \n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val*n\n        self.count += n\n        self.avg = self.sum / self.count\n        \ndef seed_torch(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    \nseed_torch(SEED)\n\ndef macro_multilabel_auc(label, pred):\n    aucs = []\n    for i in range(len(LABELS)):\n        aucs.append(roc_auc_score(label[:, i], pred[:, i]))\n    print(np.round(aucs, 4))\n    return np.mean(aucs)","metadata":{"execution":{"iopub.execute_input":"2021-07-03T19:25:31.770691Z","iopub.status.busy":"2021-07-03T19:25:31.770050Z","iopub.status.idle":"2021-07-03T19:25:31.775207Z","shell.execute_reply":"2021-07-03T19:25:31.774714Z","shell.execute_reply.started":"2021-07-03T18:29:13.268418Z"},"papermill":{"duration":0.036288,"end_time":"2021-07-03T19:25:31.775320","exception":false,"start_time":"2021-07-03T19:25:31.739032","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{"papermill":{"duration":0.022572,"end_time":"2021-07-03T19:25:31.820522","exception":false,"start_time":"2021-07-03T19:25:31.797950","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def preprocess_df(train = True):\n    if train:\n        df_image = pd.read_csv(\"../input/siim-covid19-detection/train_study_level.csv\")\n        df_det = pd.read_csv(\"../input/siim-covid19-detection/train_image_level.csv\")\n        df_image['StudyInstanceUID'] = df_image['id'].apply(lambda x : x[:-6])\n        df = df_det.merge(df_image, on='StudyInstanceUID')\n        path = []\n        TRAIN_DIR = \"../input/siim-covid19-detection/train/\"\n        for instance_id in tqdm(df['StudyInstanceUID']):\n            path.append(glob.glob(os.path.join(TRAIN_DIR, instance_id +\"/*/*\"))[0])\n        df['path'] = path\n\n        df = df.drop(['id_x', 'id_y'], axis=1)\n        return df\n    \n    else:\n        df= pd.read_csv(\"../input/siim-covid19-detection/sample_submission.csv\")\n        study_indices = df['id'].apply(lambda x : x[-6:])\n        STUDY_LEN =0\n        for i in range(len(study_indices)):\n            if study_indices[i] == '_image':\n                STUDY_LEN = i\n                break\n            \n        df['StudyInstanceUID'] = df['id'].apply(lambda x : x[:-6])\n        df = df.iloc[:STUDY_LEN,:]\n        path = []\n        TEST_DIR = \"../input/siim-covid19-detection/test\"\n        for instance_id in tqdm(df['StudyInstanceUID']):\n            path.append(glob.glob(os.path.join(TEST_DIR, instance_id +\"/*/*\"))[0])\n        df['path'] = path\n\n        return df,STUDY_LEN","metadata":{"execution":{"iopub.execute_input":"2021-07-03T19:25:31.875175Z","iopub.status.busy":"2021-07-03T19:25:31.874424Z","iopub.status.idle":"2021-07-03T19:25:31.877154Z","shell.execute_reply":"2021-07-03T19:25:31.876713Z","shell.execute_reply.started":"2021-07-03T18:29:13.286996Z"},"papermill":{"duration":0.03407,"end_time":"2021-07-03T19:25:31.877266","exception":false,"start_time":"2021-07-03T19:25:31.843196","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def dicom2array(path, voi_lut=True, fix_monochrome=True):\n    dicom = pydicom.read_file(path)\n    # VOI LUT (if available by DICOM device) is used to\n    # transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return data","metadata":{"execution":{"iopub.execute_input":"2021-07-03T19:25:31.928368Z","iopub.status.busy":"2021-07-03T19:25:31.927677Z","iopub.status.idle":"2021-07-03T19:25:31.930487Z","shell.execute_reply":"2021-07-03T19:25:31.930038Z","shell.execute_reply.started":"2021-07-03T18:29:13.402293Z"},"papermill":{"duration":0.030803,"end_time":"2021-07-03T19:25:31.930598","exception":false,"start_time":"2021-07-03T19:25:31.899795","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Stratified KFold\ndef stratifiedKFold(df,num_folds,random_state):\n    y = df [['Negative for Pneumonia','Typical Appearance', \n            'Indeterminate Appearance', 'Atypical Appearance']]\n    df['fold'] = 0\n    #split data\n    mskf = MultilabelStratifiedKFold(n_splits=num_folds, shuffle= True, random_state=random_state)\n    for i, (_, test_index) in enumerate(mskf.split(df, y)):\n        df.iloc[test_index, -1] = i\n    return df\n\ndf = preprocess_df(train = True)","metadata":{"execution":{"iopub.execute_input":"2021-07-03T19:25:31.982887Z","iopub.status.busy":"2021-07-03T19:25:31.982366Z","iopub.status.idle":"2021-07-03T19:25:52.810182Z","shell.execute_reply":"2021-07-03T19:25:52.810563Z","shell.execute_reply.started":"2021-07-03T18:29:13.414114Z"},"papermill":{"duration":20.857525,"end_time":"2021-07-03T19:25:52.810740","exception":false,"start_time":"2021-07-03T19:25:31.953215","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SIIM_COVID(Dataset):\n    def __init__(self,df,\n                 train = True,\n                 transforms=None,\n                 IMG_SIZE = 256\n                ):\n        self.imageList = df['path'].values\n        self.transform = None\n        if transforms is None:\n            self.transform = A.Compose([\n                                            A.Resize(IMG_SIZE,IMG_SIZE),\n                                            A.Normalize(\n                                                mean=[0.485, 0.456, 0.406],\n                                                std=[0.229, 0.224, 0.225],\n                                            ),\n                                            ToTensorV2()\n                                        ])\n        else:\n            self.transform = transforms\n        self.train  = train    \n        if self.train == True:\n            self.labels = df[LABELS].values\n    \n    def __len__(self):\n        return len(self.imageList)\n    \n    def __getitem__(self,idx):\n        file_path = self.imageList[idx]\n        img = dicom2array(file_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        image = self.transform(image=img)\n        if self.train == True:\n            image = image['image']\n            label = self.labels[idx]\n            return image, label\n        else:\n            return image","metadata":{"execution":{"iopub.execute_input":"2021-07-03T19:25:52.866835Z","iopub.status.busy":"2021-07-03T19:25:52.866319Z","iopub.status.idle":"2021-07-03T19:25:52.870255Z","shell.execute_reply":"2021-07-03T19:25:52.869819Z","shell.execute_reply.started":"2021-07-03T18:29:33.294808Z"},"papermill":{"duration":0.035381,"end_time":"2021-07-03T19:25:52.870376","exception":false,"start_time":"2021-07-03T19:25:52.834995","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_transform(*, train=True):\n    \n    if train:\n        return A.Compose([\n            A.RandomResizedCrop(IMG_SIZE, IMG_SIZE, scale=(0.85, 1.0)),\n            A.HorizontalFlip(p=0.5),\n            A.RandomBrightnessContrast(p=0.2, brightness_limit=(-0.2, 0.2), contrast_limit=(-0.2, 0.2)),\n            A.HueSaturationValue(p=0.2, hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2),\n            A.ShiftScaleRotate(p=0.2, shift_limit=0.0625, scale_limit=0.2, rotate_limit=20),\n            A.Normalize(mean=MEAN, std=STD),\n            ToTensorV2(), # commented now will convert to torch tensors later.\n        ])\n    else:\n        return A.Compose([\n#             A.CenterCrop(IMG_SIZE, IMG_SIZE),\n            A.Resize(IMG_SIZE, IMG_SIZE),\n            A.Normalize(mean=MEAN, std=STD, max_pixel_value=255.0, p=1.0),\n            ToTensorV2(),\n        ])","metadata":{"execution":{"iopub.execute_input":"2021-07-03T19:25:52.978481Z","iopub.status.busy":"2021-07-03T19:25:52.977737Z","iopub.status.idle":"2021-07-03T19:25:52.980406Z","shell.execute_reply":"2021-07-03T19:25:52.979827Z","shell.execute_reply.started":"2021-07-03T18:29:33.407102Z"},"papermill":{"duration":0.03311,"end_time":"2021-07-03T19:25:52.980527","exception":false,"start_time":"2021-07-03T19:25:52.947417","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{"papermill":{"duration":0.023178,"end_time":"2021-07-03T19:25:53.026629","exception":false,"start_time":"2021-07-03T19:25:53.003451","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class EffNet(nn.Module):#EFFNET\n    def __init__(self,MODEL_ARCH, num_classes, pretrained = False):\n        super().__init__()\n\n        self.model = timm.create_model(MODEL_ARCH,pretrained = pretrained )\n        n_features = self.model.classifier.in_features\n        self.model.global_pool = nn.Identity()\n        self.model.classifier = nn.Identity()\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Linear(n_features,num_classes)\n        self.soft = nn.Softmax(dim=1)\n        \n    def forward(self,x): \n        bs = x.size(0) # bs -> batch size\n        features = self.model(x)\n        pooled_features = self.pooling(features).view(bs,-1)\n        output = self.fc(pooled_features)\n        output = self.soft(output)\n\n        return output \n    \nclass ResNet200D(nn.Module):\n    def __init__(self, MODEL_ARCH, num_classes, pretrained=False):\n        super().__init__()\n        \n        self.model = timm.create_model(MODEL_ARCH, pretrained=False)\n        n_features = self.model.fc.in_features\n        self.model.global_pool = nn.Identity()\n        self.model.fc = nn.Identity()\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Linear(n_features, num_classes)\n        self.soft = nn.Softmax(dim=1)\n        \n    def forward(self,x): \n        bs = x.size(0) # bs -> batch size\n        features = self.model(x)\n        pooled_features = self.pooling(features).view(bs,-1)\n        output = self.fc(pooled_features)\n        output = self.soft(output)\n\n        return output ","metadata":{"execution":{"iopub.execute_input":"2021-07-03T19:25:53.087143Z","iopub.status.busy":"2021-07-03T19:25:53.085380Z","iopub.status.idle":"2021-07-03T19:25:53.087705Z","shell.execute_reply":"2021-07-03T19:25:53.088106Z","shell.execute_reply.started":"2021-07-03T18:29:33.420936Z"},"papermill":{"duration":0.037567,"end_time":"2021-07-03T19:25:53.088252","exception":false,"start_time":"2021-07-03T19:25:53.050685","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def GetCriterion(criterion_name, criterion=None):\n\n    if criterion_name == 'FocalLoss':\n        criterion = FocalLoss()\n    elif criterion_name == 'CustomLoss':\n        criterion = CustomLoss(WEIGHTS)\n    elif criterion_name == 'BCE':\n        criterion = nn.BCEWithLogitsLoss()\n    return criterion\n    \n    \ndef GetScheduler(scheduler_name, optimizer, batches=None):\n    if scheduler_name == 'OneCycleLR':\n        return torch.optim.lr_scheduler.OneCycleLR(optimizer,max_lr = 1e-2,epochs=EPOCHS,\n                                                   steps_per_epoch = batches+1,pct_start = 0.1)\n    if scheduler_name == 'CosineAnnealingWarmRestarts':\n        return torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0 = T_0, T_mult=1,\n                                                                    eta_min=MIN_LR, last_epoch=-1)\n    elif scheduler_name == 'CosineAnnealingLR':\n        return torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=T_MAX, eta_min=0, last_epoch=-1)\n    elif scheduler_name == 'ReduceLROnPlateau':\n        return torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.1, patience=1, threshold=0.0001,\n                                                          cooldown=0, min_lr=MIN_LR)\n    \ndef GetOptimizer(optimizer_name,parameters):\n    #['Adam','Ranger']\n    if optimizer_name == 'Adam':\n        return torch.optim.Adam(parameters, lr=LR, weight_decay=WEIGHT_DECAY, amsgrad=False)\n    elif optimizer_name == 'AdamW':\n        return torch.optim.Adam(parameters, lr=LR, weight_decay=WEIGHT_DECAY, amsgrad=False)\n    elif optimizer_name == 'AdamP':\n        return AdamP(parameters, lr=LR, weight_decay=WEIGHT_DECAY)\n    elif optimizer_name == 'Ranger':\n        return Ranger(parameters, lr = LR, alpha = 0.5, k = 6, N_sma_threshhold = 5, \n                      betas = (0.95,0.999), weight_decay=WEIGHT_DECAY)","metadata":{"execution":{"iopub.execute_input":"2021-07-03T19:25:53.198789Z","iopub.status.busy":"2021-07-03T19:25:53.198253Z","iopub.status.idle":"2021-07-03T19:25:53.200942Z","shell.execute_reply":"2021-07-03T19:25:53.201314Z","shell.execute_reply.started":"2021-07-03T18:29:33.446443Z"},"papermill":{"duration":0.035669,"end_time":"2021-07-03T19:25:53.201457","exception":false,"start_time":"2021-07-03T19:25:53.165788","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train and Validation Functions","metadata":{"papermill":{"duration":0.02283,"end_time":"2021-07-03T19:25:53.247084","exception":false,"start_time":"2021-07-03T19:25:53.224254","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def train_fn(model, dataloader, device, epoch, optimizer, criterion, scheduler):\n    \n    losses = AverageMeter()\n    accuracies = AverageMeter()\n    \n    model.train()\n    scaler = GradScaler()\n    start_time = time.time()\n    loader = tqdm(dataloader, total=len(dataloader))\n    for step, (images, labels) in enumerate(loader):\n\n        images = images.to(device).float()\n        labels = labels.to(device).float()\n\n        with autocast():\n            logits = model(images)\n            loss = criterion(logits, labels)\n            losses.update(loss.item(), BATCH_SIZE)\n    \n            scaler.scale(loss).backward()\n            grad_norm = nn.utils.clip_grad_norm_(model.parameters(), max_norm = MAX_NORM)\n\n            if (step+1) % ITERS_TO_ACCUMULATE == 0:\n                scaler.step(optimizer)\n                # Update the scale for next iteration.\n                scaler.update()\n                optimizer.zero_grad()\n        \n        if scheduler is not None and SCHEDULER_UPDATE == 'batch':\n            scheduler.step()\n\n        start_time = time.time()\n        \n        if step % ITER_FREQ == 0:\n            \n            print('Epoch: [{0}][{1}/{2}]\\t'\n                  'Loss: {loss.val:.4f} ({loss.avg:.4f})'.format((epoch+1),\n                                                                  step, len(dataloader),\n                                                                  loss=losses))\n\n        loader.set_description(f'Training Epoch {epoch+1}/{EPOCHS}')\n        loader.set_postfix(loss=losses.avg)\n#         del images, labels\n    if scheduler is not None and SCHEDULER_UPDATE == 'epoch':\n        scheduler.step()\n        \n    return losses.avg","metadata":{"execution":{"iopub.execute_input":"2021-07-03T19:25:53.303543Z","iopub.status.busy":"2021-07-03T19:25:53.302312Z","iopub.status.idle":"2021-07-03T19:25:53.304760Z","shell.execute_reply":"2021-07-03T19:25:53.305144Z","shell.execute_reply.started":"2021-07-03T18:29:33.462489Z"},"papermill":{"duration":0.03514,"end_time":"2021-07-03T19:25:53.305271","exception":false,"start_time":"2021-07-03T19:25:53.270131","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def valid_fn(epoch, model, criterion, val_loader, device, scheduler):\n    \n    model.eval()\n    losses = AverageMeter()\n    accuracies = AverageMeter()\n    PREDS = []\n    TARGETS = []\n    loader = tqdm(val_loader, total=len(val_loader))\n    with torch.no_grad():  # without torch.no_grad() will make the CUDA run OOM.\n        for step, (images, labels) in enumerate(loader):\n\n            images = images.to(device)\n            labels = labels.to(device).float()\n\n            output = model(images)\n            loss = criterion(output, labels)\n            losses.update(loss.item(), BATCH_SIZE)\n            PREDS += [output.sigmoid()]\n            TARGETS += [labels.detach().cpu()]\n            loader.set_description(f'Validating Epoch {epoch+1}/{EPOCHS}')\n            loader.set_postfix(loss=losses.avg)\n#             del images, labels\n    PREDS = torch.cat(PREDS).cpu().numpy()\n    TARGETS = torch.cat(TARGETS).cpu().numpy()\n    roc_auc = macro_multilabel_auc(TARGETS, PREDS)\n    if scheduler is not None:\n        scheduler.step()\n        \n    return losses.avg, roc_auc","metadata":{"execution":{"iopub.execute_input":"2021-07-03T19:25:53.359236Z","iopub.status.busy":"2021-07-03T19:25:53.358649Z","iopub.status.idle":"2021-07-03T19:25:53.362353Z","shell.execute_reply":"2021-07-03T19:25:53.361937Z","shell.execute_reply.started":"2021-07-03T18:29:33.474209Z"},"papermill":{"duration":0.033571,"end_time":"2021-07-03T19:25:53.362464","exception":false,"start_time":"2021-07-03T19:25:53.328893","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Engine and Main","metadata":{"papermill":{"duration":0.022829,"end_time":"2021-07-03T19:25:53.408218","exception":false,"start_time":"2021-07-03T19:25:53.385389","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def engine(device, folds, fold, model_path=None):\n\n    train_data = SIIM_COVID(df[df['fold'] != fold] , train = True,\n                                        transforms = get_transform(), IMG_SIZE = IMG_SIZE)\n    val_data = SIIM_COVID(df[df['fold'] == fold] , train = True,# To pass both image and targets \n                                        transforms = get_transform(train=False), IMG_SIZE = IMG_SIZE)\n\n    train_loader = DataLoader(train_data,\n                              batch_size=BATCH_SIZE, \n                              shuffle=False, \n                              num_workers=NUM_WORKERS,\n                              pin_memory=True, # enables faster data transfer to CUDA-enabled GPUs.\n                              drop_last=True)\n    val_loader = DataLoader(val_data, \n                            batch_size=VAL_BATCH_SIZE,\n                            num_workers=NUM_WORKERS,\n                            shuffle=False, \n                            pin_memory=True,\n                            drop_last=False)\n    \n    if model_path is not None:\n        model = EffNet(MODEL_ARCH, 4, pretrained=False)\n        model.load_state_dict(torch.load(model_path))\n#         START_EPOCH = int(model_path.split('_')[-2])\n    else:\n        model = EffNet(MODEL_ARCH, 4, pretrained=True)\n\n        START_EPOCH = 0\n    model.to(device)\n    \n    params = filter(lambda p: p.requires_grad, model.parameters())    \n    optimizer = GetOptimizer(OPTIMIZER, params)\n    \n    train_criterion = GetCriterion(TR_CRITERION).to(device)    \n    val_criterion = GetCriterion(VL_CRITERION).to(device)\n    \n    scheduler = GetScheduler(SCHEDULER, optimizer)\n    \n    loss = []\n    for epoch in range(START_EPOCH, EPOCHS):\n        \n        epoch_start = time.time()        \n        avg_loss = train_fn(model, train_loader, device, epoch, optimizer, train_criterion, scheduler)\n\n        torch.cuda.empty_cache()\n        avg_val_loss, roc_auc_score  = valid_fn(epoch, model, val_criterion, val_loader, device, scheduler)\n        epoch_end = time.time() - epoch_start\n        \n        print(f'Training Loss after epoch {epoch+1}: {avg_loss:.4f}')\n        print(f'Validation Loss after epoch {epoch+1}: {avg_val_loss:.4f}')\n        print(f'Validation ROC AUC Score after epoch {epoch+1}: {roc_auc_score:.4f}')\n        print(f'Epoch {epoch+1} finished in {epoch_end:.0f}s')\n        loss.append(avg_loss)\n        \n        content = f'Fold {fold} Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f}  avg_val_loss: {avg_val_loss:.4f} roc_auc_score: {roc_auc_score :.4f} time: {epoch_end:.0f}s'\n        with open(f'{MODEL_ARCH}_{OPTIMIZER}_{IMG_SIZE}.txt', 'a') as appender:\n            appender.write(content + '\\n')                                         \n\n        PATH = f'{MODEL_ARCH}_fold_{fold}_epoch_{(epoch+1)}_{round(roc_auc_score,4)*100}.pt'\n        torch.save({\n            'epoch': epoch,\n            'model_state_dict': model.state_dict(),\n            'optimizer_state_dict': optimizer.state_dict(),\n            'loss': avg_val_loss,\n            }, PATH)\n        torch.cuda.empty_cache()\n    \n    return loss","metadata":{"execution":{"iopub.execute_input":"2021-07-03T19:25:53.467102Z","iopub.status.busy":"2021-07-03T19:25:53.466352Z","iopub.status.idle":"2021-07-03T19:25:53.468993Z","shell.execute_reply":"2021-07-03T19:25:53.468611Z","shell.execute_reply.started":"2021-07-03T18:29:33.486641Z"},"papermill":{"duration":0.037945,"end_time":"2021-07-03T19:25:53.469112","exception":false,"start_time":"2021-07-03T19:25:53.431167","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == '__main__':\n\n    df = stratifiedKFold(df = df,num_folds = N_FOLDS,random_state = 42)\n    for fold in range(START_FOLD, N_FOLDS):\n        if fold == 4:\n            break\n        print(f'===== Fold {fold} Starting =====')\n        fold_start = time.time()\n        logs = engine(DEVICE, df, fold, MODEL_PATH)\n        print(f'Time taken in fold {fold}: {time.time()-fold_start}')","metadata":{"execution":{"iopub.execute_input":"2021-07-03T19:25:53.523246Z","iopub.status.busy":"2021-07-03T19:25:53.522526Z","iopub.status.idle":"2021-07-03T23:44:28.615485Z","shell.execute_reply":"2021-07-03T23:44:28.614854Z","shell.execute_reply.started":"2021-07-03T18:29:33.503451Z"},"papermill":{"duration":15515.1235,"end_time":"2021-07-03T23:44:28.615652","exception":false,"start_time":"2021-07-03T19:25:53.492152","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.042326,"end_time":"2021-07-03T23:44:28.701661","exception":false,"start_time":"2021-07-03T23:44:28.659335","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.042503,"end_time":"2021-07-03T23:44:28.786686","exception":false,"start_time":"2021-07-03T23:44:28.744183","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}