{"cells":[{"metadata":{"id":"gzpcedQ5YIiI"},"cell_type":"markdown","source":"# GPU Info"},{"metadata":{"id":"6D86QjIAwxlU","executionInfo":{"status":"ok","timestamp":1609106966764,"user_tz":-480,"elapsed":1014,"user":{"displayName":"Hongnan G","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgFQzjkNT4z8ejuZFOU4htGRrlSwUNwS1LxakuTCA=s64","userId":"08372665647020343177"}},"outputId":"32523d68-1f10-4d6f-b8f2-3146d8040b37","trusted":true},"cell_type":"code","source":"gpu_info = !nvidia-smi\ngpu_info = '\\n'.join(gpu_info)\nif gpu_info.find('failed') >= 0:\n    print('Select the Runtime > \"Change runtime type\" menu to enable a GPU accelerator, ')\n    print('and then re-execute this cell.')\nelse:\n    print(gpu_info)","execution_count":null,"outputs":[]},{"metadata":{"id":"D7XJ2WQwYMAS"},"cell_type":"markdown","source":"# Dependencies and Imports"},{"metadata":{"_kg_hide-output":true,"execution":{"iopub.execute_input":"2020-12-05T07:02:44.448448Z","iopub.status.busy":"2020-12-05T07:02:44.436331Z","iopub.status.idle":"2020-12-05T07:02:53.353413Z","shell.execute_reply":"2020-12-05T07:02:53.352745Z"},"papermill":{"duration":8.941911,"end_time":"2020-12-05T07:02:53.353539","exception":false,"start_time":"2020-12-05T07:02:44.411628","status":"completed"},"tags":[],"id":"WSEliw3uNoWr","executionInfo":{"status":"ok","timestamp":1609107130092,"user_tz":-480,"elapsed":13499,"user":{"displayName":"Hongnan G","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgFQzjkNT4z8ejuZFOU4htGRrlSwUNwS1LxakuTCA=s64","userId":"08372665647020343177"}},"trusted":true},"cell_type":"code","source":"!pip install -q yamale==3.0.4\n!pip install -q scikit-learn==0.23.2\n!pip install -q torch==1.7.0\n!pip install -q torchvision==0.8.1\n!pip install -q albumentations==0.5.1\n!pip install -q torchtoolbox==0.1.5\n\nimport datetime\nimport gc\nimport os\nimport random\nimport sys\nimport time\nimport warnings\nfrom abc import ABC, abstractmethod\nfrom collections import Counter\nfrom glob import glob\nfrom typing import *\nfrom typing import List, Optional\nimport albumentations\nimport cv2\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport pytz\nimport seaborn as sns\nimport sklearn\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchtoolbox\nimport torchvision\nimport yamale\nfrom albumentations.pytorch.transforms import ToTensorV2\nfrom sklearn.model_selection import GroupKFold, KFold, StratifiedKFold\nfrom torch.optim import *\nfrom torch.utils.data import DataLoader, Dataset, Subset\nfrom torchtoolbox.transform import Cutout\nfrom tqdm import tqdm\n\n\nfrom IPython.core.interactiveshell import InteractiveShell\nInteractiveShell.ast_node_interactivity = \"all\"\n\nsys.path.append('../input/hongnangeffnet/gen-efficientnet-pytorch-master-hongnan')\n#sys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\nsys.path.append('../input/autoaug')\n\nimport geffnet\nfrom auto_augment import AutoAugment, Cutout\n\n","execution_count":null,"outputs":[]},{"metadata":{"id":"M--aY1MXYOY7"},"cell_type":"markdown","source":"# Config"},{"metadata":{"id":"9GxUJCAwNzbs","executionInfo":{"status":"ok","timestamp":1609107206748,"user_tz":-480,"elapsed":1830,"user":{"displayName":"Hongnan G","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgFQzjkNT4z8ejuZFOU4htGRrlSwUNwS1LxakuTCA=s64","userId":"08372665647020343177"}},"trusted":true},"cell_type":"code","source":"class GlobalConfig:\n    seed = 1992\n    num_classes = 11\n    class_list = [0,1,2,3,4,5,6,7,8,9,10]\n    batch_size = 32\n    n_epochs = 3\n   \n    # unpack the key dict\n    scheduler = 'CosineAnnealingLR'\n    scheduler_params = {'StepLR': {'step_size':2, 'gamma':0.3, 'last_epoch':-1, 'verbose':True},\n                \n                'ReduceLROnPlateau': {'mode':'max', 'factor':0.5, 'patience':0, 'threshold':0.0001,\n                                      'threshold_mode':'rel', 'cooldown':0, 'min_lr':1e-5,\n                                      'eps':1e-08, 'verbose':True},\n                \n                'CosineAnnealingWarmRestarts': {'T_0':10, 'T_mult':1, 'eta_min':1e-6, 'last_epoch':-1,\n                                                'verbose':True},\n                'CosineAnnealingLR':{'T_max':6,'eta_min':1e-6, 'last_epoch':-1}}\n    \n    # do scheduler.step after optimizer.step\n    train_step_scheduler = False  \n    val_step_scheduler = True\n    \n    # optimizer\n    optimizer = 'AdamW'\n    optimizer_params = {'AdamW':{'lr':1e-4, 'betas':(0.9,0.999), 'eps':1e-08,\n                                 'weight_decay':1e-6,'amsgrad':False}, \n                        'Adam':{'lr':1e-4,'betas':(0.9,0.999), 'eps':1e-08,\n                                 'weight_decay':1e-6,'amsgrad':False},}\n\n    # criterion\n    criterion = 'BCEWithLogitsLoss'\n    criterion_val = 'BCEWithLogitsLoss'\n    criterion_params = {'BCEWithLogitsLoss': {'weight':None,'size_average':None,\n                                             'reduce':None, 'reduction':'mean', 'pos_weight': None},\n                        'CrossEntropyLoss': {'weight':None,'size_average':None,\n                                             'ignore_index':-100,'reduce':None,\n                                             'reduction':'mean'},\n                        'LabelSmoothingLoss': {'classes':2, 'smoothing':0.05, 'dim':-1},\n                        'FocalCosineLoss': {'alpha':1, 'gamma':2 , 'xent':0.1}}\n\n    gradient_accumulation_steps=2\n    max_grad_norm=1000\n    group_kfold_split = 'PatientID'\n    image_size = 256\n    resize = 256\n    crop_size = {128:110, 256:200, 512:400}\n    verbose = 1\n    verbose_step = 1\n    num_folds = 5\n    image_col_name = 'StudyInstanceUID'\n    class_col_name = ['ETT - Abnormal', 'ETT - Borderline', 'ETT - Normal',\n                 'NGT - Abnormal', 'NGT - Borderline', 'NGT - Incompletely Imaged', 'NGT - Normal', \n                 'CVC - Abnormal', 'CVC - Borderline', 'CVC - Normal',\n                 'Swan Ganz Catheter Present']\n    \n    paths = {'train_path': '../input/ranzcr-clip-trainset-256x256',\n             'test_path': '../input/siim-isic-melanoma-classification/jpeg/test',\n             'csv_path': '../input/ranzcr-clip-catheter-line-classification/train.csv',\n             'log_path': './log.txt',\n             'save_path': './',\n             'model_weight_path_folder': '../input/efficientnet-weights'}\n\n    effnet = 'tf_efficientnet_b5_ns'\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{"id":"CKrxHK_NPMV8","executionInfo":{"status":"ok","timestamp":1609107206749,"user_tz":-480,"elapsed":1828,"user":{"displayName":"Hongnan G","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgFQzjkNT4z8ejuZFOU4htGRrlSwUNwS1LxakuTCA=s64","userId":"08372665647020343177"}},"trusted":true},"cell_type":"code","source":"config=GlobalConfig","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# img = cv2.imread('../input/ranzcr-clip-catheter-line-classification/train/1.2.826.0.1.3680043.8.498.10000428974990117276582711948006105617.jpg') \n# filename = './testing1.jpg'\n# cv2.imwrite(filename, img) ","execution_count":null,"outputs":[]},{"metadata":{"id":"39CKKi0TYQzP"},"cell_type":"markdown","source":"# Seeding"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-05T07:02:57.102385Z","iopub.status.busy":"2020-12-05T07:02:57.101667Z","iopub.status.idle":"2020-12-05T07:02:57.109863Z","shell.execute_reply":"2020-12-05T07:02:57.109364Z"},"papermill":{"duration":0.041667,"end_time":"2020-12-05T07:02:57.109983","exception":false,"start_time":"2020-12-05T07:02:57.068316","status":"completed"},"tags":[],"id":"iybNW9rpNoWu","executionInfo":{"status":"ok","timestamp":1609107206750,"user_tz":-480,"elapsed":1823,"user":{"displayName":"Hongnan G","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgFQzjkNT4z8ejuZFOU4htGRrlSwUNwS1LxakuTCA=s64","userId":"08372665647020343177"}},"outputId":"d07d443a-79f3-441f-ba0b-e00beca59afd","trusted":true},"cell_type":"code","source":"def seed_all(seed: int = 1930):\n\n    print(\"Using Seed Number {}\".format(seed))\n\n    os.environ[\"PYTHONHASHSEED\"] = str(\n        seed)  # set PYTHONHASHSEED env var at fixed value\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.cuda.manual_seed(seed)  # pytorch (both CPU and CUDA)\n    np.random.seed(seed)  # for numpy pseudo-random generator\n    random.seed(\n        seed)  # set fixed value for python built-in pseudo-random generator\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    torch.backends.cudnn.enabled = False\n\n\ndef seed_worker(_worker_id):\n    worker_seed = torch.initial_seed() % 2**32\n    np.random.seed(worker_seed)\n    random.seed(worker_seed)\n    \nseed_all(seed=config.seed)","execution_count":null,"outputs":[]},{"metadata":{"id":"H_j3XCG9YSL6"},"cell_type":"markdown","source":"# CV Folds"},{"metadata":{"id":"NSMHll1Qr-Qj","executionInfo":{"status":"ok","timestamp":1609107206751,"user_tz":-480,"elapsed":1818,"user":{"displayName":"Hongnan G","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgFQzjkNT4z8ejuZFOU4htGRrlSwUNwS1LxakuTCA=s64","userId":"08372665647020343177"}},"outputId":"60dfe267-48f5-426c-e2ca-a8516961e1bf","trusted":true},"cell_type":"code","source":"def make_folds(train_csv: pd.DataFrame,\n               config,\n               cv_schema=None,\n               use_skf=True,\n               use_gkf=False) -> pd.DataFrame:\n    \"\"\"Split the given dataframe into training folds.\"\"\"\n    # TODO: add options for cv_scheme as it is cumbersome here.\n    if use_skf:\n        df_folds = train_csv.copy()\n        skf = StratifiedKFold(5, shuffle=True, random_state=config.seed)\n\n        for fold, (train_idx, val_idx) in enumerate(\n                skf.split(X=df_folds[config.image_col_name],\n                          y=df_folds[config.class_col_name])):\n            df_folds.loc[val_idx, \"fold\"] = int(fold + 1)\n        df_folds[\"fold\"] = df_folds[\"fold\"].astype(int)\n        print(df_folds.groupby([\"fold\", config.class_col_name]).size())\n\n    elif use_gkf:\n        df_folds = train_csv.copy()\n        gkf = GroupKFold(n_splits=config.num_folds)\n        groups = df_folds[config.group_kfold_split].values\n        for fold, (train_index, val_index) in enumerate(\n                gkf.split(df_folds,\n                          df_folds[config.class_col_name],\n                          groups=df_folds[config.group_kfold_split].values)):\n            df_folds.loc[val_index, \"fold\"] = int(fold+1)\n        df_folds[\"fold\"] = df_folds[\"fold\"].astype(int)\n        # print(df_folds.groupby([\"fold\", config.class_col_name]).size())\n\n    return df_folds\n\ntrain_csv = pd.read_csv(config.paths['csv_path']) \ndf_folds = make_folds(train_csv, config,use_skf=False, use_gkf=True)\ndf_folds","execution_count":null,"outputs":[]},{"metadata":{"id":"Vb5X3ROPYUfc"},"cell_type":"markdown","source":"# Utilities"},{"metadata":{"id":"sZhSeNKnZQ_A","executionInfo":{"status":"ok","timestamp":1609107207532,"user_tz":-480,"elapsed":2590,"user":{"displayName":"Hongnan G","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgFQzjkNT4z8ejuZFOU4htGRrlSwUNwS1LxakuTCA=s64","userId":"08372665647020343177"}},"outputId":"1c6f6966-962a-4101-e579-b92b55fe15d1","trusted":true},"cell_type":"code","source":"def get_file_type(image_folder_path: str,\n                  allowed_extensions: Optional[List] = None):\n    \"\"\"Get the file type of images in a folder.\"\"\"\n    if allowed_extensions is None:\n        allowed_extensions = ['.jpg', '.png', '.jpeg']\n\n    file_list = os.listdir(image_folder_path)\n    extension_type = [os.path.splitext(file)[-1].lower() for file in file_list]\n    extension_dict = Counter(extension_type)\n    assert len(extension_dict.keys()\n               ) == 1, \"The extension in the folder should all be the same, \"\n    \"but found {} extensions\".format(extension_dict.keys)\n    extension_type = list(extension_dict.keys())[0]\n    assert extension_type in allowed_extensions\n    return extension_type\n\n\n\ndef check_df_ext(df: pd.DataFrame,\n                 col_name: str,\n                 allowed_extensions: Optional[List] = None):\n    \"\"\"Get the image file extension used in a data frame.\"\"\"\n    if allowed_extensions is None:\n        allowed_extensions = ['.jpg', '.png', '.jpeg']\n    # check if the col has an extension, this is tricky.\n    # if no extension, it gives default \"\"\n    image_id_list = df[col_name].tolist()\n    print(image_id_list)\n    extension_type = [\n        # Review Comments: os.path.splitext is guaranteed to return a 2-tuple,\n        # so no need to use -1 index.\n        os.path.splitext(image_id)[1].lower() for image_id in image_id_list\n    ]\n    \n\n    assert len(set(extension_type)\n               ) == 1, \"The extension in the image id should all be the same\"\n\n\n    if \"\" in extension_type:\n        return False\n\n    assert list(set(extension_type))[0] in allowed_extensions\n    return True","execution_count":null,"outputs":[]},{"metadata":{"id":"7f8ZMOUaYV-6"},"cell_type":"markdown","source":"# Augmentations"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-05T07:02:58.1121Z","iopub.status.busy":"2020-12-05T07:02:58.111373Z","iopub.status.idle":"2020-12-05T07:02:58.114967Z","shell.execute_reply":"2020-12-05T07:02:58.114376Z"},"papermill":{"duration":0.040613,"end_time":"2020-12-05T07:02:58.115082","exception":false,"start_time":"2020-12-05T07:02:58.074469","status":"completed"},"tags":[],"id":"HBDGPp45NoWw","executionInfo":{"status":"ok","timestamp":1609107207532,"user_tz":-480,"elapsed":2585,"user":{"displayName":"Hongnan G","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgFQzjkNT4z8ejuZFOU4htGRrlSwUNwS1LxakuTCA=s64","userId":"08372665647020343177"}},"trusted":true},"cell_type":"code","source":"class Augmentation(ABC):\n\n    @abstractmethod\n    def augment(image):\n        \"\"\"Augment an image.\"\"\"\n        \nclass AlbumentationsAugmentation(Augmentation):\n\n    def __init__(self, transforms: albumentations.core.composition.Compose):\n        self.transforms = transforms\n\n    def augment(self, image):\n        albu_dict = {\"image\": image}\n        transform = self.transforms(**albu_dict)\n        return transform[\"image\"]\n    \nclass TorchTransforms(Augmentation):\n    def __init__(self, transforms: torchvision.transforms.transforms.Compose):\n        self.transforms = transforms\n    \n    def augment(self, image):\n        if isinstance(image, np.ndarray):\n            image = torchvision.transforms.ToPILImage()(image)\n        transformed_image = self.transforms(image)\n        return transformed_image\n    \nclass TorchToolBoxTransforms(Augmentation):\n    def __init__(self, transforms: torchtoolbox.transform.transforms.Compose):\n        self.transforms = transforms\n    \n    def augment(self, image):\n        transformed_image = self.transforms(image)\n        return transformed_image","execution_count":null,"outputs":[]},{"metadata":{"id":"JD2KOcZYuae1","executionInfo":{"status":"ok","timestamp":1609107207533,"user_tz":-480,"elapsed":2583,"user":{"displayName":"Hongnan G","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgFQzjkNT4z8ejuZFOU4htGRrlSwUNwS1LxakuTCA=s64","userId":"08372665647020343177"}},"trusted":true},"cell_type":"code","source":"class augment_config:\n    train_augmentations =  [albumentations.RandomResizedCrop(height=config.image_size, width=config.image_size),\n                            #albumentations.Transpose(p=0.5),\n                            albumentations.HorizontalFlip(p=0.5),\n                            #albumentations.VerticalFlip(p=0.5),\n                            albumentations.ShiftScaleRotate(p=0.5),\n                            albumentations.OneOf([\n                                                    albumentations.JpegCompression(),\n                                                      albumentations.Downscale(scale_min=0.1, scale_max=0.15),\n                                                      ], p=0.2),\n                            albumentations.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n                            albumentations.RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),                          \n                            albumentations.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n                            albumentations.CoarseDropout(p=0.5),\n                            albumentations.Cutout(p=0.5),\n                            ToTensorV2(p=1.0)]\n\n    val_augmentations = [albumentations.Resize(height=config.image_size, width=config.image_size, p=1.0),\n                         albumentations.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n                         ToTensorV2(p=1.0)]\n\n    test_augmentations = [albumentations.Resize(height=config.image_size, width=config.image_size, p=1.0), ToTensorV2(p=1.0)]","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-05T07:02:58.17906Z","iopub.status.busy":"2020-12-05T07:02:58.178369Z","iopub.status.idle":"2020-12-05T07:02:58.181589Z","shell.execute_reply":"2020-12-05T07:02:58.181121Z"},"papermill":{"duration":0.040213,"end_time":"2020-12-05T07:02:58.181681","exception":false,"start_time":"2020-12-05T07:02:58.141468","status":"completed"},"tags":[],"id":"MSJxblnjNoWw","executionInfo":{"status":"ok","timestamp":1609107207533,"user_tz":-480,"elapsed":2580,"user":{"displayName":"Hongnan G","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgFQzjkNT4z8ejuZFOU4htGRrlSwUNwS1LxakuTCA=s64","userId":"08372665647020343177"}},"trusted":true},"cell_type":"code","source":"def get_albu_transforms(config):\n    transforms_train = albumentations.Compose([*augment_config.train_augmentations],p=1.0)\n    transforms_val = albumentations.Compose([*augment_config.val_augmentations],p=1.0)\n\n    return transforms_train, transforms_val    \n    \ndef get_transforms_torchvision(config):\n    transforms_train = torchvision.transforms.Compose([\n    torchvision.transforms.RandomHorizontalFlip(),\n    # torchvision.transforms.RandomVerticalFlip(),\n    AutoAugment(),\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])\n])\n\n    transforms_val = torchvision.transforms.Compose([\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])\n])\n\n    return transforms_train, transforms_val\n\ndef get_torchtoolbox_transforms(config):\n    transforms_train = torchtoolbox.transform.Compose([\n      #DrawHair(),\n      torchtoolbox.transform.RandomResizedCrop(size=256, scale=(0.8, 1.0)),\n      torchtoolbox.transform.RandomHorizontalFlip(),\n      torchtoolbox.transform.RandomVerticalFlip(),\n      #Microscope(p=0.4),\n      torchtoolbox.transform.ToTensor(),\n      torchtoolbox.transform.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])\n  ])\n    \n    transforms_val = torchtoolbox.transform.Compose([\n        torchtoolbox.transform.ToTensor(),\n        torchtoolbox.transform.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])\n    ])\n\n\n    return transforms_train, transforms_val","execution_count":null,"outputs":[]},{"metadata":{"id":"BPoyI80PYZQH"},"cell_type":"markdown","source":"# Dataset"},{"metadata":{},"cell_type":"markdown","source":"Note here we will use `BCEWithLogitsLoss` because all our targets are one hot encoded to 0,1 (multilabel in a sense). "},{"metadata":{},"cell_type":"markdown","source":"TODO: Check df extension throws error because of image name having dots......"},{"metadata":{"id":"Y7VcfngJZDOc","executionInfo":{"status":"ok","timestamp":1609107207533,"user_tz":-480,"elapsed":2577,"user":{"displayName":"Hongnan G","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgFQzjkNT4z8ejuZFOU4htGRrlSwUNwS1LxakuTCA=s64","userId":"08372665647020343177"}},"trusted":true},"cell_type":"code","source":"import os\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport albumentations\nimport torch\n\nfrom typing import Optional\nfrom tqdm import tqdm\n\n\n\nclass RANZCR(torch.utils.data.Dataset):\n\n    \"\"\"The Melanoma dataset. transforms is now an abstract class\"\"\"\n\n    def __init__(self,\n                 df: pd.DataFrame,\n                 config: type,\n                 transforms: type = None,\n                 test: bool = False,\n                 transform_norm: bool = False, meta_features=None):\n        \"\"\"Construct a RANZCR dataset.\"\"\"\n\n        self.df = df\n        self.config = config\n        self.transforms = transforms\n        self.test = test\n        self.transform_norm = transform_norm\n        self.meta_features = meta_features\n\n        if self.transforms is None:\n            assert self.transform_norm is False\n            print('Transforms is None and Transform Normalization is not '\n                  'initialized!')\n\n        self.image_extension = get_file_type(\n            image_folder_path=config.paths['train_path'], allowed_extensions=None)\n        \n#         self.df_has_ext = check_df_ext(df=self.df, col_name=config.image_col_name)\n\n#         if self.df_has_ext is True:\n#             self.image_extension = \"\"\n            \n    def __len__(self):\n        \"\"\"Get the dataset length.\"\"\"\n        return len(self.df)\n\n    def __getitem__(self, idx: int):\n        \"\"\"Get a row from the dataset.\"\"\"\n\n        image_id = self.df[self.config.image_col_name].values[idx]\n        # simple hack to bypass testset df may not have label as column name and throw error when \n        # iterating through the dataset.\n        label = None\n        label = torch.zeros(1)\n\n        \n        if self.test:\n            image_path = os.path.join(\n                self.config.paths['test_path'], \"{}{}\".format(image_id,\n                                                     self.image_extension))\n        else:\n            label = self.df[self.config.class_col_name].values[idx]\n            label = torch.as_tensor(data=label, dtype=torch.float32, device=None)\n            image_path = os.path.join(\n                self.config.paths['train_path'], \"{}{}\".format(image_id,\n                                                      self.image_extension))\n\n        image = cv2.imread(image_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n        if self.transform_norm is False:\n            image = image.astype(np.float32) / 255.0\n\n        if self.transforms is not None:           \n            image = self.transforms.augment(image)\n        else:\n            image = torch.as_tensor(data=image,\n                                    dtype=torch.float32,\n                                    device=None)\n            \n        if self.meta_features is not None:            \n            meta = np.array(self.df.iloc[idx][self.meta_features].values, dtype=np.float32) \n            return image_id, (image, meta), label\n            \n        return image_id, image, label\n","execution_count":null,"outputs":[]},{"metadata":{"id":"zvWxwXeBYavl"},"cell_type":"markdown","source":"# Model"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-05T07:02:58.244501Z","iopub.status.busy":"2020-12-05T07:02:58.243564Z","iopub.status.idle":"2020-12-05T07:02:58.248405Z","shell.execute_reply":"2020-12-05T07:02:58.247849Z"},"papermill":{"duration":0.039236,"end_time":"2020-12-05T07:02:58.2485","exception":false,"start_time":"2020-12-05T07:02:58.209264","status":"completed"},"tags":[],"id":"MYpbNZXJNoWx","executionInfo":{"status":"ok","timestamp":1609107207534,"user_tz":-480,"elapsed":2575,"user":{"displayName":"Hongnan G","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgFQzjkNT4z8ejuZFOU4htGRrlSwUNwS1LxakuTCA=s64","userId":"08372665647020343177"}},"trusted":true},"cell_type":"code","source":"class CustomEfficientNet(nn.Module):\n    def __init__(self, config: type, pretrained: bool=True):\n        super().__init__()\n        self.config = config\n        self.model = geffnet.create_model(\n            model_weight_path_folder=config.paths['model_weight_path_folder'],\n            model_name=config.effnet,\n            pretrained=pretrained)\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, config.num_classes)\n        \n\n    def forward(self, input_neurons):\n        # TODO: add dropout layers, or the likes.\n        output_predictions = self.model(input_neurons)\n        return output_predictions\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"id":"gY1w5doDYb2n"},"cell_type":"markdown","source":"# Custom Loss Functions"},{"metadata":{"id":"Ee8SuOJm12id"},"cell_type":"markdown","source":"https://towardsdatascience.com/what-is-label-smoothing-108debd7ef06\n\nhttps://www.kaggle.com/c/cassava-leaf-disease-classification/discussion/203103\n\nhttps://www.kaggle.com/c/cassava-leaf-disease-classification/discussion/203271"},{"metadata":{"id":"ZXDMkFgO1bGQ","executionInfo":{"status":"ok","timestamp":1609107207534,"user_tz":-480,"elapsed":2572,"user":{"displayName":"Hongnan G","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgFQzjkNT4z8ejuZFOU4htGRrlSwUNwS1LxakuTCA=s64","userId":"08372665647020343177"}},"trusted":true},"cell_type":"code","source":"# ====================================================\n# Label Smoothing\n# ====================================================\nclass LabelSmoothingLoss(nn.Module): \n    def __init__(self, classes=2, smoothing=0.0, dim=-1): \n        super(LabelSmoothingLoss, self).__init__() \n        self.confidence = 1.0 - smoothing \n        self.smoothing = smoothing \n        self.classes = classes \n        self.dim = dim \n    def forward(self, input, target): \n        pred = input.log_softmax(dim=self.dim) \n        with torch.no_grad():\n            true_dist = torch.zeros_like(pred) \n            true_dist.fill_(self.smoothing / (self.classes - 1)) \n            true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence) \n        return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))","execution_count":null,"outputs":[]},{"metadata":{"id":"Yt6O8VLNZMUc","executionInfo":{"status":"ok","timestamp":1609107207534,"user_tz":-480,"elapsed":2569,"user":{"displayName":"Hongnan G","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgFQzjkNT4z8ejuZFOU4htGRrlSwUNwS1LxakuTCA=s64","userId":"08372665647020343177"}},"trusted":true},"cell_type":"code","source":"class FocalCosineLoss(nn.Module):\n    def __init__(self, alpha=1, gamma=2, xent=.1):\n        super(FocalCosineLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n\n        self.xent = xent\n\n        self.y = torch.Tensor([1]).cuda()\n\n    def forward(self, input, target, reduction=\"mean\"):\n        cosine_loss = F.cosine_embedding_loss(input, F.one_hot(target, num_classes=input.size(-1)), self.y, reduction=reduction)\n\n        cent_loss = F.cross_entropy(F.normalize(input), target, reduce=False)\n        pt = torch.exp(-cent_loss)\n        focal_loss = self.alpha * (1-pt)**self.gamma * cent_loss\n\n        if reduction == \"mean\":\n            focal_loss = torch.mean(focal_loss)\n\n        return cosine_loss + self.xent * focal_loss","execution_count":null,"outputs":[]},{"metadata":{"id":"6hBFO-dpYeDZ"},"cell_type":"markdown","source":"# Meters"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-05T07:02:58.321742Z","iopub.status.busy":"2020-12-05T07:02:58.320994Z","iopub.status.idle":"2020-12-05T07:02:58.323776Z","shell.execute_reply":"2020-12-05T07:02:58.324322Z"},"papermill":{"duration":0.046825,"end_time":"2020-12-05T07:02:58.324432","exception":false,"start_time":"2020-12-05T07:02:58.277607","status":"completed"},"tags":[],"id":"3LwlfZCdNoWx","executionInfo":{"status":"ok","timestamp":1609107207535,"user_tz":-480,"elapsed":2568,"user":{"displayName":"Hongnan G","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgFQzjkNT4z8ejuZFOU4htGRrlSwUNwS1LxakuTCA=s64","userId":"08372665647020343177"}},"trusted":true},"cell_type":"code","source":"class AverageLossMeter:\n    \"\"\"\n    Computes and stores the average and current loss\n    \"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.curr_batch_avg_loss = 0\n        self.avg = 0\n        self.running_total_loss = 0\n        self.count = 0\n\n    def update(self, curr_batch_avg_loss: float, batch_size: str):\n        self.curr_batch_avg_loss = curr_batch_avg_loss\n        self.running_total_loss += curr_batch_avg_loss * batch_size\n        self.count += batch_size\n        self.avg = self.running_total_loss / self.count\n\n\n\n\nclass AccuracyMeter:\n    def __init__(self):        \n        self.reset()\n     \n        \n    def reset(self):\n        self.score = 0\n        self.count = 0\n        self.sum = 0\n\n    def update(self, y_true, y_pred, batch_size=1):\n\n        # so we just need to count total num of images / batch_size\n        #self.count += num_steps\n        self.batch_size = batch_size\n        self.count += self.batch_size\n        # this part here already got an acc score for the 4 images, so no need divide batch size\n        self.score = sklearn.metrics.accuracy_score(y_true, y_pred)\n        total_score = self.score * self.batch_size\n\n        self.sum += total_score\n        \n\n    @property\n    def avg(self):        \n        self.avg_score = self.sum/self.count\n        return self.avg_score","execution_count":null,"outputs":[]},{"metadata":{"id":"PSglObQpYfVK"},"cell_type":"markdown","source":"# Callbacks"},{"metadata":{"id":"JnUxYImKZyOa","executionInfo":{"status":"ok","timestamp":1609107207535,"user_tz":-480,"elapsed":2565,"user":{"displayName":"Hongnan G","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgFQzjkNT4z8ejuZFOU4htGRrlSwUNwS1LxakuTCA=s64","userId":"08372665647020343177"}},"trusted":true},"cell_type":"code","source":"from enum import Enum\nfrom typing import Union\n\n\n\nclass Mode(Enum):\n    MIN = np.inf\n    MAX = -np.inf\n\n\nclass EarlyStopping:\n\n    \"\"\"Class for Early Stopping.\"\"\"\n\n    # Review Comments:\n    #\n    # You may want to investigate using Python's built-in Enum class\n    # instead (see https://docs.python.org/3.6/library/enum.html).\n    mode_dict = {'min': np.inf, 'max': -np.inf}\n\n    def __init__(self,\n                 patience: int = 5,\n                 mode: Mode = Mode.MIN,\n                 min_delta: float = 1e-5):\n        \"\"\"Construct an EarlyStopping instance.\n        Arguments:\n            patience : Number of epochs with no improvement after\n                       which training will be stopped. (Default = 5)\n            mode : One of {\"min\", \"max\"}. In min mode, training will\n                   stop when the quantity monitored has stopped\n                   decreasing.  In \"max\" mode it will stop when the\n                   quantity monitored has stopped increasing.\n            min_delta : Minimum change in the monitored quantity to\n                        qualify as an improvement.\n        \"\"\"\n        self.patience = patience\n        self.mode = mode\n        self.min_delta = min_delta\n        self.stopping_counter = 0\n        self.early_stop = False\n        self.best_score = mode.value\n\n    def improvement(self, curr_epoch_score: Union[float, int],\n                    curr_best_score: Union[float, int]):\n        # bool_flag = False, consider the reset bool_flag = True trick\n        if self.mode == Mode.MIN:\n            return curr_epoch_score <= (curr_best_score - self.min_delta)\n\n        return curr_epoch_score >= (curr_best_score + self.min_delta)\n\n    @property\n    def monitor_op(self):\n        return self.mode.value\n\n    # Review Comments:\n    #\n    # I don't think using __call__ makes sense here. In general, you\n    # should use double-underscore methods like __call__ only if you\n    # really need to.  In this case, I think this would be better\n    # implemented as a should_stop method that returns the updated\n    # value of self.early_stop. There is no need for the class\n    # instance itself to be callable as a function, which is what your\n    # use of __call__ does.\n    def should_stop(self, curr_epoch_score):\n        \"\"\"\n        The actual algorithm of early stopping.\n        Arguments:\n            epoch_score : The value of metric or loss which you montoring for that epoch.\n            mode : The model which is being trained.\n            model_path : The path to save the model.\n            \n            rmb false or true --> true, one is true is enough in boolean logic in or clause.\n        \"\"\"\n        # may not need if self.best_score is None or etc\n\n        if self.improvement(curr_epoch_score=curr_epoch_score,\n                            curr_best_score=self.best_score):\n\n            # update self.best_score\n            self.best_score = curr_epoch_score\n            # self.checkpoint_model(model=model, model_path=model_path)\n\n        else:\n            self.stopping_counter += 1\n            print(\"Early Stopping Counter {} out of {}\".format(\n                self.stopping_counter, self.patience))\n\n        if self.stopping_counter >= self.patience:\n\n            print(\"Early Stopping and since it is early stopping, we will not \"\n                  \"save the model since the metric has not improved for {} \"\n                  \"epochs\".format(self.patience))\n            # set flag to true, and in Trainer class, one this is\n            # true, stop training.LOL\n            self.early_stop = True\n\n        return self.best_score, self.early_stop","execution_count":null,"outputs":[]},{"metadata":{"id":"DXAVmSrQYhI3"},"cell_type":"markdown","source":"# Trainer"},{"metadata":{"id":"JE2UPri4dxKQ","executionInfo":{"status":"ok","timestamp":1609107207535,"user_tz":-480,"elapsed":2562,"user":{"displayName":"Hongnan G","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgFQzjkNT4z8ejuZFOU4htGRrlSwUNwS1LxakuTCA=s64","userId":"08372665647020343177"}},"trusted":true},"cell_type":"code","source":"from torch.cuda.amp import autocast, GradScaler","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# IMPORTANT\n\nNote a very important here is we use Sigmoid to calculate, and this is very different from softmax, where softmax will add up to 1, but applying sigmoid on 11 classes will not give u 1. So becareful, because ur multiclass roc function will screw up if u pass in softmax preds instead of the sigmoid preds."},{"metadata":{"execution":{"iopub.execute_input":"2020-12-05T07:02:58.397597Z","iopub.status.busy":"2020-12-05T07:02:58.391864Z","iopub.status.idle":"2020-12-05T07:02:58.429058Z","shell.execute_reply":"2020-12-05T07:02:58.428539Z"},"papermill":{"duration":0.077904,"end_time":"2020-12-05T07:02:58.429152","exception":false,"start_time":"2020-12-05T07:02:58.351248","status":"completed"},"tags":[],"id":"HJ7mvz9_NoWy","executionInfo":{"status":"ok","timestamp":1609107207537,"user_tz":-480,"elapsed":2561,"user":{"displayName":"Hongnan G","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgFQzjkNT4z8ejuZFOU4htGRrlSwUNwS1LxakuTCA=s64","userId":"08372665647020343177"}},"trusted":true},"cell_type":"code","source":"class Trainer:\n\n    \"\"\"A class to perform model training.\"\"\"\n\n    def __init__(self, model, config, early_stopping=None):\n        \"\"\"Construct a Trainer instance.\"\"\"\n        self.model = model\n        \n        self.config = config\n        self.early_stopping = early_stopping\n        self.epoch = 0\n        self.best_auc = 0\n        self.best_acc = 0\n        self.best_loss = np.inf\n        self.save_path = config.paths['save_path']\n        if not os.path.exists(self.save_path):\n            os.makedirs(self.save_path)\n\n        self.criterion = getattr(torch.nn, config.criterion_val)(**config.criterion_params[config.criterion_val])\n        self.criterion_val = getattr(torch.nn, config.criterion_val)(**config.criterion_params[config.criterion_val])\n        self.optimizer = getattr(torch.optim, config.optimizer)(self.model.parameters(), **config.optimizer_params[config.optimizer])\n        self.scheduler = getattr(torch.optim.lr_scheduler, config.scheduler)(optimizer=self.optimizer, **config.scheduler_params[config.scheduler])\n\n\n        self.val_predictions = None\n        self.monitored_metrics = None\n        self.date = datetime.datetime.now(pytz.timezone(\"Asia/Singapore\")).strftime(\"%Y-%m-%d\")\n  \n\n        self.log(\"Trainer prepared. We are using {} device.\".format(\n            self.config.device))\n\n    def fit(self, train_loader, val_loader, fold: int):\n        \"\"\"Fit the model on the given fold.\"\"\"\n        self.log(\"Training on Fold {} and using {}\".format(fold, config.effnet))\n\n        for _epoch in range(self.config.n_epochs):\n            # Getting the learning rate after each epoch!\n            lr = self.optimizer.param_groups[0][\"lr\"]\n            timestamp = datetime.datetime.now(pytz.timezone(\"Asia/Singapore\")).strftime(\"%Y-%m-%d %H-%M-%S\")\n            # printing the lr and the timestamp after each epoch.\n            self.log(\"\\n{}\\nLR: {}\".format(timestamp, lr))\n\n            # start time of training on the training set\n            train_start_time = time.time()\n\n            # train one epoch on the training set\n            avg_train_loss = self.train_one_epoch(\n                train_loader)\n            # end time of training on the training set\n            train_end_time = time.time()\n\n            # formatting time to make it nicer\n            train_elapsed_time = time.strftime(\n                \"%H:%M:%S\", time.gmtime(train_end_time - train_start_time))\n            self.log(\n                \"[RESULT]: Train. Epoch {} | Avg Train Summary Loss: {:.6f} | \"\n                \"Train Accuracy: {:6f} | Time Elapsed: {}\".format(\n                    self.epoch + 1, avg_train_loss, avg_train_loss,\n                    train_elapsed_time))\n\n            val_start_time = time.time()\n            # note here has val predictions... in actual fact it is\n            # repeated because its same as avg_val_acc_score\n            avg_val_loss, val_predictions, avg_roc_score, multi_class_roc_auc_score, score, scores= \\\n                self.valid_one_epoch(val_loader)\n            # not sure if it is good practice to write it here\n            self.val_predictions = val_predictions\n            val_end_time = time.time()\n            val_elapsed_time = time.strftime(\n                \"%H:%M:%S\", time.gmtime(val_end_time - val_start_time))\n\n            self.log(\"[RESULT]: Validation. Epoch: {} | \"\n                     \"Avg Validation Summary Loss: {:.6f} | \"\n                     \"Validation ROC Kaggle method: {:.6f} | Validation ROC: {:.6f} | Multiclass ROC: {} {} | Time Elapsed: {}\".format(\n                         self.epoch + 1, avg_val_loss, score,\n                         avg_roc_score,multi_class_roc_auc_score,scores,\n                         val_elapsed_time))\n\n            # added this flag right before early stopping to let user\n            # know which metric im monitoring.\n            self.monitored_metrics = avg_roc_score\n\n            if self.early_stopping is not None:\n\n                best_score, early_stop = self.early_stopping.should_stop(\n                    curr_epoch_score=self.monitored_metrics)\n                self.best_loss = best_score\n                self.save(\"{}_best_loss_fold_{}.pt\".format(\n                    self.config.effnet, fold))\n                if early_stop:\n                    break\n\n            else:\n                # note here we use avg_val_loss, not train_val_loss! It is\n                # just right to use val_loss as benchmark\n                if avg_val_loss < self.best_loss:\n                    self.best_loss = avg_val_loss\n\n#             if self.best_acc < avg_val_acc_score:\n#                 self.best_acc = avg_val_acc_score\n\n                \n            if avg_roc_score > self.best_auc:\n                self.best_auc = avg_roc_score\n                self.save(os.path.join(self.save_path, \"{}_{}_best_roc_fold_{}.pt\".format(self.date,\n                        self.config.effnet, fold)))\n\n\n\n            if self.config.val_step_scheduler:\n                if isinstance(self.scheduler, torch.optim.lr_scheduler.ReduceLROnPlateau):\n                    self.scheduler.step(self.monitored_metrics)\n                else:\n                    self.scheduler.step()\n\n            # end of training, epoch + 1 so that self.epoch can be updated.\n            self.epoch += 1\n\n        # this is where we end the epoch training for the current\n        # fold/model, therefore we can call the final \"best weight\n        # saved\" by this exact name that we saved earlier on.\n        curr_fold_best_checkpoint = self.load(\n            os.path.join(\n                self.save_path, \"{}_{}_best_roc_fold_{}.pt\".format(self.date, self.config.effnet, fold)\n            )\n        )\n        # return the checkpoint for further usage.\n        return curr_fold_best_checkpoint\n\n    def train_one_epoch(self, train_loader):\n        \"\"\"Train one epoch of the model.\"\"\"\n        # set to train mode\n        self.model.train()\n\n        # log metrics\n        summary_loss = AverageLossMeter()\n        accuracy_scores = AccuracyMeter()\n\n        # timer\n        start_time = time.time()\n\n        # Scaler https://pytorch.org/docs/stable/notes/amp_examples.html\n        scaler = torch.cuda.amp.GradScaler()\n        # looping through train loader for one epoch, steps is the\n        # number of times to go through each epoch\n        for step, (_image_ids, images, labels) in enumerate(train_loader):\n\n            with torch.cuda.amp.autocast():\n            \n                images = images.to(self.config.device).float()\n                labels = labels.to(self.config.device)\n                batch_size = labels.shape[0]\n                logits = self.model(images)\n                loss = self.criterion(input=logits, target=labels)\n\n            summary_loss.update(loss.item(), batch_size)\n\n            if config.gradient_accumulation_steps > 1:\n                loss = loss / config.gradient_accumulation_steps\n\n            \n            scaler.scale(loss).backward()\n            grad_norm = torch.nn.utils.clip_grad_norm_(self.model.parameters(), config.max_grad_norm)\n            if (step + 1) % config.gradient_accumulation_steps == 0:\n                scaler.step(self.optimizer)\n                scaler.update()\n                self.optimizer.zero_grad()\n                \n            # self.optimizer.zero_grad()\n            # loss.backward()\n\n            # self.optimizer.step()\n\n\n            y_true = labels.cpu().numpy()\n            softmax_preds = torch.nn.Softmax(dim=1)(\n                input=logits).to(\"cpu\").detach().numpy()\n            y_preds = np.argmax(a=softmax_preds, axis=1)\n\n            #accuracy_scores.update(y_true, y_preds, batch_size=batch_size)\n\n            # not too sure yet KIV\n            if self.config.train_step_scheduler:\n                self.scheduler.step()\n\n            # measure elapsed time\n            end_time = time.time()\n\n            if self.config.verbose:\n                if (step % self.config.verbose_step) == 0:\n                    print(\n                        f\"Train Steps {step}/{len(train_loader)}, \"\n                        f\"summary_loss: {summary_loss.avg:.3f}, \"\n                        f\"acc: {summary_loss.avg:.3f} \"\n                        f\"time: {(end_time - start_time):.3f}\",\n                        end=\"\\r\",\n                    )\n\n        return summary_loss.avg\n\n    def valid_one_epoch(self, val_loader):\n        \"\"\"Validate one training epoch.\"\"\"\n        # set to eval mode\n        self.model.eval()\n\n        # log metrics\n        summary_loss = AverageLossMeter()\n        accuracy_scores = AccuracyMeter()\n\n        # timer\n        start_time = time.time()\n        naka=[]\n        val_gt_label_list, val_preds_softmax_list, val_preds_roc_list, val_preds_argmax_list = [], [], [], []\n\n        # Looping through val loader for one epoch, steps is the\n        # number of times to go through each epoch; with\n        # torch.no_grad(): off gradients for torch when validating\n        # because we do not need to store gradients for each logits.\n        with torch.no_grad():\n            for step, (_image_ids, images, labels) in enumerate(val_loader):\n\n\n                images = images.to(self.config.device).float()\n\n                labels = labels.to(self.config.device)\n                batch_size = labels.shape[0]\n\n                logits = self.model(images)\n                loss = self.criterion_val(input=logits, target=labels)\n                summary_loss.update(loss.item(), batch_size)\n\n\n                y_true = labels.cpu().numpy()\n                softmax_preds = torch.nn.Softmax(dim=1)(input=logits).to(\"cpu\").numpy()\n                positive_class_preds = softmax_preds[:,1]\n                y_preds = np.argmax(a=softmax_preds, axis=1)\n                #accuracy_scores.update(y_true, y_preds, batch_size=batch_size)\n\n                # here is the same result u get from using sigmoid instead of softmax\n                val_preds_roc_list.append(positive_class_preds)\n                \n                val_gt_label_list.append(y_true)\n                val_preds_softmax_list.append(softmax_preds)\n                val_preds_argmax_list.append(y_preds)\n                \n                naka.append(logits.sigmoid().to('cpu').numpy())\n                               \n                \n                end_time = time.time()\n\n                if config.verbose:\n                    if (step % config.verbose_step) == 0:\n                        print(\n                            f\"Validation Steps {step}/{len(val_loader)}, \" +\n                            f\"summary_loss: {summary_loss.avg:.3f}, val_acc: {summary_loss.avg:.6f} \"\n                            + f\"time: {(end_time - start_time):.3f}\",\n                            end=\"\\r\",\n                        )\n            \n            val_gt_label_array  = np.concatenate(val_gt_label_list, axis=0)\n            val_preds_softmax_array = np.concatenate(val_preds_softmax_list, axis=0)\n            val_preds_argmax_array = np.concatenate(val_preds_argmax_list,axis=0)\n            val_preds_roc_array = np.concatenate(val_preds_roc_list, axis=0)\n            naka_array = np.concatenate(naka, axis = 0)\n            multi_class_roc_auc_score, avg_roc_score = multiclass_roc(y_true=val_gt_label_array,\n                                                                      y_preds_softmax_array=naka_array,\n                                                                      config=self.config)\n            #print(naka_array)\n            #print(val_preds_softmax_array)\n            score, scores = get_score(val_gt_label_array, naka_array)\n            \n            if self.config.num_classes > 2:                \n                val_roc_auc_score =  sklearn.metrics.roc_auc_score(y_true=val_gt_label_array,\n                                                                   y_score=val_preds_softmax_array,\n                                                                   multi_class='ovr')\n            else:\n                val_roc_auc_score =  sklearn.metrics.roc_auc_score(y_true=val_gt_label_array,\n                                                                   y_score=val_preds_roc_array)\n\n\n        return summary_loss.avg, val_preds_softmax_array, avg_roc_score, multi_class_roc_auc_score, score, scores\n\n    def save_model(self, path):\n        \"\"\"Save the trained model.\"\"\"\n        self.model.eval()\n        torch.save(self.model.state_dict(), path)\n\n    # will save the weight for the best val loss and corresponding oof preds\n    def save(self, path):\n        \"\"\"Save the weight for the best evaluation loss.\"\"\"\n        self.model.eval()\n        torch.save(\n            {\n                \"model_state_dict\": self.model.state_dict(),\n                \"optimizer_state_dict\": self.optimizer.state_dict(),\n                \"scheduler_state_dict\": self.scheduler.state_dict(),\n                \"best_acc\": self.best_acc,\n                \"best_auc\": self.best_auc,\n                \"best_loss\": self.best_loss,\n                \"epoch\": self.epoch,\n                \"oof_preds\": self.val_predictions,\n            },\n            path,\n        )\n\n    def load(self, path):\n        \"\"\"Load a model checkpoint from the given path.\"\"\"\n        checkpoint = torch.load(path)\n        return checkpoint\n\n\n    def log(self, message):\n        \"\"\"Log a message.\"\"\"\n        if self.config.verbose:\n            print(message)\n        with open(self.config.paths['log_path'], \"a+\") as logger:\n            logger.write(f\"{message}\\n\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_score(y_true, y_pred):\n    scores = []\n    for i in range(y_true.shape[1]):\n        #print(y_true.shape)\n        score = sklearn.metrics.roc_auc_score(y_true[:,i], y_pred[:,i])\n        scores.append(score)\n    avg_score = np.mean(scores)\n    return avg_score, scores","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-05T07:02:58.55649Z","iopub.status.busy":"2020-12-05T07:02:58.551146Z","iopub.status.idle":"2020-12-05T07:02:58.566024Z","shell.execute_reply":"2020-12-05T07:02:58.565543Z"},"papermill":{"duration":0.049689,"end_time":"2020-12-05T07:02:58.566118","exception":false,"start_time":"2020-12-05T07:02:58.516429","status":"completed"},"tags":[],"id":"11pUDPuvNoW0","executionInfo":{"status":"ok","timestamp":1609107207537,"user_tz":-480,"elapsed":2558,"user":{"displayName":"Hongnan G","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgFQzjkNT4z8ejuZFOU4htGRrlSwUNwS1LxakuTCA=s64","userId":"08372665647020343177"}},"trusted":true},"cell_type":"code","source":"def train_on_fold(df_folds: pd.DataFrame, config, fold: int):\n    \"\"\"Train the model on the given fold.\"\"\"\n    model = CustomEfficientNet(config=config, pretrained=True)\n    model.to(config.device)\n\n    transforms_train, transforms_val = get_albu_transforms(config)\n\n    train_df = df_folds[df_folds[\"fold\"] != fold].reset_index(drop=True)\n    val_df = df_folds[df_folds[\"fold\"] == fold].reset_index(drop=True)\n\n    train_set = RANZCR(train_df, config, transforms=AlbumentationsAugmentation(transforms=transforms_train),transform_norm=True, meta_features=None)\n    train_loader = DataLoader(train_set,\n                              batch_size=config.batch_size,\n                              shuffle=True,\n                              num_workers=4,\n                              worker_init_fn=seed_worker, pin_memory=True)\n\n    val_set = RANZCR(val_df, config, transforms=AlbumentationsAugmentation(transforms=transforms_val), transform_norm=True, meta_features=None)\n    val_loader = DataLoader(val_set,batch_size=32,shuffle=False,num_workers=4,worker_init_fn=seed_worker,pin_memory=True)\n\n\n    hns_detector = Trainer(model=model, config=config)\n\n    curr_fold_best_checkpoint = hns_detector.fit(train_loader, val_loader,\n                                                      fold)\n\n    # loading checkpoint for all 10 epochs for this current fold\n\n    val_df[[str(c) for c in range(config.num_classes)]] = curr_fold_best_checkpoint[\"oof_preds\"]\n    val_df[\"preds\"] = curr_fold_best_checkpoint[\"oof_preds\"].argmax(1)\n\n    return val_df\n\n\ndef multiclass_roc(y_true,y_preds_softmax_array,config):\n    label_dict = dict()   \n    fpr = dict()\n    tpr = dict()\n    roc_auc = dict()\n    roc_scores = []\n    for label_num in range(len(config.class_list)):\n        naka_score = sklearn.metrics.roc_auc_score(y_true[:,label_num], y_preds_softmax_array[:,label_num])\n        y_true_multiclass_array = sklearn.preprocessing.label_binarize(y_true, classes=config.class_list)\n        y_true_for_curr_class = y_true_multiclass_array[:,label_num]\n        y_preds_for_curr_class = y_preds_softmax_array[:, label_num]\n\n        fpr[label_num],tpr[label_num],_ = sklearn.metrics.roc_curve(y_true=y_true_for_curr_class,\n                                                       y_score=y_preds_for_curr_class,\n                                                       pos_label=1)\n        roc_auc[label_num] = sklearn.metrics.auc(fpr[label_num], tpr[label_num])\n        roc_scores.append(roc_auc[label_num])\n        if config.num_classes == 2:\n            roc_auc[config.class_list[1]] = 1 - roc_auc[label_num]\n            break\n    avg_roc_score = np.mean(roc_scores)    \n    return roc_auc, avg_roc_score\n    \ndef get_acc_score(config, result_df):\n    \"\"\"Get the accuracy of model predictions.\"\"\"\n    preds = result_df[\"preds\"].values\n    labels = result_df[config.class_col_name].values\n    score = sklearn.metrics.accuracy_score(y_true=labels, y_pred=preds)\n    return score\n\ndef get_roc_score(config, result_df):\n    max_label = str(np.max(result_df[config.class_col_name].values))\n    preds = result_df[max_label].values\n    labels = result_df[config.class_col_name].values\n    score = sklearn.metrics.roc_auc_score(y_true=labels, y_score=preds, multi_class='ovo')\n    return score\n\ndef train_loop(df_folds: pd.DataFrame,config,fold_num: int = None,train_one_fold=False):\n    \"\"\"Perform the training loop on all folds.\"\"\"\n    # here The CV score is the average of the validation fold metric.\n    cv_score_list = []\n    oof_df = pd.DataFrame()\n    if train_one_fold:\n        _oof_df = train_on_fold(df_folds=df_folds,config=config,fold=fold_num)\n        oof_df = pd.concat([oof_df, _oof_df])\n#         curr_fold_best_score = get_roc_score(config, _oof_df)\n#         print(\"Fold {} OOF Score is {}\".format(fold_num,\n#                                                curr_fold_best_score))\n    else:\n        # the below for loop guarantees it starts from 1 for fold.\n        # https://stackoverflow.com/questions/33282444/pythonic-way-to-iterate-through-a-range-starting-at-1\n        for fold in (number+1 for number in range(config.num_folds)):\n            _oof_df = train_on_fold(df_folds=df_folds,config=config, fold=fold)\n            oof_df = pd.concat([oof_df, _oof_df])\n            #curr_fold_best_score = get_roc_score(config, _oof_df)\n            #cv_score_list.append(curr_fold_best_score)\n            #print(\"\\n\\n\\nOOF Score for Fold {}: {}\\n\\n\\n\".format(fold, curr_fold_best_score))\n\n        #print(\"CV score\", np.mean(cv_score_list))\n        #print(\"Variance\", np.var(cv_score_list))\n        #print(\"Five Folds OOF\", get_roc_score(config, oof_df))\n        oof_df.to_csv(\"oof.csv\")\n        #oof_df.to_csv(os.path.join(config.paths['save_path'], 'oof.csv'))\n","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-05T07:02:58.62612Z","iopub.status.busy":"2020-12-05T07:02:58.625404Z","iopub.status.idle":"2020-12-05T07:55:45.555455Z","shell.execute_reply":"2020-12-05T07:55:45.554513Z"},"papermill":{"duration":3166.962232,"end_time":"2020-12-05T07:55:45.555575","exception":false,"start_time":"2020-12-05T07:02:58.593343","status":"completed"},"tags":[],"id":"ZF1PufBDNoW0","executionInfo":{"status":"ok","timestamp":1609142340219,"user_tz":-480,"elapsed":35135231,"user":{"displayName":"Hongnan G","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgFQzjkNT4z8ejuZFOU4htGRrlSwUNwS1LxakuTCA=s64","userId":"08372665647020343177"}},"outputId":"03857c96-c273-445e-c07f-dc442a4b5f78","trusted":true},"cell_type":"code","source":"# train_five_folds = train_loop(df_folds=df_folds, config=config)\n\ntrain_one_fold = train_loop(df_folds=df_folds, config=config, fold_num=1,train_one_fold=True)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}