{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Version 13: EfficientB3NS\n\nVersion 14: Skipped because I am superstitous and do not like this number LOL\n\nVersion 15: EfficientnetB5NS\n\nVersion 16: Changed albumentations and image size to native resolution. Changed to Normalization mean using [this](https://www.kaggle.com/c/cassava-leaf-disease-classification/discussion/207450).\n\nVersion 17: Fixed error in albumentation pipeline for normalization.\n\nVersion 18-20: Fixing bugs cause I want to implement `LabelSmoothing`.\n\nVersion 21: 7 Jan fixed label smoothing classes from 2 to 5\n\nVersion 22: LIES! training more epochs don't seem good on my pipeline! Maybe my scheduler is bad!!! Let us try something else, maybe `Cosine Annealing Warm Restart`? Changed to 256 image size and return to normal mean std for augs.","metadata":{}},{"cell_type":"markdown","source":"# Foreword","metadata":{}},{"cell_type":"markdown","source":"This is a pipeline on training using PyTorch. If anyone finds any improvement, please comment in the notebook! I have spent a lot of time to construct such a pipeline for image classification tasks, this is because I am a strong believer in code reusability. The purpose is to change a minimal amount of code every time you run a new training. Neat and clean brings one a long way as ML/DL is more than just knowing the theories, but also includes software engineering.\n\nMuch thanks to @shonenkov as my `Trainer Class` is largely modified from his previous works! Also, thanks to the first Quadruple GM @abhishek as his [youtube videos](https://www.youtube.com/user/abhisheksvnit) inspired me to (try my best to) write clean code.\n\nPS: It does not look as neat in notebook format, but I made a GitHub link to store individual classes into respective files, which is a better practice. I will also further attach my detailed explanation of each line when I have the time.\n\n\nDo give a like/upvote if you feel this is useful to you!","metadata":{}},{"cell_type":"markdown","source":"# Check GPU","metadata":{}},{"cell_type":"code","source":"gpu_info = !nvidia-smi\ngpu_info = '\\n'.join(gpu_info)\nif gpu_info.find('failed') >= 0:\n    print('Select the Runtime > \"Change runtime type\" menu to enable a GPU accelerator, ')\n    print('and then re-execute this cell.')\nelse:\n    print(gpu_info)","metadata":{"id":"6D86QjIAwxlU","outputId":"8a243bb8-6c88-4cd8-b0cb-452d260a1262","execution":{"iopub.status.busy":"2021-11-19T17:04:43.131395Z","iopub.execute_input":"2021-11-19T17:04:43.132303Z","iopub.status.idle":"2021-11-19T17:04:43.192730Z","shell.execute_reply.started":"2021-11-19T17:04:43.132244Z","shell.execute_reply":"2021-11-19T17:04:43.191651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dependencies and Imports","metadata":{}},{"cell_type":"code","source":"!pip install -q yamale==3.0.4\n!pip install -q scikit-learn==0.23.2\n!pip install -q torch==1.7.0\n!pip install -q torchvision==0.8.1\n!pip install -q albumentations==0.5.1\n!pip install -q torchtoolbox==0.1.5","metadata":{"_kg_hide-output":true,"papermill":{"duration":8.941911,"end_time":"2020-12-05T07:02:53.353539","exception":false,"start_time":"2020-12-05T07:02:44.411628","status":"completed"},"tags":[],"id":"WSEliw3uNoWr","outputId":"98ec32ac-2aaf-4420-fd6b-9ea4afbc5586","_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-11-19T17:04:43.195391Z","iopub.execute_input":"2021-11-19T17:04:43.195717Z","iopub.status.idle":"2021-11-19T17:05:40.321415Z","shell.execute_reply.started":"2021-11-19T17:04:43.195684Z","shell.execute_reply":"2021-11-19T17:05:40.319968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import datetime\nimport gc\nimport os\nimport random\nimport sys\nimport time\nimport warnings\nfrom abc import ABC, abstractmethod\nfrom collections import Counter\nfrom glob import glob\nfrom typing import *\nfrom typing import List, Optional\nimport albumentations\nimport cv2\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport pytz\nimport seaborn as sns\nimport sklearn\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchtoolbox\nimport torchvision\nimport yamale\nfrom albumentations.pytorch.transforms import ToTensorV2\nfrom sklearn.model_selection import GroupKFold, KFold, StratifiedKFold\nfrom torch.optim import *\nfrom torch.utils.data import DataLoader, Dataset, Subset\nfrom torchtoolbox.transform import Cutout\nfrom tqdm import tqdm\n\n\nfrom IPython.core.interactiveshell import InteractiveShell\nInteractiveShell.ast_node_interactivity = \"all\"\n\nsys.path.append('../input/hongnangeffnet/gen-efficientnet-pytorch-master-hongnan')\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\nsys.path.append('../input/autoaug')\nimport geffnet\nimport timm\nfrom auto_augment import AutoAugment, Cutout","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":1.741971,"end_time":"2020-12-05T07:02:55.172661","exception":false,"start_time":"2020-12-05T07:02:53.43069","status":"completed"},"tags":[],"id":"-vencTieNoWt","_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-11-19T17:05:40.324368Z","iopub.execute_input":"2021-11-19T17:05:40.325101Z","iopub.status.idle":"2021-11-19T17:05:45.672538Z","shell.execute_reply.started":"2021-11-19T17:05:40.325048Z","shell.execute_reply":"2021-11-19T17:05:45.671322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"class GlobalConfig:\n    seed = 1992\n    num_classes = 5\n    batch_size = 32\n    n_epochs = 10\n\n    \n    # unpack the key dict\n    scheduler = 'CosineAnnealingWarmRestarts'\n    scheduler_params = {'StepLR': {'step_size':2, 'gamma':0.3, 'last_epoch':-1, 'verbose':True},\n                \n                'ReduceLROnPlateau': {'mode':'max', 'factor':0.5, 'patience':0, 'threshold':0.0001,\n                                      'threshold_mode':'rel', 'cooldown':0, 'min_lr':1e-6,\n                                      'eps':1e-08, 'verbose':True},\n                \n                'CosineAnnealingWarmRestarts': {'T_0':10, 'T_mult':1, 'eta_min':1e-6, 'last_epoch':-1,\n                                                'verbose':True}}\n    \n    # do scheduler.step after optimizer.step\n    train_step_scheduler = False  \n    val_step_scheduler = True\n    \n    # optimizer\n    optimizer = 'AdamW'\n    optimizer_params = {'AdamW':{'lr':1e-3, 'betas':(0.9,0.999), 'eps':1e-08,\n                                 'weight_decay':0.001,'amsgrad':False},\n                       'Adam':{'lr':1e-4, 'betas':(0.9,0.999), 'eps':1e-08,\n                                 'weight_decay':0.001,'amsgrad':False}}\n\n    # criterion\n    criterion = 'CrossEntropyLoss'\n    criterion_params = {'CrossEntropyLoss': {'weight':None,'size_average':None,\n                                             'ignore_index':-100,'reduce':None,\n                                             'reduction':'mean'},\n                        'LabelSmoothingLoss': {'classes':5, 'smoothing':0.05, 'dim':-1},\n                        'FocalCosineLoss': {'alpha':1, 'gamma':2 , 'xent':0.1}}\n    \n    image_size = 256\n    resize = 256\n    crop_size = {128:110, 256:200, 456:384, 512:400}\n    verbose = 1\n    verbose_step = 1\n    num_folds = 5\n    image_col_name = 'image_id'\n    class_col_name = 'label'\n    paths = {'train_path': '../input/cassava-jpeg-256x256/kaggle/train_images_jpeg',\n             'test_path': '../input/cassava-leaf-disease-classification/test_images/2216849948.jpg',\n             'csv_path': '../input/cassava-leaf-disease-classification/train.csv',\n             'log_path': './log.txt',\n             'save_path': './',\n             'model_weight_path_folder': '../input/efficientnet-weights'}\n\n    effnet = 'tf_efficientnet_b4_ns'\n    model_name = 'resnext50_32x4d'\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    \nconfig=GlobalConfig","metadata":{"id":"9GxUJCAwNzbs","execution":{"iopub.status.busy":"2021-11-19T17:05:45.675572Z","iopub.execute_input":"2021-11-19T17:05:45.676441Z","iopub.status.idle":"2021-11-19T17:05:45.748735Z","shell.execute_reply.started":"2021-11-19T17:05:45.676389Z","shell.execute_reply":"2021-11-19T17:05:45.747352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Seeding","metadata":{}},{"cell_type":"code","source":"def seed_all(seed: int = 1930):\n\n    print(\"Using Seed Number {}\".format(seed))\n\n    os.environ[\"PYTHONHASHSEED\"] = str(\n        seed)  # set PYTHONHASHSEED env var at fixed value\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.cuda.manual_seed(seed)  # pytorch (both CPU and CUDA)\n    np.random.seed(seed)  # for numpy pseudo-random generator\n    random.seed(\n        seed)  # set fixed value for python built-in pseudo-random generator\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    torch.backends.cudnn.enabled = False\n\n\ndef seed_worker(_worker_id):\n    worker_seed = torch.initial_seed() % 2**32\n    np.random.seed(worker_seed)\n    random.seed(worker_seed)\n    \nseed_all(seed=config.seed)","metadata":{"papermill":{"duration":0.041667,"end_time":"2020-12-05T07:02:57.109983","exception":false,"start_time":"2020-12-05T07:02:57.068316","status":"completed"},"tags":[],"id":"iybNW9rpNoWu","outputId":"ed760d16-5503-4724-d380-643d7ba2799d","execution":{"iopub.status.busy":"2021-11-19T17:05:45.755378Z","iopub.execute_input":"2021-11-19T17:05:45.755944Z","iopub.status.idle":"2021-11-19T17:05:45.773121Z","shell.execute_reply.started":"2021-11-19T17:05:45.755902Z","shell.execute_reply":"2021-11-19T17:05:45.771610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utilities","metadata":{}},{"cell_type":"code","source":"def get_file_type(image_folder_path: str,\n                  allowed_extensions: Optional[List] = None):\n    \"\"\"Get the file type of images in a folder.\"\"\"\n    if allowed_extensions is None:\n        allowed_extensions = ['.jpg', '.png', '.jpeg']\n\n    file_list = os.listdir(image_folder_path)\n    extension_type = [os.path.splitext(file)[-1].lower() for file in file_list]\n    extension_dict = Counter(extension_type)\n    assert len(extension_dict.keys()\n               ) == 1, \"The extension in the folder should all be the same, \"\n    \"but found {} extensions\".format(extension_dict.keys)\n    extension_type = list(extension_dict.keys())[0]\n    assert extension_type in allowed_extensions\n    return extension_type\n\n\n''' Consider modifying this function below to check if the dataframe's\nimage id column has extension or not '''\n\n\ndef check_df_ext(df: pd.DataFrame,\n                 col_name: str,\n                 allowed_extensions: Optional[List] = None):\n    \"\"\"Get the image file extension used in a data frame.\"\"\"\n    if allowed_extensions is None:\n        allowed_extensions = ['.jpg', '.png', '.jpeg']\n    # check if the col has an extension, this is tricky.\n    # if no extension, it gives default \"\"\n    image_id_list = df[col_name].tolist()\n    extension_type = [\n        # Review Comments: os.path.splitext is guaranteed to return a 2-tuple,\n        # so no need to use -1 index.\n        os.path.splitext(image_id)[1].lower() for image_id in image_id_list\n    ]\n\n    assert len(set(extension_type)\n               ) == 1, \"The extension in the image id should all be the same\"\n\n\n    if \"\" in extension_type:\n        return False\n\n    # Review Comments: No need to use else after return.\n    assert list(set(extension_type))[0] in allowed_extensions\n    return True","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-11-19T17:05:45.775754Z","iopub.execute_input":"2021-11-19T17:05:45.776253Z","iopub.status.idle":"2021-11-19T17:05:45.797388Z","shell.execute_reply.started":"2021-11-19T17:05:45.776208Z","shell.execute_reply":"2021-11-19T17:05:45.795982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make Folds (Cross Validation)","metadata":{}},{"cell_type":"code","source":"def make_folds(train_csv: pd.DataFrame, config: type, cv_schema=None) -> pd.DataFrame:\n    \"\"\"Split the given dataframe into training folds.\"\"\"\n    #TODO: add options for cv_scheme.\n    df_folds = train_csv.copy()\n    skf = StratifiedKFold(5, shuffle=True, random_state=config.seed)\n\n    for fold, (train_idx, val_idx) in enumerate(skf.split(X=df_folds[config.image_col_name], y=df_folds[config.class_col_name])):\n        df_folds.loc[val_idx, 'fold'] = int(fold+1)\n    df_folds['fold'] = df_folds['fold'].astype(int)\n    print(df_folds.groupby(['fold', config.class_col_name]).size())\n\n    return df_folds\n\ntrain_csv = pd.read_csv(config.paths['csv_path']) \ndf_folds = make_folds(train_csv, config)\ndf_folds","metadata":{"id":"NSMHll1Qr-Qj","outputId":"ebf2f159-9c96-42c2-9cf6-29297597febc","execution":{"iopub.status.busy":"2021-11-19T17:05:45.799378Z","iopub.execute_input":"2021-11-19T17:05:45.800615Z","iopub.status.idle":"2021-11-19T17:05:45.882449Z","shell.execute_reply.started":"2021-11-19T17:05:45.800570Z","shell.execute_reply":"2021-11-19T17:05:45.881090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Augmentations","metadata":{}},{"cell_type":"code","source":"class Augmentation(ABC):\n\n    @abstractmethod\n    def augment(image):\n        \"\"\"Augment an image.\"\"\"\n        \nclass AlbumentationsAugmentation(Augmentation):\n\n    def __init__(self, transforms: albumentations.core.composition.Compose):\n        self.transforms = transforms\n\n    def augment(self, image):\n        albu_dict = {\"image\": image}\n        transform = self.transforms(**albu_dict)\n        return transform[\"image\"]","metadata":{"papermill":{"duration":0.040613,"end_time":"2020-12-05T07:02:58.115082","exception":false,"start_time":"2020-12-05T07:02:58.074469","status":"completed"},"tags":[],"id":"HBDGPp45NoWw","execution":{"iopub.status.busy":"2021-11-19T17:05:45.884488Z","iopub.execute_input":"2021-11-19T17:05:45.885140Z","iopub.status.idle":"2021-11-19T17:05:45.895969Z","shell.execute_reply.started":"2021-11-19T17:05:45.885076Z","shell.execute_reply":"2021-11-19T17:05:45.894524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class augment_config:\n    train_augmentations =  [#albumentations.RandomSizedCrop(min_max_height=(config.crop_size[config.image_size],config.crop_size[config.image_size]),height=config.image_size, width=config.image_size, p=0.5),\n                            albumentations.RandomResizedCrop(height=config.image_size, width=config.image_size),                    \n                            albumentations.RandomRotate90(p=0.5),\n                            albumentations.VerticalFlip(p=0.5),\n                            albumentations.HorizontalFlip(p=0.5),\n                            albumentations.Cutout(p=0.5),\n                            albumentations.Resize(height=config.image_size, width=config.image_size, p=1.0),\n                            albumentations.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n                            ToTensorV2(p=1.0)]\n\n    val_augmentations = [albumentations.Resize(height=config.image_size, width=config.image_size, p=1.0),\n                         albumentations.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n                         ToTensorV2(p=1.0)]\n\n    test_augmentations = [albumentations.Resize(height=config.image_size, width=config.image_size, p=1.0),\n                          ToTensorV2(p=1.0)]\n    \n\ndef get_albu_transforms(config):\n    transforms_train = albumentations.Compose([*augment_config.train_augmentations],p=1.0)\n    transforms_val = albumentations.Compose([*augment_config.val_augmentations],p=1.0)\n\n    return transforms_train, transforms_val    ","metadata":{"execution":{"iopub.status.busy":"2021-11-19T17:05:45.897907Z","iopub.execute_input":"2021-11-19T17:05:45.898526Z","iopub.status.idle":"2021-11-19T17:05:45.914559Z","shell.execute_reply.started":"2021-11-19T17:05:45.898478Z","shell.execute_reply":"2021-11-19T17:05:45.913519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"class Cassava(torch.utils.data.Dataset):\n\n    \"\"\"The Cassava dataset. transforms is now an abstract class\"\"\"\n\n    def __init__(self,\n                 df: pd.DataFrame,\n                 config: type,\n                 transforms: type = None,\n                 test: bool = False,\n                 transform_norm: bool = False, meta_features=None):\n        \"\"\"Construct a Cassava dataset.\"\"\"\n\n        self.df = df\n        self.config = config\n        self.transforms = transforms\n        self.test = test\n        self.transform_norm = transform_norm\n        self.meta_features = meta_features\n\n        if self.transforms is None:\n            assert self.transform_norm is False\n            print('Transforms is None and Transform Normalization is not '\n                  'initialized!')\n\n        self.image_extension = get_file_type(\n            image_folder_path=config.paths['train_path'], allowed_extensions=None)\n        self.df_has_ext = check_df_ext(df=self.df, col_name=config.image_col_name)\n\n        if self.df_has_ext is True:\n            self.image_extension = \"\"\n            \n    def __len__(self):\n        \"\"\"Get the dataset length.\"\"\"\n        return len(self.df)\n\n    def __getitem__(self, idx: int):\n        \"\"\"Get a row from the dataset.\"\"\"\n\n        image_id = self.df[self.config.image_col_name].values[idx]\n        # simple hack to bypass testset df may not have label as column name and throw error when \n        # iterating through the dataset.\n        label = None\n        label = torch.zeros(1)\n\n        \n        if self.test:\n            image_path = os.path.join(\n                self.config.paths['test_path'], \"{}{}\".format(image_id,\n                                                     self.image_extension))\n        else:\n            label = self.df[self.config.class_col_name].values[idx]\n            label = torch.as_tensor(data=label, dtype=torch.int64, device=None)\n            image_path = os.path.join(\n                self.config.paths['train_path'], \"{}{}\".format(image_id,\n                                                      self.image_extension))\n\n        image = cv2.imread(image_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n        if self.transform_norm is False:\n            image = image.astype(np.float32) / 255.0\n\n        if self.transforms is not None:\n            \n            image = self.transforms.augment(image)\n        else:\n            image = torch.as_tensor(data=image,\n                                    dtype=torch.float32,\n                                    device=None)\n            \n        if self.meta_features is not None:            \n            meta = np.array(self.df.iloc[idx][self.meta_features].values, dtype=np.float32) \n            return image_id, (image, meta), label\n            \n        return image_id, image, label\n","metadata":{"id":"Y7VcfngJZDOc","execution":{"iopub.status.busy":"2021-11-19T17:05:45.916485Z","iopub.execute_input":"2021-11-19T17:05:45.917180Z","iopub.status.idle":"2021-11-19T17:05:45.942438Z","shell.execute_reply.started":"2021-11-19T17:05:45.917125Z","shell.execute_reply":"2021-11-19T17:05:45.941244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Creation","metadata":{}},{"cell_type":"markdown","source":"This is a modified version of [roff's geffnet](https://github.com/rwightman/gen-efficientnet-pytorch). I merely added some lines of code so that one can load his own weights for pretrained. Also' if one wants to use [roff's timm module instead](https://github.com/rwightman/pytorch-image-models), then one just need to change the code below ever so slightly.","metadata":{}},{"cell_type":"code","source":"class CustomEfficientNet(nn.Module):\n    def __init__(self, config: type, pretrained: bool=True):\n        super().__init__()\n        self.config = config\n        self.model = geffnet.create_model(\n            model_weight_path_folder=config.paths['model_weight_path_folder'],\n            model_name=config.effnet,\n            pretrained=pretrained)\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, config.num_classes)\n        \n\n    def forward(self, input_neurons):\n        # TODO: add dropout layers, or the likes.\n        output_predictions = self.model(input_neurons)\n        return output_predictions","metadata":{"papermill":{"duration":0.039236,"end_time":"2020-12-05T07:02:58.2485","exception":false,"start_time":"2020-12-05T07:02:58.209264","status":"completed"},"tags":[],"id":"MYpbNZXJNoWx","execution":{"iopub.status.busy":"2021-11-19T17:05:45.946035Z","iopub.execute_input":"2021-11-19T17:05:45.946752Z","iopub.status.idle":"2021-11-19T17:05:45.957636Z","shell.execute_reply.started":"2021-11-19T17:05:45.946703Z","shell.execute_reply":"2021-11-19T17:05:45.956652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Using `timm` module instead of `geffnet` module (both are from the same author).","metadata":{}},{"cell_type":"code","source":"print(\"Available Vision Transformer Models: \")\ntimm.list_models(\"vit*\")","metadata":{"execution":{"iopub.status.busy":"2021-11-19T17:05:45.959570Z","iopub.execute_input":"2021-11-19T17:05:45.960295Z","iopub.status.idle":"2021-11-19T17:05:45.974507Z","shell.execute_reply.started":"2021-11-19T17:05:45.960244Z","shell.execute_reply":"2021-11-19T17:05:45.973082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nclass CustomResNext(nn.Module):\n    def __init__(self,config, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name=config.model_name, pretrained=pretrained)\n        n_features = self.model.fc.in_features\n        self.model.fc = nn.Linear(n_features, config.image_size)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2021-11-19T17:05:45.976557Z","iopub.execute_input":"2021-11-19T17:05:45.977454Z","iopub.status.idle":"2021-11-19T17:05:45.987736Z","shell.execute_reply.started":"2021-11-19T17:05:45.977405Z","shell.execute_reply":"2021-11-19T17:05:45.986351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomEfficientNet_pytorch_image_models(nn.Module):\n    def __init__(self, config: type, pretrained: bool=True):\n        super().__init__()\n        self.config = config\n        self.model = timm.create_model(model_name=config.model_name, pretrained=pretrained)\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, config.num_classes)\n        \n\n    def forward(self, input_neurons):\n        # TODO: add dropout layers, or the likes.\n        output_predictions = self.model(input_neurons)\n        return output_predictions","metadata":{"execution":{"iopub.status.busy":"2021-11-19T17:05:45.990155Z","iopub.execute_input":"2021-11-19T17:05:45.991060Z","iopub.status.idle":"2021-11-19T17:05:46.001967Z","shell.execute_reply.started":"2021-11-19T17:05:45.991011Z","shell.execute_reply":"2021-11-19T17:05:46.000524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Meters","metadata":{}},{"cell_type":"code","source":"class AverageLossMeter:\n    \"\"\"\n    Computes and stores the average and current loss\n    \"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.curr_batch_avg_loss = 0\n        self.avg = 0\n        self.running_total_loss = 0\n        self.count = 0\n\n    def update(self, curr_batch_avg_loss: float, batch_size: str):\n        self.curr_batch_avg_loss = curr_batch_avg_loss\n        self.running_total_loss += curr_batch_avg_loss * batch_size\n        self.count += batch_size\n        self.avg = self.running_total_loss / self.count\n\nclass AccuracyMeter:\n    def __init__(self):        \n        self.reset()\n     \n        \n    def reset(self):\n        self.score = 0\n        self.count = 0\n        self.sum = 0\n\n    def update(self, y_true, y_pred, batch_size=1):\n\n        # so we just need to count total num of images / batch_size\n        #self.count += num_steps\n        self.batch_size = batch_size\n        self.count += self.batch_size\n        # this part here already got an acc score for the 4 images, so no need divide batch size\n        self.score = sklearn.metrics.accuracy_score(y_true, y_pred)\n        total_score = self.score * self.batch_size\n\n        self.sum += total_score\n        \n\n    @property\n    def avg(self):        \n        self.avg_score = self.sum/self.count\n        return self.avg_score","metadata":{"papermill":{"duration":0.046825,"end_time":"2020-12-05T07:02:58.324432","exception":false,"start_time":"2020-12-05T07:02:58.277607","status":"completed"},"tags":[],"id":"3LwlfZCdNoWx","execution":{"iopub.status.busy":"2021-11-19T17:05:46.003957Z","iopub.execute_input":"2021-11-19T17:05:46.004531Z","iopub.status.idle":"2021-11-19T17:05:46.022840Z","shell.execute_reply.started":"2021-11-19T17:05:46.004484Z","shell.execute_reply":"2021-11-19T17:05:46.021491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loss","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# Label Smoothing\n# ====================================================\nclass LabelSmoothingLoss(nn.Module): \n    def __init__(self, classes=2, smoothing=0.0, dim=-1): \n        super(LabelSmoothingLoss, self).__init__() \n        self.confidence = 1.0 - smoothing \n        self.smoothing = smoothing \n        self.classes = classes \n        self.dim = dim \n    def forward(self, input, target): \n        pred = input.log_softmax(dim=self.dim) \n        with torch.no_grad():\n            true_dist = torch.zeros_like(pred) \n            true_dist.fill_(self.smoothing / (self.classes - 1)) \n            true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence) \n        return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))","metadata":{"execution":{"iopub.status.busy":"2021-11-19T17:05:46.024631Z","iopub.execute_input":"2021-11-19T17:05:46.025237Z","iopub.status.idle":"2021-11-19T17:05:46.037643Z","shell.execute_reply.started":"2021-11-19T17:05:46.025191Z","shell.execute_reply":"2021-11-19T17:05:46.036595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Callbacks","metadata":{}},{"cell_type":"code","source":"from enum import Enum\nfrom typing import Union\n\n\n\nclass Mode(Enum):\n    MIN = np.inf\n    MAX = -np.inf\n\n\nclass EarlyStopping:\n\n    \"\"\"Class for Early Stopping.\"\"\"\n\n    mode_dict = {'min': np.inf, 'max': -np.inf}\n\n    def __init__(self,\n                 patience: int = 5,\n                 mode: Mode = Mode.MIN,\n                 min_delta: float = 1e-5):\n        \"\"\"Construct an EarlyStopping instance.\n        Arguments:\n            patience : Number of epochs with no improvement after\n                       which training will be stopped. (Default = 5)\n            mode : One of {\"min\", \"max\"}. In min mode, training will\n                   stop when the quantity monitored has stopped\n                   decreasing.  In \"max\" mode it will stop when the\n                   quantity monitored has stopped increasing.\n            min_delta : Minimum change in the monitored quantity to\n                        qualify as an improvement.\n        \"\"\"\n        self.patience = patience\n        self.mode = mode\n        self.min_delta = min_delta\n        self.stopping_counter = 0\n        self.early_stop = False\n        self.best_score = mode.value\n\n    def improvement(self, curr_epoch_score: Union[float, int],\n                    curr_best_score: Union[float, int]):\n        # bool_flag = False, consider the reset bool_flag = True trick\n        if self.mode == Mode.MIN:\n            return curr_epoch_score <= (curr_best_score - self.min_delta)\n\n        return curr_epoch_score >= (curr_best_score + self.min_delta)\n\n    @property\n    def monitor_op(self):\n        return self.mode.value\n\n\n    def should_stop(self, curr_epoch_score):\n        \"\"\"\n        The actual algorithm of early stopping.\n        Arguments:\n            epoch_score : The value of metric or loss which you montoring for that epoch.\n            mode : The model which is being trained.\n            model_path : The path to save the model.\n            \n            rmb false or true --> true, one is true is enough in boolean logic in or clause.\n        \"\"\"\n        # may not need if self.best_score is None or etc\n\n        if self.improvement(curr_epoch_score=curr_epoch_score,\n                            curr_best_score=self.best_score):\n\n            # update self.best_score\n            self.best_score = curr_epoch_score\n            # self.checkpoint_model(model=model, model_path=model_path)\n\n        else:\n            self.stopping_counter += 1\n            print(\"Early Stopping Counter {} out of {}\".format(\n                self.stopping_counter, self.patience))\n\n        if self.stopping_counter >= self.patience:\n\n            print(\"Early Stopping and since it is early stopping, we will not \"\n                  \"save the model since the metric has not improved for {} \"\n                  \"epochs\".format(self.patience))\n            # set flag to true, and in Trainer class, one this is\n            # true, stop training.LOL\n            self.early_stop = True\n\n        return self.best_score, self.early_stop","metadata":{"id":"JnUxYImKZyOa","execution":{"iopub.status.busy":"2021-11-19T17:05:46.040409Z","iopub.execute_input":"2021-11-19T17:05:46.040844Z","iopub.status.idle":"2021-11-19T17:05:46.061259Z","shell.execute_reply.started":"2021-11-19T17:05:46.040759Z","shell.execute_reply":"2021-11-19T17:05:46.059847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Trainer (Alex Style)","metadata":{}},{"cell_type":"code","source":"class Trainer:\n\n    \"\"\"A class to perform model training.\"\"\"\n\n    def __init__(self, model, config, early_stopping=None):\n        \"\"\"Construct a Trainer instance.\"\"\"\n        self.model = model\n        \n        self.config = config\n        self.early_stopping = early_stopping\n        self.epoch = 0\n        self.best_auc = 0\n        self.best_acc = 0\n        self.best_loss = np.inf\n        # self.save_path = config.save_path\n        # if not os.path.exists(self.save_path):\n        #     os.makedirs(self.save_path)\n        \n        # self.criterion=LabelSmoothingLoss(**config.criterion_params[config.criterion]).to(self.config.device)\n        self.criterion = getattr(torch.nn, config.criterion)(**config.criterion_params[config.criterion])\n        self.optimizer = getattr(torch.optim, config.optimizer)(self.model.parameters(), **config.optimizer_params[config.optimizer])\n        self.scheduler = getattr(torch.optim.lr_scheduler, config.scheduler)(optimizer=self.optimizer, **config.scheduler_params[config.scheduler])\n\n\n        self.val_predictions = None\n        self.monitored_metrics = None\n        self.date = datetime.datetime.now(pytz.timezone(\"Asia/Singapore\")).strftime(\"%Y-%m-%d\")\n\n        self.log(\"Trainer prepared. We are using {} device.\".format(\n            self.config.device))\n\n    def fit(self, train_loader, val_loader, fold: int):\n        \"\"\"Fit the model on the given fold.\"\"\"\n        self.log(\"Training on Fold {} and using {}\".format(fold, config.effnet))\n\n        for _epoch in range(self.config.n_epochs):\n            # Getting the learning rate after each epoch!\n            lr = self.optimizer.param_groups[0][\"lr\"]\n            timestamp = datetime.datetime.now(pytz.timezone(\"Asia/Singapore\")).strftime(\"%Y-%m-%d %H-%M-%S\")\n            # printing the lr and the timestamp after each epoch.\n            self.log(\"\\n{}\\nLR: {}\".format(timestamp, lr))\n\n            # start time of training on the training set\n            train_start_time = time.time()\n\n            # train one epoch on the training set\n            avg_train_loss, avg_train_acc_score = self.train_one_epoch(\n                train_loader)\n            # end time of training on the training set\n            train_end_time = time.time()\n\n            # formatting time to make it nicer\n            train_elapsed_time = time.strftime(\n                \"%H:%M:%S\", time.gmtime(train_end_time - train_start_time))\n            self.log(\n                \"[RESULT]: Train. Epoch {} | Avg Train Summary Loss: {:.6f} | \"\n                \"Train Accuracy: {:6f} | Time Elapsed: {}\".format(\n                    self.epoch + 1, avg_train_loss, avg_train_acc_score,\n                    train_elapsed_time))\n\n            val_start_time = time.time()\n            # note here has val predictions... in actual fact it is\n            # repeated because its same as avg_val_acc_score\n            avg_val_loss, avg_val_acc_score, val_predictions, val_roc_auc = \\\n                self.valid_one_epoch(val_loader)\n            # not sure if it is good practice to write it here\n            self.val_predictions = val_predictions\n            val_end_time = time.time()\n            val_elapsed_time = time.strftime(\n                \"%H:%M:%S\", time.gmtime(val_end_time - val_start_time))\n\n            self.log(\"[RESULT]: Validation. Epoch: {} | \"\n                     \"Avg Validation Summary Loss: {:.6f} | \"\n                     \"Validation Accuracy: {:.6f} | Validation ROC: {:.6f} | Time Elapsed: {}\".format(\n                         self.epoch + 1, avg_val_loss, avg_val_acc_score,\n                         val_roc_auc,\n                         val_elapsed_time))\n\n            # added this flag right before early stopping to let user\n            # know which metric im monitoring.\n            self.monitored_metrics = avg_val_acc_score\n\n            if self.early_stopping is not None:\n\n                best_score, early_stop = self.early_stopping.should_stop(curr_epoch_score=self.monitored_metrics)\n                self.best_loss = best_score\n                self.save(\"{}_best_loss_fold_{}.pt\".format(\n                    self.config.effnet, fold))\n                if early_stop:\n                    break\n\n            else:\n                # note here we use avg_val_loss, not train_val_loss! It is\n                # just right to use val_loss as benchmark\n                if avg_val_loss < self.best_loss:\n                    self.best_loss = avg_val_loss\n\n            if self.best_acc < avg_val_acc_score:\n                self.best_acc = avg_val_acc_score\n   \n                self.save(os.path.join(self.config.paths['save_path'],\n                                       \"{}_{}_best_acc_fold_{}.pt\".format(self.date,\n                                                                          self.config.effnet, fold)))\n                \n            if val_roc_auc > self.best_auc:\n                self.best_auc = val_roc_auc\n\n\n\n            if self.config.val_step_scheduler:\n                if isinstance(self.scheduler, torch.optim.lr_scheduler.ReduceLROnPlateau):\n                    self.scheduler.step(self.monitored_metrics)\n                else:\n                    self.scheduler.step()\n\n            # end of training, epoch + 1 so that self.epoch can be updated.\n            self.epoch += 1\n\n        # this is where we end the epoch training for the current\n        # fold/model, therefore we can call the final \"best weight\n        # saved\" by this exact name that we saved earlier on.\n        curr_fold_best_checkpoint = self.load(\n            os.path.join(\n                self.config.paths[\"save_path\"],\n                \"{}_{}_best_acc_fold_{}.pt\".format(self.date, self.config.effnet, fold)\n            )\n        )\n        # return the checkpoint for further usage.\n        return curr_fold_best_checkpoint\n\n    def train_one_epoch(self, train_loader):\n        \"\"\"Train one epoch of the model.\"\"\"\n        # set to train mode\n        self.model.train()\n\n        # log metrics\n        summary_loss = AverageLossMeter()\n        accuracy_scores = AccuracyMeter()\n\n        # timer\n        start_time = time.time()\n\n        # looping through train loader for one epoch, steps is the\n        # number of times to go through each epoch\n        for step, (_image_ids, images, labels) in enumerate(train_loader):\n\n            \n            images = images.to(self.config.device).float()\n            labels = labels.to(self.config.device)\n            batch_size = labels.shape[0]\n            logits = self.model(images)\n            \n            #print(logits.shape, labels.shape)\n            loss = self.criterion(input=logits, target=labels)\n            summary_loss.update(loss.item(), batch_size)\n            self.optimizer.zero_grad()\n            loss.backward()\n            self.optimizer.step()\n            y_true = labels.cpu().numpy()\n            softmax_preds = torch.nn.Softmax(dim=1)(input=logits).to(\"cpu\").detach().numpy()\n            y_preds = np.argmax(a=softmax_preds, axis=1)\n            \n            accuracy_scores.update(y_true, y_preds, batch_size=batch_size)\n\n            if self.config.train_step_scheduler:\n                self.scheduler.step()\n\n            # measure elapsed time\n            end_time = time.time()\n\n            if self.config.verbose:\n                if (step % self.config.verbose_step) == 0:\n                    print(\n                        f\"Train Steps {step}/{len(train_loader)}, \"\n                        f\"summary_loss: {summary_loss.avg:.3f}, \"\n                        f\"acc: {accuracy_scores.avg:.3f} \"\n                        f\"time: {(end_time - start_time):.3f}\",\n                        end=\"\\r\",\n                    )\n        \n        return summary_loss.avg, accuracy_scores.avg\n\n    def valid_one_epoch(self, val_loader):\n        \"\"\"Validate one training epoch.\"\"\"\n        # set to eval mode\n        self.model.eval()\n\n        # log metrics\n        summary_loss = AverageLossMeter()\n        accuracy_scores = AccuracyMeter()\n\n        # timer\n        start_time = time.time()\n\n        val_gt_label_list, val_preds_softmax_list, val_preds_roc_list, val_preds_argmax_list = [], [], [], []\n\n        with torch.no_grad():\n            for step, (_image_ids, images, labels) in enumerate(val_loader):\n\n                images = images.to(self.config.device).float()\n                labels = labels.to(self.config.device)\n                batch_size = labels.shape[0]\n                logits = self.model(images)\n                loss = self.criterion(input=logits, target=labels)\n                summary_loss.update(loss.item(), batch_size)\n                y_true = labels.cpu().numpy()\n                softmax_preds = torch.nn.Softmax(dim=1)(input=logits).to(\"cpu\").numpy()\n                positive_class_preds = softmax_preds[:,1]\n                y_preds = np.argmax(a=softmax_preds, axis=1)\n                accuracy_scores.update(y_true, y_preds, batch_size=batch_size)\n                val_preds_roc_list.append(positive_class_preds)\n                val_gt_label_list.append(y_true)\n                val_preds_softmax_list.append(softmax_preds)\n                val_preds_argmax_list.append(y_preds)\n                end_time = time.time()\n\n                if config.verbose:\n                    if (step % config.verbose_step) == 0:\n                        print(\n                            f\"Validation Steps {step}/{len(val_loader)}, \" +\n                            f\"summary_loss: {summary_loss.avg:.3f}, val_acc: {accuracy_scores.avg:.6f} \"\n                            + f\"time: {(end_time - start_time):.3f}\",\n                            end=\"\\r\",\n                        )\n            val_gt_label_array  = np.concatenate(val_gt_label_list, axis=0)\n            val_preds_softmax_array = np.concatenate(val_preds_softmax_list, axis=0)\n            val_preds_argmax_array = np.concatenate(val_preds_argmax_list,axis=0)\n            val_preds_roc_array = np.concatenate(val_preds_roc_list, axis=0)\n            print(val_gt_label_array)\n            print(val_preds_argmax_array)\n            if self.config.num_classes > 2:                \n                val_roc_auc_score =  sklearn.metrics.roc_auc_score(y_true=val_gt_label_array, y_score=val_preds_softmax_array,multi_class='ovr')\n            else:\n                val_roc_auc_score =  sklearn.metrics.roc_auc_score(y_true=val_gt_label_array, y_score=val_preds_roc_array)\n                \n        return summary_loss.avg, accuracy_scores.avg, val_preds_softmax_array, val_roc_auc_score\n\n    def save_model(self, path):\n        \"\"\"Save the trained model.\"\"\"\n        self.model.eval()\n        torch.save(self.model.state_dict(), path)\n\n    # will save the weight for the best val loss and corresponding oof preds\n    def save(self, path):\n        \"\"\"Save the weight for the best evaluation loss.\"\"\"\n        self.model.eval()\n        torch.save(\n            {\n                \"model_state_dict\": self.model.state_dict(),\n                \"optimizer_state_dict\": self.optimizer.state_dict(),\n                \"scheduler_state_dict\": self.scheduler.state_dict(),\n                \"best_acc\": self.best_acc,\n                \"best_auc\": self.best_auc,\n                \"best_loss\": self.best_loss,\n                \"epoch\": self.epoch,\n                \"oof_preds\": self.val_predictions,\n            },\n            path,\n        )\n\n    def load(self, path):\n        \"\"\"Load a model checkpoint from the given path.\"\"\"\n        checkpoint = torch.load(path)\n        return checkpoint\n\n\n    def log(self, message):\n        \"\"\"Log a message.\"\"\"\n        if self.config.verbose:\n            print(message)\n        with open(self.config.paths['log_path'], \"a+\") as logger:\n            logger.write(f\"{message}\\n\")","metadata":{"papermill":{"duration":0.077904,"end_time":"2020-12-05T07:02:58.429152","exception":false,"start_time":"2020-12-05T07:02:58.351248","status":"completed"},"tags":[],"id":"HJ7mvz9_NoWy","execution":{"iopub.status.busy":"2021-11-19T17:06:39.605450Z","iopub.execute_input":"2021-11-19T17:06:39.605898Z","iopub.status.idle":"2021-11-19T17:06:39.671330Z","shell.execute_reply.started":"2021-11-19T17:06:39.605863Z","shell.execute_reply":"2021-11-19T17:06:39.670195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train on Folds","metadata":{}},{"cell_type":"code","source":"def train_on_fold(df_folds: pd.DataFrame, config, fold: int):\n    \"\"\"Train the model on the given fold.\"\"\"\n    model = CustomEfficientNet(config=config, pretrained=True)\n    #model = CustomResNext(config=config, pretrained=True)\n    \n    model.to(config.device)\n\n    transforms_train, transforms_val = get_albu_transforms(config)\n    \n    debug = True\n    if debug:\n        train_df = df_folds[df_folds[\"fold\"] != fold].sample(\n        32* 32\n        )\n        val_df = df_folds[df_folds[\"fold\"] == fold].sample(\n        32 * 32\n        )\n    else:\n        train_df = df_folds[df_folds[\"fold\"] != fold].reset_index(drop=True)\n        val_df = df_folds[df_folds[\"fold\"] == fold].reset_index(drop=True)\n\n    train_set = Cassava(train_df, config, transforms=AlbumentationsAugmentation(transforms=transforms_train),\n                        transform_norm=True, meta_features=None)\n    train_loader = DataLoader(train_set,\n                              batch_size=config.batch_size,\n                              shuffle=True,\n                              num_workers=4,\n                              worker_init_fn=seed_worker)\n\n    val_set = Cassava(val_df, config, transforms=AlbumentationsAugmentation(transforms=transforms_val),\n                      transform_norm=True, meta_features=None)\n    val_loader = DataLoader(val_set,batch_size=config.batch_size,shuffle=False,num_workers=4,worker_init_fn=seed_worker)\n\n    melanoma_detector = Trainer(model=model, config=config)\n\n    curr_fold_best_checkpoint = melanoma_detector.fit(train_loader, val_loader,\n                                                      fold)\n\n    # loading checkpoint for all 10 epochs for this current fold\n\n    val_df[[str(c) for c in range(config.num_classes)]] = curr_fold_best_checkpoint[\"oof_preds\"]\n    val_df[\"preds\"] = curr_fold_best_checkpoint[\"oof_preds\"].argmax(1)\n\n    return val_df\n\n\n\ndef get_acc_score(config, result_df):\n    \"\"\"Get the accuracy of model predictions.\"\"\"\n    preds = result_df[\"preds\"].values\n    labels = result_df[config.class_col_name].values\n    score = sklearn.metrics.accuracy_score(y_true=labels, y_pred=preds)\n    return score\n\ndef get_roc_score(config, result_df):\n    max_label = str(np.max(result_df[config.class_col_name].values))\n    preds = result_df[max_label].values\n    labels = result_df[config.class_col_name].values\n    score = sklearn.metrics.roc_auc_score(y_true=labels, y_score=preds, multi_class='ovr')\n    return score\n\ndef train_loop(df_folds: pd.DataFrame,config,fold_num: int = None,train_one_fold=False):\n    \"\"\"Perform the training loop on all folds.\"\"\"\n    # here The CV score is the average of the validation fold metric.\n    cv_score_list = []\n    oof_df = pd.DataFrame()\n    if train_one_fold:\n        _oof_df = train_on_fold(df_folds=df_folds,config=config,fold=fold_num)\n        oof_df = pd.concat([oof_df, _oof_df])\n        curr_fold_best_score = get_acc_score(config, _oof_df)\n        print(\"Fold {} OOF Score is {}\".format(fold_num,\n                                               curr_fold_best_score))\n    else:\n        # the below for loop guarantees it starts from 1 for fold.\n        # https://stackoverflow.com/questions/33282444/pythonic-way-to-iterate-through-a-range-starting-at-1\n        for fold in (number+1 for number in range(config.num_folds)):\n            _oof_df = train_on_fold(df_folds=df_folds,config=config, fold=fold)\n            oof_df = pd.concat([oof_df, _oof_df])\n            curr_fold_best_score = get_acc_score(config, _oof_df)\n            cv_score_list.append(curr_fold_best_score)\n            print(\"\\n\\n\\nOOF Score for Fold {}: {}\\n\\n\\n\".format(fold, curr_fold_best_score))\n\n        print(\"CV score\", np.mean(cv_score_list))\n        print(\"Variance\", np.var(cv_score_list))\n        print(\"Five Folds OOF\", get_acc_score(config, oof_df))\n    oof_df.to_csv(\"oof.csv\")\n","metadata":{"papermill":{"duration":0.049689,"end_time":"2020-12-05T07:02:58.566118","exception":false,"start_time":"2020-12-05T07:02:58.516429","status":"completed"},"tags":[],"id":"11pUDPuvNoW0","execution":{"iopub.status.busy":"2021-11-19T17:08:44.252274Z","iopub.execute_input":"2021-11-19T17:08:44.252654Z","iopub.status.idle":"2021-11-19T17:08:44.283139Z","shell.execute_reply.started":"2021-11-19T17:08:44.252620Z","shell.execute_reply":"2021-11-19T17:08:44.281593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_fold_1 = train_loop(df_folds=df_folds, config=config, fold_num=1, train_one_fold=True)\n#train_fold_2 = train_loop(df_folds=df_folds, config=config, fold_num=2, train_one_fold=True)","metadata":{"papermill":{"duration":3166.962232,"end_time":"2020-12-05T07:55:45.555575","exception":false,"start_time":"2020-12-05T07:02:58.593343","status":"completed"},"tags":[],"id":"ZF1PufBDNoW0","outputId":"86db90d8-820a-4bab-f97e-9dfc4e959176","execution":{"iopub.status.busy":"2021-11-19T17:08:55.408352Z","iopub.execute_input":"2021-11-19T17:08:55.408703Z","iopub.status.idle":"2021-11-19T17:09:30.014327Z","shell.execute_reply.started":"2021-11-19T17:08:55.408671Z","shell.execute_reply":"2021-11-19T17:09:30.011585Z"},"trusted":true},"execution_count":null,"outputs":[]}]}