{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# This is the code to create the dataset for training/validation","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-01T06:42:19.91326Z","iopub.execute_input":"2023-06-01T06:42:19.913685Z","iopub.status.idle":"2023-06-01T06:42:26.089112Z","shell.execute_reply.started":"2023-06-01T06:42:19.913653Z","shell.execute_reply":"2023-06-01T06:42:26.088022Z"}}},{"cell_type":"markdown","source":"# Import libraries","metadata":{}},{"cell_type":"code","source":"\nfrom sklearn.metrics import roc_auc_score, accuracy_score, f1_score, log_loss\nimport pickle\nfrom torch.utils.data import DataLoader\nfrom torch.cuda.amp import autocast, GradScaler\nimport warnings\nimport sys\nimport pandas as pd\nimport os\nimport gc\nimport sys\nimport math\nimport time\nimport random\nimport shutil\nfrom pathlib import Path\nfrom contextlib import contextmanager\nfrom collections import defaultdict, Counter\nimport cv2\n\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nfrom tqdm.auto import tqdm\nfrom functools import partial\n\nimport argparse\nimport importlib\nimport torch\nimport torch.nn as nn\nfrom torch.optim import Adam, SGD, AdamW\n\nimport datetime","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:27:06.091710Z","iopub.execute_input":"2023-06-01T10:27:06.092025Z","iopub.status.idle":"2023-06-01T10:27:11.382247Z","shell.execute_reply.started":"2023-06-01T10:27:06.091996Z","shell.execute_reply":"2023-06-01T10:27:11.381243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install segmentation_models_pytorch","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:27:11.384421Z","iopub.execute_input":"2023-06-01T10:27:11.385294Z","iopub.status.idle":"2023-06-01T10:27:28.551123Z","shell.execute_reply.started":"2023-06-01T10:27:11.385259Z","shell.execute_reply":"2023-06-01T10:27:28.549910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import segmentation_models_pytorch as smp","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:27:28.557488Z","iopub.execute_input":"2023-06-01T10:27:28.558764Z","iopub.status.idle":"2023-06-01T10:27:31.125462Z","shell.execute_reply.started":"2023-06-01T10:27:28.558726Z","shell.execute_reply":"2023-06-01T10:27:31.124521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install warmup_scheduler","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:27:31.128068Z","iopub.execute_input":"2023-06-01T10:27:31.128809Z","iopub.status.idle":"2023-06-01T10:27:44.030875Z","shell.execute_reply.started":"2023-06-01T10:27:31.128772Z","shell.execute_reply":"2023-06-01T10:27:44.029742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom torch.utils.data import DataLoader, Dataset\nimport cv2\nimport torch\nimport os\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:27:44.032642Z","iopub.execute_input":"2023-06-01T10:27:44.033339Z","iopub.status.idle":"2023-06-01T10:27:45.191954Z","shell.execute_reply.started":"2023-06-01T10:27:44.033298Z","shell.execute_reply":"2023-06-01T10:27:45.191015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configuration","metadata":{}},{"cell_type":"markdown","source":"**We can change the stride, tile size, number of channels etc here**","metadata":{}},{"cell_type":"code","source":"import os\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nclass CFG:\n    # ============== comp exp name =============\n    comp_name = 'vesuvius'\n\n    # comp_dir_path = './'\n    comp_dir_path = '/kaggle/input/'\n    comp_folder_name = 'vesuvius-challenge-ink-detection'\n    # comp_dataset_path = f'{comp_dir_path}datasets/{comp_folder_name}/'\n    comp_dataset_path = f'{comp_dir_path}{comp_folder_name}/'\n    \n    exp_name = 'vesuvius_2d_slide_exp021'\n\n    # ============== pred target =============\n    target_size = 1\n\n    # ============== model cfg =============\n    model_name = 'Unet'\n#     backbone = 'efficientnet-b0'\n    backbone = 'mit_b2'\n\n    in_chans = 3 # 65\n    # ============== training cfg =============\n    size = 224\n    tile_size = 224\n    stride = tile_size \n\n    train_batch_size = 16 # 32\n    valid_batch_size = train_batch_size * 2\n    use_amp = True    #automatic mixed precission\n\n    scheduler = 'GradualWarmupSchedulerV2'     #study about the schedulers...\n    # scheduler = 'CosineAnnealingLR'\n    epochs = 15 # 30\n\n    # adamW warmupあり\n    warmup_factor = 10\n    # lr = 1e-4 / warmup_factor\n    lr = 1e-4 / warmup_factor\n\n    # ============== fold =============\n    \n\n    # objective_cv = 'binary'  # 'binary', 'multiclass', 'regression'\n    metric_direction = 'maximize'  # maximize, 'minimize'\n    # metrics = 'dice_coef'\n\n    # ============== fixed =============\n    pretrained = True\n    inf_weight = 'best'  # 'best'\n\n    min_lr = 1e-6\n    weight_decay = 1e-6\n    max_grad_norm = 1000\n\n    print_freq = 50\n    num_workers = 4\n\n    seed = 42\n\n    # ============== set dataset path =============\n    print('set dataset path')\n\n    outputs_path = f'/kaggle/working/outputs/{comp_name}/{exp_name}/'\n\n    submission_dir = outputs_path + 'submissions/'\n    submission_path = submission_dir + f'submission_{exp_name}.csv'\n\n    model_dir = outputs_path + \\\n        f'{comp_name}-models/'\n    \n    data_dir = outputs_path + \\\n        f'{comp_name}-data/'\n\n    figures_dir = outputs_path + 'figures/'\n\n    log_dir = outputs_path + 'logs/'\n    log_path = log_dir + f'{exp_name}.txt'\n\n    # ============== augmentation =============\n    train_aug_list = [\n        # A.RandomResizedCrop(\n        #     size, size, scale=(0.85, 1.0)),\n        A.Resize(size, size),\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.5),\n        A.RandomBrightnessContrast(p=0.75),\n        A.ShiftScaleRotate(p=0.75),\n        A.OneOf([\n                A.GaussNoise(var_limit=[10, 50]),\n                A.GaussianBlur(),\n                A.MotionBlur(),\n                ], p=0.4),\n        A.GridDistortion(num_steps=5, distort_limit=0.3, p=0.5),\n        A.CoarseDropout(max_holes=1, max_width=int(size * 0.3), max_height=int(size * 0.3), \n                        mask_fill_value=0, p=0.5),\n        # A.Cutout(max_h_size=int(size * 0.6),\n        #          max_w_size=int(size * 0.6), num_holes=1, p=1.0),\n        A.Normalize(\n            mean= [0] * in_chans,\n            std= [1] * in_chans\n        ),\n        ToTensorV2(transpose_mask=True),\n    ]\n\n    valid_aug_list = [\n        A.Resize(size, size),\n        A.Normalize(\n            mean= [0] * in_chans,\n            std= [1] * in_chans\n        ),\n        ToTensorV2(transpose_mask=True),\n    ]\n","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:27:45.193568Z","iopub.execute_input":"2023-06-01T10:27:45.193938Z","iopub.status.idle":"2023-06-01T10:27:45.210781Z","shell.execute_reply.started":"2023-06-01T10:27:45.193904Z","shell.execute_reply":"2023-06-01T10:27:45.209719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def init_logger(log_file):\n    from logging import getLogger, INFO, FileHandler, Formatter, StreamHandler\n    logger = getLogger(__name__)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=log_file)\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    return logger\n\ndef set_seed(seed=None, cudnn_deterministic=True):\n    if seed is None:\n        seed = 42\n\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = cudnn_deterministic\n    torch.backends.cudnn.benchmark = False","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:27:45.212253Z","iopub.execute_input":"2023-06-01T10:27:45.212863Z","iopub.status.idle":"2023-06-01T10:27:45.250111Z","shell.execute_reply.started":"2023-06-01T10:27:45.212830Z","shell.execute_reply":"2023-06-01T10:27:45.249142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:27:45.251753Z","iopub.execute_input":"2023-06-01T10:27:45.252517Z","iopub.status.idle":"2023-06-01T10:27:45.260705Z","shell.execute_reply.started":"2023-06-01T10:27:45.252484Z","shell.execute_reply":"2023-06-01T10:27:45.259782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_dirs(cfg):\n    for dir in [cfg.model_dir, cfg.figures_dir, cfg.submission_dir, cfg.log_dir, CFG.data_dir]:\n        os.makedirs(dir, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:27:45.262085Z","iopub.execute_input":"2023-06-01T10:27:45.262580Z","iopub.status.idle":"2023-06-01T10:27:45.273476Z","shell.execute_reply.started":"2023-06-01T10:27:45.262550Z","shell.execute_reply":"2023-06-01T10:27:45.272578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cfg_init(cfg, mode='train'):\n    set_seed(cfg.seed)\n    # set_env_name()\n    # set_dataset_path(cfg)\n\n    if mode == 'train':\n        make_dirs(cfg)","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:27:45.278307Z","iopub.execute_input":"2023-06-01T10:27:45.278599Z","iopub.status.idle":"2023-06-01T10:27:45.284893Z","shell.execute_reply.started":"2023-06-01T10:27:45.278576Z","shell.execute_reply":"2023-06-01T10:27:45.283453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cfg_init(CFG)\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\nLogger = init_logger(log_file=CFG.log_path)\n\nLogger.info('\\n\\n-------- exp_info -----------------')\n# Logger.info(datetime.datetime.now().strftime('%Y年%m月%d日 %H:%M:%S'))","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:27:45.287166Z","iopub.execute_input":"2023-06-01T10:27:45.287963Z","iopub.status.idle":"2023-06-01T10:27:45.324239Z","shell.execute_reply.started":"2023-06-01T10:27:45.287931Z","shell.execute_reply":"2023-06-01T10:27:45.323445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Change the input channels according to trainig config","metadata":{}},{"cell_type":"code","source":"def read_image_mask(fragment_id):\n\n    images = []\n\n    # idxs = range(65)\n    mid = 65 // 2\n    start = 29\n    end = 32\n#     start = mid - CFG.in_chans // 2\n#     end = mid + CFG.in_chans // 2\n    idxs = range(start, end)\n\n    for i in tqdm(idxs):\n        \n        image = cv2.imread(CFG.comp_dataset_path + f\"train/{fragment_id}/surface_volume/{i:02}.tif\", 0)\n\n        pad0 = (CFG.tile_size - image.shape[0] % CFG.tile_size)\n        pad1 = (CFG.tile_size - image.shape[1] % CFG.tile_size)\n\n        image = np.pad(image, [(0, pad0), (0, pad1)], constant_values=0)\n\n        images.append(image)\n    images = np.stack(images, axis=2)\n\n    mask = cv2.imread(CFG.comp_dataset_path + f\"train/{fragment_id}/inklabels.png\", 0)\n    mask = np.pad(mask, [(0, pad0), (0, pad1)], constant_values=0)\n\n    mask = mask.astype('float32')\n    mask /= 255.0\n    \n    return images, mask","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:27:45.326285Z","iopub.execute_input":"2023-06-01T10:27:45.327320Z","iopub.status.idle":"2023-06-01T10:27:45.335848Z","shell.execute_reply.started":"2023-06-01T10:27:45.327288Z","shell.execute_reply":"2023-06-01T10:27:45.334811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_valid_dataset():\n    images = []\n    masks = []\n    xyxys = []\n    fragment_no = []\n\n    for fragment_id in range(1, 4):\n\n        image, mask = read_image_mask(fragment_id)\n        print(image.shape)\n\n        x1_list = list(range(0, image.shape[1]-CFG.tile_size+1, CFG.stride))\n        y1_list = list(range(0, image.shape[0]-CFG.tile_size+1, CFG.stride))\n\n        for y1 in y1_list:\n            for x1 in x1_list:\n                y2 = y1 + CFG.tile_size\n                x2 = x1 + CFG.tile_size\n                # xyxys.append((x1, y1, x2, y2))\n        \n                \n                images.append(image[y1:y2, x1:x2])\n                masks.append(mask[y1:y2, x1:x2, None])\n                fragment_no.append(fragment_id)\n                xyxys.append([x1, y1, x2, y2])\n               \n\n    return images, masks, xyxys, fragment_no","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:27:45.337406Z","iopub.execute_input":"2023-06-01T10:27:45.337815Z","iopub.status.idle":"2023-06-01T10:27:45.347331Z","shell.execute_reply.started":"2023-06-01T10:27:45.337784Z","shell.execute_reply":"2023-06-01T10:27:45.346362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images, masks, xyxys, frag_no = get_train_valid_dataset()","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:27:45.348699Z","iopub.execute_input":"2023-06-01T10:27:45.349140Z","iopub.status.idle":"2023-06-01T10:28:07.743367Z","shell.execute_reply.started":"2023-06-01T10:27:45.349108Z","shell.execute_reply":"2023-06-01T10:28:07.741469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# xyxys = np.stack(xyxys)   USE THIS CODE FOR TRAINING LATER ON ","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:28:07.744870Z","iopub.execute_input":"2023-06-01T10:28:07.745233Z","iopub.status.idle":"2023-06-01T10:28:07.749560Z","shell.execute_reply.started":"2023-06-01T10:28:07.745198Z","shell.execute_reply":"2023-06-01T10:28:07.748572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(images),len(masks),len(xyxys),len(frag_no)","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:28:07.751125Z","iopub.execute_input":"2023-06-01T10:28:07.751838Z","iopub.status.idle":"2023-06-01T10:28:07.763634Z","shell.execute_reply.started":"2023-06-01T10:28:07.751805Z","shell.execute_reply":"2023-06-01T10:28:07.762361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type(images),type(masks)","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:28:07.765080Z","iopub.execute_input":"2023-06-01T10:28:07.765490Z","iopub.status.idle":"2023-06-01T10:28:07.774426Z","shell.execute_reply.started":"2023-06-01T10:28:07.765461Z","shell.execute_reply":"2023-06-01T10:28:07.773589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame({'Frag_no': pd.Series(frag_no),\n                   'Images': pd.Series(images),\n                   'Masks': pd.Series(masks),\n                   'XYXYS': pd.Series(xyxys)})","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:28:07.775881Z","iopub.execute_input":"2023-06-01T10:28:07.776358Z","iopub.status.idle":"2023-06-01T10:28:07.791821Z","shell.execute_reply.started":"2023-06-01T10:28:07.776328Z","shell.execute_reply":"2023-06-01T10:28:07.790975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:28:07.794908Z","iopub.execute_input":"2023-06-01T10:28:07.795209Z","iopub.status.idle":"2023-06-01T10:28:41.091172Z","shell.execute_reply.started":"2023-06-01T10:28:07.795185Z","shell.execute_reply":"2023-06-01T10:28:41.089996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:28:41.092803Z","iopub.execute_input":"2023-06-01T10:28:41.093197Z","iopub.status.idle":"2023-06-01T10:28:41.108572Z","shell.execute_reply.started":"2023-06-01T10:28:41.093164Z","shell.execute_reply":"2023-06-01T10:28:41.107449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_csv = CFG.data_dir+'data.csv'\ndf.to_csv(data_csv, index=False)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:28:41.109885Z","iopub.execute_input":"2023-06-01T10:28:41.110309Z","iopub.status.idle":"2023-06-01T10:28:46.288588Z","shell.execute_reply.started":"2023-06-01T10:28:41.110275Z","shell.execute_reply":"2023-06-01T10:28:46.287639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Stratified K Fold","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:28:46.289873Z","iopub.execute_input":"2023-06-01T10:28:46.290837Z","iopub.status.idle":"2023-06-01T10:28:46.295631Z","shell.execute_reply.started":"2023-06-01T10:28:46.290797Z","shell.execute_reply":"2023-06-01T10:28:46.294521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_folds = 5  # Number of folds for cross-validation\nskf = StratifiedKFold(n_splits=num_folds, shuffle=True, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:28:46.296997Z","iopub.execute_input":"2023-06-01T10:28:46.297886Z","iopub.status.idle":"2023-06-01T10:28:46.310418Z","shell.execute_reply.started":"2023-06-01T10:28:46.297844Z","shell.execute_reply":"2023-06-01T10:28:46.309497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom torch.utils.data import DataLoader, Dataset\nimport cv2\nimport torch\nimport os\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:28:46.311991Z","iopub.execute_input":"2023-06-01T10:28:46.312506Z","iopub.status.idle":"2023-06-01T10:28:46.320316Z","shell.execute_reply.started":"2023-06-01T10:28:46.312472Z","shell.execute_reply":"2023-06-01T10:28:46.319126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_transforms(data, cfg):\n    if data == 'train':\n        aug = A.Compose(cfg.train_aug_list)\n    elif data == 'valid':\n        aug = A.Compose(cfg.valid_aug_list)\n\n    # print(aug)\n    return aug\n\nclass CustomDataset(Dataset):\n    def __init__(self, images, cfg, labels=None, transform=None):\n        self.images = images\n        self.cfg = cfg\n        self.labels = labels\n        self.transform = transform\n\n    def __len__(self):\n        # return len(self.df)\n        return len(self.images)\n\n    def __getitem__(self, idx):\n        image = self.images[idx]\n        label = self.labels[idx]\n\n        if self.transform:\n            data = self.transform(image=image, mask=label)\n            image = data['image']\n            label = data['mask']\n\n        return image, label","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:28:46.322431Z","iopub.execute_input":"2023-06-01T10:28:46.322853Z","iopub.status.idle":"2023-06-01T10:28:46.332464Z","shell.execute_reply.started":"2023-06-01T10:28:46.322819Z","shell.execute_reply":"2023-06-01T10:28:46.331456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomModel(nn.Module):\n    def __init__(self, cfg, weight=None):\n        super().__init__()\n        self.cfg = cfg\n\n        self.encoder = smp.Unet(\n            encoder_name=cfg.backbone, \n            encoder_weights=weight,\n            in_channels=cfg.in_chans,\n            classes=cfg.target_size,\n            activation=None,\n        )\n\n    def forward(self, image):\n        output = self.encoder(image)\n        # output = output.squeeze(-1)\n        return output\n\n\ndef build_model(cfg, weight=\"imagenet\"):\n    print('model_name', cfg.model_name)\n    print('backbone', cfg.backbone)\n\n    model = CustomModel(cfg, weight)\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:29:47.531961Z","iopub.execute_input":"2023-06-01T10:29:47.532320Z","iopub.status.idle":"2023-06-01T10:29:47.539437Z","shell.execute_reply.started":"2023-06-01T10:29:47.532291Z","shell.execute_reply":"2023-06-01T10:29:47.538566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport torch.nn as nn\nimport torch\nimport math\nimport time\nimport numpy as np\nimport torch\n\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau\nfrom warmup_scheduler import GradualWarmupScheduler\n\n\nclass GradualWarmupSchedulerV2(GradualWarmupScheduler):\n    \"\"\"\n    https://www.kaggle.com/code/underwearfitting/single-fold-training-of-resnet200d-lb0-965\n    \"\"\"\n    def __init__(self, optimizer, multiplier, total_epoch, after_scheduler=None):\n        super(GradualWarmupSchedulerV2, self).__init__(\n            optimizer, multiplier, total_epoch, after_scheduler)\n\n    def get_lr(self):\n        if self.last_epoch > self.total_epoch:\n            if self.after_scheduler:\n                if not self.finished:\n                    self.after_scheduler.base_lrs = [\n                        base_lr * self.multiplier for base_lr in self.base_lrs]\n                    self.finished = True\n                return self.after_scheduler.get_lr()\n            return [base_lr * self.multiplier for base_lr in self.base_lrs]\n        if self.multiplier == 1.0:\n            return [base_lr * (float(self.last_epoch) / self.total_epoch) for base_lr in self.base_lrs]\n        else:\n            return [base_lr * ((self.multiplier - 1.) * self.last_epoch / self.total_epoch + 1.) for base_lr in self.base_lrs]\n\ndef get_scheduler(cfg, optimizer):\n    scheduler_cosine = torch.optim.lr_scheduler.CosineAnnealingLR(\n        optimizer, cfg.epochs, eta_min=1e-7)\n    scheduler = GradualWarmupSchedulerV2(\n        optimizer, multiplier=10, total_epoch=1, after_scheduler=scheduler_cosine)\n\n    return scheduler\n\ndef scheduler_step(scheduler, avg_val_loss, epoch):\n    scheduler.step(epoch)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:29:48.310687Z","iopub.execute_input":"2023-06-01T10:29:48.311050Z","iopub.status.idle":"2023-06-01T10:29:48.323756Z","shell.execute_reply.started":"2023-06-01T10:29:48.311020Z","shell.execute_reply":"2023-06-01T10:29:48.322678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = build_model(CFG)\nmodel.to(device)\n\noptimizer = AdamW(model.parameters(), lr=CFG.lr)\nscheduler = get_scheduler(CFG, optimizer)","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:29:48.926923Z","iopub.execute_input":"2023-06-01T10:29:48.927274Z","iopub.status.idle":"2023-06-01T10:29:57.459353Z","shell.execute_reply.started":"2023-06-01T10:29:48.927245Z","shell.execute_reply":"2023-06-01T10:29:57.458316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nDiceLoss = smp.losses.DiceLoss(mode='binary')\nBCELoss = smp.losses.SoftBCEWithLogitsLoss()\n\nalpha = 0.5\nbeta = 1 - alpha\nTverskyLoss = smp.losses.TverskyLoss(\n    mode='binary', log_loss=False, alpha=alpha, beta=beta)    #Tversky coefficient = (true positives) / ((true positives) + alpha * (false negatives) + beta * (false positives))\n\ndef criterion(y_pred, y_true):\n    # return 0.5 * BCELoss(y_pred, y_true) + 0.5 * DiceLoss(y_pred, y_true)\n    return BCELoss(y_pred, y_true)\n    # return 0.5 * BCELoss(y_pred, y_true) + 0.5 * TverskyLoss(y_pred, y_true)","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:29:57.465560Z","iopub.execute_input":"2023-06-01T10:29:57.468673Z","iopub.status.idle":"2023-06-01T10:29:57.477364Z","shell.execute_reply.started":"2023-06-01T10:29:57.468634Z","shell.execute_reply":"2023-06-01T10:29:57.476133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_fn(train_loader, model, criterion, optimizer, device):\n    model.train()\n\n    scaler = GradScaler(enabled=CFG.use_amp)\n    losses = AverageMeter()\n\n    for step, (images, labels) in tqdm(enumerate(train_loader), total=len(train_loader)):\n        images = images.to(device)\n        labels = labels.to(device)\n        batch_size = labels.size(0)\n\n        with autocast(CFG.use_amp):\n            y_preds = model(images)\n            loss = criterion(y_preds, labels)\n\n        losses.update(loss.item(), batch_size)\n        scaler.scale(loss).backward()\n\n        grad_norm = torch.nn.utils.clip_grad_norm_(\n            model.parameters(), CFG.max_grad_norm)\n\n        scaler.step(optimizer)\n        scaler.update()\n        optimizer.zero_grad()\n\n    return losses.avg\n\ndef valid_fn(valid_loader, model, criterion, device):\n\n    model.eval()\n    losses = AverageMeter()\n\n    for step, (images, labels) in tqdm(enumerate(valid_loader), total=len(valid_loader)):\n        images = images.to(device)\n        labels = labels.to(device)\n        batch_size = labels.size(0)\n\n        with torch.no_grad():\n            y_preds = model(images)\n            loss = criterion(y_preds, labels)\n        losses.update(loss.item(), batch_size)\n        \n        y_preds = torch.sigmoid(y_preds).to('cpu').numpy()\n\n\n\n    return losses.avg, y_preds, labels.to('cpu').numpy()","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:43:57.045055Z","iopub.execute_input":"2023-06-01T10:43:57.045448Z","iopub.status.idle":"2023-06-01T10:43:57.056503Z","shell.execute_reply.started":"2023-06-01T10:43:57.045410Z","shell.execute_reply":"2023-06-01T10:43:57.055443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import fbeta_score\n\ndef fbeta_numpy(targets, preds, beta=0.5, smooth=1e-5):\n    \"\"\"\n    https://www.kaggle.com/competitions/vesuvius-challenge-ink-detection/discussion/397288\n    \"\"\"\n    y_true_count = targets.sum()\n    ctp = preds[targets==1].sum()\n    cfp = preds[targets==0].sum()\n    beta_squared = beta * beta\n\n    c_precision = ctp / (ctp + cfp + smooth)\n    \n    c_recall = ctp / (y_true_count + smooth)\n    dice = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall + smooth)\n\n    return dice\n\ndef calc_fbeta(mask, mask_pred):\n    mask = mask.flatten()\n    mask_pred = mask_pred.flatten()\n\n    best_th = 0\n    best_dice = 0\n    for th in np.array(range(10, 50+1, 5)) / 100:\n        \n        # dice = fbeta_score(mask, (mask_pred >= th).astype(int), beta=0.5)\n        dice = fbeta_numpy(mask, (mask_pred >= th).astype(int), beta=0.5)\n        print(f'th: {th}, fbeta: {dice}')\n\n        if dice > best_dice:\n            best_dice = dice\n            best_th = th\n    \n    Logger.info(f'best_th: {best_th}, fbeta: {best_dice}')\n    return best_dice, best_th\n\n\ndef calc_cv(mask_gt, mask_pred):\n    best_dice, best_th = calc_fbeta(mask_gt, mask_pred)\n\n    return best_dice, best_th","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:43:58.567680Z","iopub.execute_input":"2023-06-01T10:43:58.568414Z","iopub.status.idle":"2023-06-01T10:43:58.580336Z","shell.execute_reply.started":"2023-06-01T10:43:58.568362Z","shell.execute_reply":"2023-06-01T10:43:58.579426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = df[['Images', 'Masks', 'XYXYS']]  # Features\ny = df['Frag_no']  # Stratification based on the \"Frag_no\" column\n\nfor train_index, val_index in skf.split(X, y):\n    fold=0\n    X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n    y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n    train_images=X_train['Images'].tolist()\n    train_masks= X_train['Masks'].tolist()\n    valid_images= X_val['Images'].tolist()\n    valid_masks =X_val['Masks'].tolist()\n    valid_xyxys= X_val['XYXYS'].tolist()\n    \n    valid_xyxys = np.stack(valid_xyxys)  \n    \n    train_dataset = CustomDataset(\n    train_images, CFG, labels=train_masks, transform=get_transforms(data='train', cfg=CFG))\n    valid_dataset = CustomDataset(\n    valid_images, CFG, labels=valid_masks, transform=get_transforms(data='valid', cfg=CFG))\n\n    train_loader = DataLoader(train_dataset,\n                          batch_size=CFG.train_batch_size,\n                          shuffle=True,\n                          num_workers=CFG.num_workers, pin_memory=True, drop_last=True,\n                          )\n    valid_loader = DataLoader(valid_dataset,\n                          batch_size=CFG.valid_batch_size,\n                          shuffle=False,\n                          num_workers=CFG.num_workers, pin_memory=True, drop_last=False)\n    \n    \n    \n    if CFG.metric_direction == 'minimize':\n        best_score = np.inf\n    elif CFG.metric_direction == 'maximize':\n        best_score = -1\n\n    best_loss = np.inf\n\n    for epoch in range(CFG.epochs):\n\n        start_time = time.time()\n\n        # train\n        avg_loss = train_fn(train_loader, model, criterion, optimizer, device)\n\n        # eval\n        avg_val_loss, mask_pred, labels = valid_fn(\n            valid_loader, model, criterion, device)\n\n        scheduler_step(scheduler, avg_val_loss, epoch)\n\n        best_dice, best_th = calc_cv(labels, mask_pred)\n\n        # score = avg_val_loss\n        score = best_dice\n\n        elapsed = time.time() - start_time\n\n        Logger.info(\n            f'Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f}  avg_val_loss: {avg_val_loss:.4f}  time: {elapsed:.0f}s')\n        # Logger.info(f'Epoch {epoch+1} - avgScore: {avg_score:.4f}')\n        Logger.info(\n            f'Epoch {epoch+1} - avgScore: {score:.4f}')\n    \n        if CFG.metric_direction == 'minimize':\n            update_best = score < best_score\n        elif CFG.metric_direction == 'maximize':\n            update_best = score > best_score\n    \n        if update_best:\n            best_loss = avg_val_loss\n            best_score = score\n    \n            Logger.info(\n                f'Epoch {epoch+1} - Save Best Score: {best_score:.4f} Model')\n            Logger.info(\n                f'Epoch {epoch+1} - Save Best Loss: {best_loss:.4f} Model')\n            \n            torch.save({'model': model.state_dict(),\n                        'preds': mask_pred},\n                        CFG.model_dir + f'{CFG.model_name}_fold{fold}_best.pth')\n    fold=fold+1\n            \n        \n            \n    ","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:47:10.124960Z","iopub.execute_input":"2023-06-01T10:47:10.125354Z","iopub.status.idle":"2023-06-01T10:48:58.489836Z","shell.execute_reply.started":"2023-06-01T10:47:10.125321Z","shell.execute_reply":"2023-06-01T10:48:58.488327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels.shape","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:37:43.089298Z","iopub.execute_input":"2023-06-01T10:37:43.089798Z","iopub.status.idle":"2023-06-01T10:37:43.097023Z","shell.execute_reply.started":"2023-06-01T10:37:43.089757Z","shell.execute_reply":"2023-06-01T10:37:43.095947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type(mask_pred)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:42:19.127196Z","iopub.execute_input":"2023-06-01T10:42:19.128152Z","iopub.status.idle":"2023-06-01T10:42:19.135123Z","shell.execute_reply.started":"2023-06-01T10:42:19.128115Z","shell.execute_reply":"2023-06-01T10:42:19.134168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dice = fbeta_numpy(labels, (mask_pred >= 0.4).astype(int), beta=0.5)","metadata":{"execution":{"iopub.status.busy":"2023-06-01T10:41:00.661431Z","iopub.execute_input":"2023-06-01T10:41:00.661792Z","iopub.status.idle":"2023-06-01T10:41:00.731702Z","shell.execute_reply.started":"2023-06-01T10:41:00.661764Z","shell.execute_reply":"2023-06-01T10:41:00.730333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}