{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# 比赛 https://www.kaggle.com/c/ranzcr-clip-catheter-line-classification\n# 参考 Training: https://www.kaggle.com/ttahara/ranzcr-multi-head-model-training?scriptVersionId=55258318\n# Inference https://www.kaggle.com/rsinda/38th-place-solution-0-972-single-model-5-fold\n\nfrom IPython.core.interactiveshell import InteractiveShell\nInteractiveShell.ast_node_interactivity = 'all'  # 'last_expr'\n\nimport os\nimport sys\nimport gc\nimport math\nimport pickle\nimport random\nimport time\nimport psutil\nimport pytz\nfrom datetime import datetime\nfrom collections import defaultdict\nfrom contextlib import contextmanager\n\nimport warnings\nwarnings.filterwarnings('ignore')  # warnings.filterwarnings(action='once')\n\nfrom tqdm.auto import tqdm\nfrom tqdm import tqdm_notebook\n\nimport numpy as np\nimport pandas as pd\n_ = np.seterr(divide='ignore', invalid='ignore')\n\npd.set_option('display.max_columns', None)\n# pd.set_option('display.max_columns', 100)\npd.set_option('display.max_rows', None)\n# pd.set_option('display.max_rows', 100)\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport matplotlib.style as style\nstyle.use('fivethirtyeight')\nimport seaborn as sns\n\n# 直接在cell中显示图片，支持jpg、png、jpeg等格式，\n# Image('./2.JPG') 或者指定显示尺寸 Image(\"./2.png\",width=900,height=400)\nfrom IPython.display import Image  \n\nimport lightgbm as lgb\nfrom sklearn.metrics import roc_auc_score\n\nimport tensorflow as tf\nfrom tensorflow import keras\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\n\ndef show_process_mem_usage(info_str=''):    ## 显示当前进程占用内存大小\n    process = psutil.Process(os.getpid())\n    memory_usage = process.memory_info().rss\n    percent = psutil.virtual_memory().percent\n    \n    tz = pytz.timezone('Asia/Shanghai')\n    now = datetime.now(tz)\n    dt_str = now.strftime(\"%Y-%m-%d %H:%M:%S\")\n    \n    if memory_usage >= 2.**30:\n        print(f'{info_str} current process memory usage: {memory_usage/2.**30:.3f} GB, percentage: {percent:.2f}% 【{dt_str}】')\n    elif memory_usage >= 2.**20:\n        print(f'{info_str} current process memory usage: {memory_usage/2.**20:.3f} MB, percentage: {percent:.2f}% 【{dt_str}】')\n    elif memory_usage >= 2.**10:\n        print(f'{info_str} current process memory usage: {memory_usage/2.**10:.3f} KB, percentage: {percent:.2f}% 【{dt_str}】')\n    else:\n        print(f'{info_str} current process memory usage: {memory_usage} B, percentage: {percent:.2f}% 【{dt_str}】')\n\ndef logging(*info, file_name='./running_log.txt'):\n    log_info = ' '.join([str(s) for s in info])\n    with open(file_name, 'a') as f:\n        f.write(log_info + '\\n')\n\n@contextmanager\ndef trace(trace_msg):    ## 追踪内存变化和运行时间\n    t0 = time.time()\n    p = psutil.Process(os.getpid())\n    m0 = p.memory_info()[0] / 2. ** 30\n    yield\n    m1 = p.memory_info()[0] / 2. ** 30\n    delta = m1 - m0\n    sign = '+' if delta >= 0 else '-'\n    delta = math.fabs(delta)\n    trace_msg = str(trace_msg)\n    \n    tz = pytz.timezone('Asia/Shanghai')\n    now = datetime.now(tz)\n    dt_str = now.strftime(\"%Y-%m-%d %H:%M:%S\")\n    print(f\"[{m1:.3f}GB({sign}{delta:.3f}GB):{time.time() - t0:.3f}sec] {trace_msg} 【{dt_str}】\", file=sys.stdout)\n    \ndef get_time_random_seed():\n    t = int(time.time() * 1000.0)\n    return  (((t & 0xff000000) >> 24) +\n             ((t & 0x00ff0000) >>  8) +\n             ((t & 0x0000ff00) <<  8) +\n             ((t & 0x000000ff) << 24))\n\t\ndef seed_all(random_seed=42):\n    os.environ['PYTHONHASHSEED'] = str(random_seed)\n    random.seed(random_seed)\n    np.random.seed(random_seed)\n    tf.random.set_seed(random_seed)\n    torch.manual_seed(random_seed)\n    torch.cuda.manual_seed(random_seed)\n    torch.backends.cudnn.deterministic = True\n\ndef keepbusy(num=10000):\n    start_t = time.time()\n    for i in range(num):\n        ftpt(f'i: {i}, taken time: {time.time() - start_t:.7f}')\n        time.sleep(60)\n\ndef ftpt(msg = 'having run this cell'):  # foot_print   \n    tz = pytz.timezone('Asia/Shanghai')\n    now = datetime.now(tz)\n    dt_string = now.strftime(\"%Y-%m-%d %H:%M:%S\")\n    print(f'{dt_string}: {msg}')\n    \n# 质数列表  [7, 53, 97, 317, 577, 997, 7753, 9973, 53113, 99991, 153133, 377171, 515371, 737353, 999983, 5157133, 7757537, 9999991, 99999989, 999999937]\nRANDOM_SEED = 53113\nseed_all(RANDOM_SEED)\n    \nftpt()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prepare"},{"metadata":{},"cell_type":"markdown","source":"## import"},{"metadata":{"trusted":true},"cell_type":"code","source":"import copy\nimport shutil\nimport typing as tp\nfrom pathlib import Path\n\nimport yaml\nimport numpy as np\nimport pandas as pd\nfrom scipy.sparse import coo_matrix\n\nfrom tqdm import tqdm\nfrom joblib import Parallel, delayed\n\nimport cv2\nimport albumentations\nfrom albumentations.core.transforms_interface import ImageOnlyTransform, DualTransform\nfrom albumentations.pytorch import ToTensorV2\n\nfrom torch.utils import data\nfrom torchvision import models as torchvision_models\n\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\nimport timm\n\nsys.path.append('../input/pytorch-pfn-extras/pytorch-pfn-extras-0.3.2')\nimport pytorch_pfn_extras as ppe\nfrom pytorch_pfn_extras.training import extensions as ppe_extensions\n\nftpt()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ROOT = Path.cwd().parent\nINPUT = ROOT / 'input'\nOUTPUT = ROOT / 'output'\nDATA = INPUT / \"ranzcr-clip-catheter-line-classification\"\nTRAIN = DATA / 'train'\nTEST = DATA/ 'test'\n\nTRAIN_NPY = INPUT / 'ranzcr-clip-train-numpy'\nTMP = ROOT / 'tmp'\nTMP.mkdir(exist_ok=True)\n\nRANDOM_SEED = 1086\nN_CLASSES = 11\nFOLDS = [1,]\nN_FOLD = 5\n\nCLASSES = [\n    'ETT - Abnormal',\n    'ETT - Borderline',\n    'ETT - Normal',\n    'NGT - Abnormal',\n    'NGT - Borderline',\n    'NGT - Incompletely Imaged',\n    'NGT - Normal',\n    'CVC - Abnormal',\n    'CVC - Borderline',\n    'CVC - Normal',\n    'Swan Ganz Catheter Present'\n]\n\nftpt()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## read data"},{"metadata":{"trusted":true},"cell_type":"code","source":"for p in DATA.iterdir():\n    print(p.name)\n    \ntrain = pd.read_csv(DATA / 'train.csv')\nsmpl_sub = pd.read_csv(DATA / 'sample_submission.csv')\nftpt()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## split fold"},{"metadata":{"trusted":true},"cell_type":"code","source":"def multi_label_stratified_group_k_fold(label_arr: np.array, gid_arr: np.array, n_fold: int, seed: int=42):\n    np.random.seed(seed)\n    random.seed(seed)\n    start_time = time.time()\n    n_train, n_class = label_arr.shape\n    gid_unique = sorted(set(gid_arr))\n    n_group = len(gid_unique)\n    \n    gid2aid = dict(zip(gid_unique, range(n_group)))\n    aid_arr = np.vectorize(lambda x:gid2aid[x])(gid_arr)\n    \n    cnts_by_class = label_arr.sum(axis=0)\n    \n    col, row = np.array(sorted(enumerate(aid_arr), key=lambda x: x[1])).T\n    cnts_by_group = coo_matrix(\n        (np.ones(len(label_arr)), (row, col))\n    ).dot(coo_matrix(label_arr)).toarray().astype(int)\n    del col\n    del row\n    cnts_by_fold = np.zeros((n_fold, n_class), int)\n    \n    groups_by_fold = [[] for fid in range(n_fold)]\n    group_and_cnts = list(enumerate(cnts_by_group))\n    np.random.shuffle(group_and_cnts)\n    print('finished preparation', time.time()-start_time)\n    for aid, cnt_by_g in sorted(group_and_cnts, key=lambda x: -np.std(x[1])):\n        best_fold = None\n        min_eval = None\n        for fid in range(n_fold):\n            cnts_by_fold[fid] += cnt_by_g\n            fold_eval = (cnts_by_fold / cnts_by_class).std(axis=0).mean()\n            cnts_by_fold[fid] -= cnt_by_g\n            \n            if min_eval is None or fold_eval < min_eval:\n                min_eval = fold_eval\n                best_fold = fid\n                \n        cnts_by_fold[best_fold] += cnt_by_g\n        groups_by_fold[best_fold].append(aid)\n    print('finished assignment: ', time.time() - start_time)\n    \n    gc.collect()\n    idx_arr = np.arange(n_train)\n    for fid in range(n_fold):\n        val_groups = groups_by_fold[fid]\n        \n        val_indexs_bool = np.isin(aid_arr, val_groups)\n        train_indexs = idx_arr[~val_indexs_bool]\n        val_indexs = idx_arr[val_indexs_bool]\n        \n        print(f'[fold {fid}]', end=' ')\n        print(f'n_group: (train, val) = ({n_group-len(val_groups)}, {len(val_groups)})', end=' ')\n        print(f'n_sample: (train, val) = ({len(train_indexs)}, {len(val_indexs)})')\n        \n        yield train_indexs, val_indexs\n        \nftpt()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_arr = train[CLASSES].values\ngroup_id = train.PatientID.values\n\ntrain_val_indexs = list(multi_label_stratified_group_k_fold(label_arr, group_id, N_FOLD, RANDOM_SEED))\nftpt()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['fold'] = -1\nfor fold_id, (trn_idx, val_idx) in enumerate(train_val_indexs):\n    train.loc[val_idx, 'fold'] = fold_id\n    \ntrain.groupby('fold')[CLASSES].sum()\nftpt()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_activation(activ_name: str='relu'):\n    act_dict = {\n        'relu': nn.ReLU(inplace=True),\n        'tanh': nn.Tanh(),\n        'sigmoid': nn.Sigmoid(),\n        'identity': nn.Identity(),\n    }\n    if activ_name in act_dict:\n        return act_dict[activ_name]\n    else:\n        raise NotImplementedError\n        \nclass Conv2dBNActiv(nn.Module):\n    def __init__(self, in_channels: int, out_channels: int,\n                 kernel_size: int, stride: int=1, padding: int=0,\n                 bias: bool=False, use_bn: bool=True, activ: str='relu'):\n        super().__init__()\n        layers = []\n        layers.append(nn.Conv2d(in_channels, out_channels, kernel_size, stride, padding, bias=bias))\n        if use_bn:\n            layers.append(nn.BatchNorm2d(out_channels))\n            \n        layers.append(get_activation(activ))\n        self.layers = nn.Sequential(*layers)\n        \n    def forward(self, x):\n        return self.layers(x)\n    \nclass SSEBlock(nn.Module):\n    def __init__(self, in_channels: int):\n        super().__init__()\n        self.channel_squeeze = nn.Conv2d(in_channels=in_channels, \n                                         out_channels=1, kernel_size=1, stride=1,\n                                         padding=0, bias=False)\n        self.sigmoid = nn.Sigmoid()\n        \n    def forward(self, x):\n        h = self.sigmoid(self.channel_squeeze(x))\n        return x*h\n    \nclass SpatialAttentionBlock(nn.Module):\n    def __init__(self, in_channels: int, out_channels_list: tp.List[int]):\n        super().__init__()\n        self.n_layers = len(out_channels_list)\n        channels_list = [in_channels] + out_channels_list\n        assert self.n_layers > 0\n        assert channels_list[-1]==1\n        \n        for i in range(self.n_layers -1):\n            in_chs, out_chs = channels_list[i: i+2]\n            layer = Conv2dBNActiv(in_chs, out_chs, 3, 1, 1, activ='relu')\n            setattr(self, f'conv{i+1}', layer)\n            \n        in_chs, out_chs = channels_list[-2:]\n        layer = Conv2dBNActiv(in_chs, out_chs, 3, 1, 1, activ='sigmoid')\n        setattr(self, f'conv{self.n_layers}', layer)\n        \n    def forward(self, x):\n        h = x\n        for i in range(self.n_layers):\n            h = getattr(self, f'conv{i+1}')(h)\n        \n        h = h*x\n        return h\n    \nftpt()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class SingleHeadModel(nn.Module):\n    def __init__(self, base_name: str='resnext50_32x4d', out_dim: int=11, pretrained=False):\n        self.base_name = base_name\n        super().__init__()\n        \n        base_model = timm.create_model(base_name, pretrained=pretrained)\n        in_features = base_model.num_features\n        \n        base_model.reset_classifier(0)\n        \n        self.backbone = base_model\n        self.head_fc = nn.Sequential(\n            nn.Linear(in_features, in_features),\n            nn.ReLU(inplace=True),\n            nn.Dropout(0.5),\n            nn.Linear(in_features, out_dim)\n        )\n        \n    def forward(self, x):\n        h = self.backbone(x)\n        h = self.head_fc(h)\n        return h\n    \nclass MultiHeadModel(nn.Module):\n    def __init__(self, base_name: str='resnext50_32x4d',\n                 out_dims_head: tp.List[int]=[3, 4, 3, 1], pretrained=False):\n        self.base_name = base_name\n        self.n_heads = len(out_dims_head)\n        super().__init__()\n        \n        base_model = timm.create_model(self.base_name, num_classes=sum(out_dims_head), pretrained=False)\n        in_features = base_model.num_features\n            \n        ## remove global pooling and head classifier\n        base_model.reset_classifier(0, '')\n        \n        self.backbone = base_model\n        \n        for i, out_dim in enumerate(out_dims_head):\n            layer_name = f'head_{i}'\n            layer = nn.Sequential(\n                SpatialAttentionBlock(in_features, [64, 32, 16, 1]),\n                nn.AdaptiveAvgPool2d(output_size=1),\n                nn.Flatten(start_dim=1),\n                nn.Linear(in_features, in_features),\n                nn.ReLU(inplace=True),\n                nn.Dropout(0.5),\n                nn.Linear(in_features, out_dim)\n            )\n            setattr(self, layer_name, layer)\n            \n    def forward(self, x):\n        h = self.backbone(x)\n        hs = [\n            getattr(self, f'head_{i}')(h) for i in range(self.n_heads)\n        ]\n        y = torch.cat(hs, axis=1)\n        return y\n    \nclass MultiHeadResNet200D(nn.Module):\n    def __init__(self, out_dims_head: tp.List[int]=[3, 4, 3, 1], pretrained=False):\n        self.base_name = 'resnet200d_320'\n        self.n_heads = len(out_dims_head)\n        super().__init__()\n        \n        base_model = timm.create_model(self.base_name, num_classes=sum(out_dims_head), pretrained=False)\n        in_features = base_model.num_features\n        \n        if pretrained:\n            pretrained_model_path = '../input/startingpointschestx/resnet200d_320_chestx.pth'\n            state_dict = dict()\n            for k, v in torch.load(pretrained_model_path, map_location='cpu')['model'].items():\n                if k[:6] == 'model.':\n                    k = k.replace('model.', '')\n                state_dict[k] = v\n            base_model.load_state_dict(state_dict)\n            \n        ## remove global pooling and head classifier\n        base_model.reset_classifier(0, '')\n        \n        self.backbone = base_model\n        \n        for i, out_dim in enumerate(out_dims_head):\n            layer_name = f'head_{i}'\n            layer = nn.Sequential(\n                SpatialAttentionBlock(in_features, [64, 32, 16, 1]),\n                nn.AdaptiveAvgPool2d(output_size=1),\n                nn.Flatten(start_dim=1),\n                nn.Linear(in_features, in_features),\n                nn.ReLU(inplace=True),\n                nn.Dropout(0.5),\n                nn.Linear(in_features, out_dim)\n            )\n            setattr(self, layer_name, layer)\n            \n    def forward(self, x):\n        h = self.backbone(x)\n        hs = [\n            getattr(self, f'head_{i}')(h) for i in range(self.n_heads)\n        ]\n        y = torch.cat(hs, axis=1)\n        return y\n\n\nm = MultiHeadResNet200D([3, 4, 3, 1], True)\nm = m.eval()\n\nx = torch.randn(1, 3, 256, 256)\nwith torch.no_grad():\n    y = m(x)\nprint('[forward test]')\nprint(f'input:\\t{x.shape}\\noutput:\\t:{y.shape}')\n\ndel m, x, y\ngc.collect()\n\nftpt()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"class LabeledImageDatasetNumpy(data.Dataset):\n    def __init__(self, \n                 file_list: tp.List[\n                     tp.Tuple[np.ndarray, tp.Union[int, float, np.ndarray]]],\n                 transform_list: tp.List[tp.Dict],\n                 copy_in_channels=True, in_channels=3,):\n        self.file_list = file_list\n        self.transform = ImageTransformForCls(transform_list)\n        self.copy_in_channels = copy_in_channels\n        self.in_channels = in_channels\n        \n    def __len__(self):\n        return len(self.file_list)\n    \n    def __getitem__(self, index):\n        img, label = self.file_list[index]\n        if img.shape[-1] == 2:\n            img = img[..., None]\n        \n        if self.copy_in_channels:\n            img = np.repeat(img, self.in_channels, axis=2)\n            \n        img, label = self.transform((img, label))\n        return img, label\n    \ndef get_file_list_with_array(stgs, train_all):\n    use_fold = stgs['globals']['val_fold']\n    \n    train_idx = train_all[train_all['fold'] != use_fold].index.values\n    if stgs['globals']['debug']:\n        train_idx = train_idx[:len(train_idx) // 20]\n    val_idx = train_all[train_all['fold'] == use_fold].index.values\n    \n    train_data_path = TRAIN_NPY / '{}.npy'.format(stgs['globals']['dataset_name'])\n    print(train_data_path)\n    \n    train_data_arr = np.load(train_data_path, mmap_mode='r')\n    label_arr = train_all[CLASSES].values.astype('f')\n    print(train_data_arr.shape, label_arr.shape)\n    \n    train_file_list = [(train_data_arr[idx][..., None], label_arr[idx]) for idx in train_idx]\n    val_file_list = [(train_data_arr[idx][..., None], label_arr[idx]) for idx in val_idx]\n    \n    return train_file_list, val_file_list\n\ndef get_dataloaders_cls(\n    stgs: tp.Dict, \n    train_file_list: tp.List[tp.List],\n    val_file_list: tp.List[tp.List],\n    dataset_class: data.Dataset\n):\n    train_loader = val_loader = None\n    if train_file_list is not None:\n        train_dataset = dataset_class(train_file_list, **stgs['dataset']['train'])\n        train_loader = data.DataLoader(train_dataset, **stgs['loader']['train'])\n        \n    if val_file_list is not None:\n        val_dataset = dataset_class(val_file_list, **stgs['dataset']['val'])\n        val_loader = data.DataLoader(val_dataset, **stgs['loader']['val'])\n        \n    return train_loader, val_loader\n\nftpt()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## image transform"},{"metadata":{"trusted":true},"cell_type":"code","source":"class ImageTransformBase:\n    def __init__(self, data_augmentations: tp.List[tp.Tuple[str, tp.Dict]]):\n        augmentations_list = [\n            self._get_augmentation(aug_name)(**params)\n            for aug_name, params in data_augmentations]\n        self.data_aug = albumentations.Compose(augmentations_list)\n        \n    def __call__(self, pair: tp.Tuple[np.ndarray]) -> tp.Tuple[np.ndarray]:\n        raise NotImplementedError\n        \n    def _get_augmentation(self, aug_name: str) -> tp.Tuple[ImageOnlyTransform, DualTransform]:\n        if hasattr(albumentations, aug_name):\n            return getattr(albumentations, aug_name)\n        else:\n            return eval(aug_name)\n        \nclass ImageTransformForCls(ImageTransformBase):\n    def __init__(self, data_augmentations: tp.List[tp.Tuple[str, tp.Dict]]):\n        super(ImageTransformForCls, self).__init__(data_augmentations)\n        \n    def __call__(self, in_arrs: tp.Tuple[np.ndarray]) -> tp.Tuple[np.ndarray]:\n        img, label = in_arrs\n        augmented = self.data_aug(image=img)\n        img = augmented['image']\n        return img, label\n    \nftpt()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## metric"},{"metadata":{"trusted":true},"cell_type":"code","source":"class EvalFuncManager(nn.Module):\n    def __init__(self, iters_per_epoch: int, evalfunc_dict: tp.Dict[str, nn.Module], \n                 prefix: str='val') -> None:\n        self.tmp_iter = 0\n        self.iters_per_epoch = iters_per_epoch\n        self.prefix = prefix\n        self.metric_names = []\n        super(EvalFuncManager, self).__init__()\n        for k, v in evalfunc_dict.items():\n            setattr(self, k, v)\n            self.metric_names.append(k)\n        self.reset()\n        \n    def reset(self) -> None:\n        self.tmp_iter = 0\n        for name in self.metric_names:\n            getattr(self, name).reset()\n            \n    def __call__(self, y: torch.Tensor, t: torch.Tensor) -> None:\n        for name in self.metric_names:\n            getattr(self, name).update(y, t)\n        self.tmp_iter += 1\n        \n        if self.tmp_iter == self.iters_per_epoch:\n            ppe.reporting.report({\n                '{}/{}'.format(self.prefix, name): getattr(self, name).compute()\n                for name in self.metric_names\n            })\n            self.reset()\n            \n            \nclass MeanLoss(nn.Module):\n    def __init__(self):\n        super(MeanLoss, self).__init__()\n        self.loss_sum = 0\n        self.n_examples = 0\n        \n    def forward(self, y: torch.Tensor, t: torch.Tensor):\n        return self.loss_func(y, t)\n    \n    def reset(self):\n        self.loss_sum = 0\n        self.n_examples = 0\n        \n    def update(self, y: torch.Tensor, t: torch.Tensor):\n        self.loss_sum += self(y, t).item() * y.shape[0]\n        self.n_examples += y.shape[0]\n        \n    def compute(self):\n        return self.loss_sum / self.n_examples\n    \nclass MyLogLoss(MeanLoss):\n    def __init__(self, **params):\n        super().__init__()\n        self.loss_func = nn.BCEWithLogitsLoss(**params)\n        \nclass MyROCAUC(nn.Module):\n    def __init__(self, average='macro') -> None:\n        self.average = average\n        self._pred_list = []\n        self._true_list = []\n        super(MyROCAUC, self).__init__()\n        \n    def reset(self) -> None:\n        self._pred_list = []\n        self._true_list = []\n        \n    def update(self, y_pred: torch.Tensor, y_true: torch.Tensor) -> None:\n        self._pred_list.append(y_pred.detach().cpu().numpy())\n        self._true_list.append(y_true.detach().cpu().numpy())\n        \n    def compute(self) -> float:\n        y_pred = np.concatenate(self._pred_list, axis=0)\n        y_true = np.concatenate(self._true_list, axis=0)\n        score = roc_auc_score(y_true, y_pred, average=self.average)\n        return score\n    \n    def forward(self, y_pred: torch.Tensor, y_true: torch.Tensor) -> float:\n        self.reset()\n        self.update(y_pred, y_true)\n        return self.compute()\n    \nftpt()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## training utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"def set_random_seed(seed: int=42, deterministic: bool=False):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    \n    \ndef get_stepper(manager, stgs, scheduler):\n    def dummy_step():\n        pass\n    \n    def step():\n        scheduler.step()\n        \n    def step_with_epoch_detail():\n        scheduler.step(manager.epoch_detail)\n        \n    if stgs['scheduler']['name'] == None:\n        return dummy_step, dummy_step\n    elif stgs['scheduler']['name'] == 'CosineAnnealingWarmRestarts':\n        return dummy_step, step_with_epoch_detail\n    elif stgs['scheduler']['name'] == 'OneCycleLR':\n        return dummy_step, step\n    else:\n        return step, dummy_step\n    \ndef run_train_loop(manager, stgs, model, device, train_loader, optimizer, scheduler, loss_func):\n    step_scheduler_by_epoch, step_scheduler_by_iter = get_stepper(manager, stgs, scheduler)\n    \n    if stgs['globals']['use_amp']:\n        while not manager.stop_trigger:\n            model.train()\n            scaler = torch.cuda.amp.GradScaler()\n            for x, t in train_loader:\n                with manager.run_iteration():\n                    x, t = x.to(device), t.to(device)\n                    optimizer.zero_grad()\n                    with torch.cuda.amp.autocast():\n                        y = model(x)\n                        loss = loss_func(y, t)\n                    ppe.reporting.report({'train/loss': loss.item()})\n                    scaler.scale(loss).backward()\n                    scaler.step(optimizer)\n                    scaler.update()\n                    step_scheduler_by_iter()\n            step_scheduler_by_epoch()\n    else:\n        while not manager.stop_trigger:\n            model.train()\n            for x, t in train_loader:\n                with manager.run_iteration():\n                    x, t = x.to(device), t.to(device)\n                    optimizer.zero_grad()\n                    y = model(x)\n                    loss = loss_func(y, t)\n                    ppe.reporting.report({'train/loss': loss.item()})\n                    loss.backward()\n                    optimizer.step()\n                    step_scheduler_by_iter()\n            step_scheduler_by_epoch()\n            \n        \ndef run_eval(stgs, model, device, batch, eval_manager):\n    model.eval()\n    x, t = batch\n    if stgs['globals']['use_amp']:\n        with torch.cuda.amp.autocast():\n            y = model(x.to(device))\n            eval_manager(y, t.to(device))\n    else:\n        y = model(x.to(device))\n        eval_manager(y, t.to(device))\n        \nftpt()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def set_extensions(manager, args, model, device, val_loader, \n                   optimizer, eval_manager, print_progress: bool=False):\n    eval_names = [f'val/{name}' for name in eval_manager.metric_names]\n    \n    log_extensions = [\n        ppe_extensions.observe_lr(optimizer=optimizer),\n        ppe_extensions.LogReport(),\n        ppe_extensions.PlotReport(['train/loss', 'val/loss'], 'epoch', filename='loss.png'),\n        ppe_extensions.PlotReport(['lr'], 'epoch', filename='lr.png'),\n        ppe_extensions.PrintReport([\n            'epoch', 'iteration', 'lr', 'train/loss', *eval_names, 'elapsed_time'])\n    ]\n    if print_progress:\n        log_extensions.append(ppe_extensions.ProgressBar(update_iterval=20))\n        \n    for ext in log_extensions:\n        manager.extend(ext)\n        \n    manager.extend(\n        ppe_extensions.Evaluator(\n            val_loader, model,\n            eval_func=lambda *batch: run_eval(args, model, device, batch, eval_manager)),\n        trigger = (1, 'epoch'))\n    \n    manager.extend(\n        ppe_extensions.snapshot(target=model, filename='snapshot_epoch_{.epoch}.pth'),\n        trigger=ppe.training.triggers.MaxValueTrigger(key='val/metric', trigger=(1, 'epoch')))\n    \n    return manager\n    \ndef train_one_fold(settings, train_all, output_path, print_progress=False):\n    torch.backends.cudnn.benchmark = True\n    set_random_seed(settings['globals']['seed'])\n    \n    train_file_list, val_file_list = get_file_list_with_array(settings, train_all)\n    print(f'train: {len(train_file_list)}, val: {len(val_file_list)}')\n    \n    device = torch.device(settings['globals']['device'])\n    train_loader, val_loader = get_dataloaders_cls(\n        settings, train_file_list, val_file_list, LabeledImageDatasetNumpy)\n    \n    model = MultiHeadResNet200D(**settings['model']['params'])\n    model.to(device)\n    \n    ## get optimizer\n    optimizer = getattr(\n        torch.optim, settings['optimizer']['name']\n    )(model.parameters(), **settings['optimizer']['params'])\n    \n    ## get scheduler\n    if settings['scheduler']['name'] == 'OneCycleLR':\n        settings['scheduler']['params']['epochs'] = settings['globals']['max_epoch']\n        settings['scheduler']['params']['step_per_epoch'] = len(train_loader)\n    scheduler = getattr(\n        torch.optim.lr_scheduler, settings['scheduler']['name']\n    )(optimizer, **settings['scheduler']['params'])\n    \n    if hasattr(nn, settings['loss']['name']):\n        loss_func = getattr(nn, settings['loss']['name'])(**settings['loss']['params'])\n    else:\n        loss_func = eval(settings['loss']['name'])(**settings['loss']['params'])\n    loss_func.to(device)\n    \n    eval_manager = EvalFuncManager(\n        len(val_loader), {\n            metric['report_name']: eval(metric['name'])(**metric['params'])\n            for metric in settings['eval']\n        })\n    eval_manager.to(device)\n    \n    trigger = ppe.training.triggers.EarlyStoppingTrigger(\n        check_trigger = (1, 'epoch'),\n        monitor = 'val/metric', mode='max',\n        patience = settings['globals']['patience'], verbose=False,\n        max_trigger = (settings['globals']['max_epoch'], 'epoch'),\n    )\n    manager = ppe.training.ExtensionsManager(\n        model, optimizer, settings['globals']['max_epoch'],\n        iters_per_epoch=len(train_loader),\n        stop_trigger=trigger, out_dir=output_path,\n    )\n    manager = set_extensions(\n        manager, settings, model, device, val_loader, \n        optimizer, eval_manager, print_progress)\n    \n    run_train_loop(manager, settings, model, device, train_loader,\n                   optimizer, scheduler, loss_func)\n    \nftpt()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"stgs_str = \"\"\"\nglobals:\n  seed: 1086\n  device: cuda\n  max_epoch: 16\n  patience: 3\n  dataset_name: train_512x512\n  use_amp: True\n  val_fold: 0\n  debug: False\n\ndataset:\n  name: LabeledImageDatasetNumpy\n  train:\n    transform_list:\n      - [HorizontalFlip, {p: 0.5}]\n      - [ShiftScaleRotate, {\n          p: 0.5, shift_limit: 0.2, scale_limit: 0.2,\n          rotate_limit: 20, border_mode: 0, value: 0, mask_value: 0}]\n      - [RandomResizedCrop, {height: 512, width: 512, scale: [0.9, 1.0]}]\n      - [Cutout, {max_h_size: 51, max_w_size: 51, num_holes: 5, p: 0.5}]\n      - [Normalize, {\n          always_apply: True, max_pixel_value: 255.0,\n          mean: [0.4887381077884414], std: [0.23064819430546407]}]\n      - [ToTensorV2, {always_apply: True}]\n  val:\n    transform_list:\n      - [Normalize, {\n          always_apply: True, max_pixel_value: 255.0,\n          mean: [0.4887381077884414], std: [0.23064819430546407]}]\n      - [ToTensorV2, {always_apply: True}]\n\nloader:\n  train: {batch_size: 16, shuffle: True, num_workers: 2, pin_memory: True, drop_last: True}\n  val: {batch_size: 32, shuffle: False, num_workers: 2, pin_memory: True, drop_last: False}\n\nmodel:\n  name: MultiHeadResNet200D\n  params:\n    # base_name: resnet200D_320\n    out_dims_head: [3, 4, 3, 1]\n    pretrained: True\n\nloss: {name: BCEWithLogitsLoss, params: {}}\n\neval:\n  - {name: MyLogLoss, report_name: loss, params: {}}\n  - {name: MyROCAUC, report_name: metric, params: {average: macro}}\n\noptimizer:\n    name: Adam\n    params:\n      lr: 2.5e-04\n\nscheduler:\n  name: CosineAnnealingWarmRestarts\n  params:\n    T_0: 16\n    T_mult: 1\n\"\"\"\nstgs = yaml.safe_load(stgs_str)\n\nif stgs[\"globals\"][\"debug\"]:\n    stgs[\"globals\"][\"max_epoch\"] = 1\n    \nftpt()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"stgs_list = []\nfor fold_id in FOLDS:\n    tmp_stgs = copy.deepcopy(stgs)\n    tmp_stgs['globals']['val_fold'] = fold_id\n    stgs_list.append(tmp_stgs)\n    \nftpt()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.cuda.empty_cache()\ngc.collect()\n\ntrain_start_t = time.time()\nfor fold_id, tmp_stgs in zip(FOLDS, stgs_list):\n    print('fold_id: ', fold_id)\n    train_one_fold(tmp_stgs, train, TMP / f'fold{fold_id}', False)\n    torch.cuda.empty_cache()\n    gc.collect()\n    \nprint(f'train finished, total cost time: {time.time()-train_start_t:.4f}')\n\nftpt()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Inference OOF"},{"metadata":{"trusted":true},"cell_type":"code","source":"ls ../tmp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_log_list = []\nfor fold_id, tmp_stgs in zip(FOLDS, stgs_list):\n    exp_dir_path = TMP / f'fold{fold_id}'\n    log = pd.read_json(exp_dir_path / 'log')\n    best_log = log.iloc[[log['val/metric'].idxmax()],]\n    best_epoch = best_log.epoch.values[0]\n    best_log_list.append(best_log)\n    \n    best_model_path = exp_dir_path / f'snapshot_epoch_{best_epoch}.pth'\n    copy_to = f'./best_model_fold{fold_id}.pth'\n    shutil.copy(best_model_path, copy_to)\n    \n    for p in exp_dir_path.glob('*.pth'):\n        p.unlink()\n        \n    shutil.copytree(exp_dir_path, f'./fold{fold_id}')\n    with open(f'./fold{fold_id}/settings.yml', 'w') as fw:\n        yaml.dump(tmp_stgs, fw)\n        \npd.concat(best_log_list, axis=0, ignore_index=True)\n\nftpt()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}