{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"kaggle = True\nmode = 'training'\n#mode = 'inference'\ndebug = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\n\nif kaggle:\n    sys.path.append('/kaggle/input/pytorch-image-models/pytorch-image-models-master')\n    sys.path.append('/kaggle/input/timm-pretrained-efficientnet/efficientnet')\n    sys.path.append('/kaggle/input/pycm-master/pycm-master')\n    sys.path.append('/kaggle/input/utilities')\nelse:\n    sys.path.append('./utils')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport cv2\nimport random\nimport time\nimport gc\nimport json\nimport numbers\nimport copy\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom functools import partial\nfrom collections import OrderedDict\nfrom pathlib import Path\nfrom PIL import Image\nfrom tqdm import tqdm\nfrom pycm import ConfusionMatrix\n\nimport timm\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.cuda.amp import autocast, GradScaler\n\nimport torchvision.transforms as transforms\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom sklearn import metrics\nfrom sklearn.model_selection import StratifiedKFold\nfrom losses import LabelSmoothing, MyCrossEntropyLoss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if kaggle:\n    data_dir = '/kaggle/input/cassava-leaf-disease-classification'\n    cache_dir = '/kaggle/working'\n    input_dir = '/kaggle/input'\nelse:\n    data_dir = '../../../data/cassava'\n    cache_dir = './'\n    input_dir = './cache'\n\ntrain_images_dir = Path(data_dir) / 'train_images'\ntest_images_dir = Path(data_dir) / 'test_images'\nsave_weights_dir = Path(cache_dir) / 'weights'\nload_weights_dir = Path(input_dir) / 'weights'\n\nos.makedirs(str(save_weights_dir), exist_ok=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class GlobalConfig:\n    device_ids = [0, 1]\n    device = torch.device(f'cuda:{device_ids[0]}' if torch.cuda.is_available() else \"cpu\")\n    image_model_name = 'tf_efficientnet_b4_ns'\n    load_weights = False\n    use_multi_gpus = False\n    num_workers = 8\n    batch_size = 16\n    num_folds = 2 if debug else 5\n    num_epochs = 3 if debug else 20\n    seed = 42\n    shuffle = False\n    onehot_cols = ['label_0', 'label_1', 'label_2', 'label_3', 'label_4']\n    num_targets = len(onehot_cols)\n    use_tta = True\n    weights_mode = 'loss'\n\nclass DataConfig:\n    batch_size = GlobalConfig.batch_size\n    num_workers = GlobalConfig.num_workers\n    onehot_cols = GlobalConfig.onehot_cols\n    scale_size = (512, 512)\n    input_size = (512, 512)\n    # data transforms\n    trans_params = {\n        'interpolation': 'BILINEAR',\n        'random_resized_crop_scale': (0.08, 1.0),\n        'random_resized_crop_ratio': (0.75, 1.3333333333333333),\n        'rgb_mean': (0.485, 0.456, 0.406),\n        'rgb_std': (0.229, 0.224, 0.225),\n        'hue_shift_limit': (-10, 10),\n        'sat_shift_limit': (-15, 15),\n        'val_shift_limit': (-10, 10),\n        'brightness_limit': (-0.05, 0.05),\n        'contrast_limit': (-0.05, 0.05)\n    }\n\nclass ModelConfig:\n    image_model_name = GlobalConfig.image_model_name\n    num_targets = GlobalConfig.num_targets\n    if kaggle:\n        json_path = f\"/kaggle/input/timm-pretrained-efficientnet/index.json\"\n        weights_dict = {}\n        with open(json_path, mode=\"r\") as f:\n            weights_dict = json.load(f)\n        weight_name = weights_dict['efficientnet'][f'{image_model_name}']\n        path = f\"/kaggle/input/timm-pretrained-efficientnet/efficientnet/{weight_name}\"\n\nclass FitterConfig:\n    device = GlobalConfig.device\n    num_epochs = GlobalConfig.num_epochs\n    batch_size = GlobalConfig.batch_size\n    onehot_cols = GlobalConfig.onehot_cols\n    use_tta = GlobalConfig.use_tta\n    weights_mode = GlobalConfig.weights_mode\n    num_targets = len(onehot_cols)\n    num_augs_tta = 3\n    early_stopping_start = 0 if debug else 8\n    early_stopping_rounds = 4\n    iters_to_accumulate = 2\n    change_device_config = True\n    # criterion\n    train_criterion = LabelSmoothing(smoothing=0.05)\n#    train_criterion = MyCrossEntropyLoss()\n    valid_criterion = MyCrossEntropyLoss()\n    # optimizer\n    optimizer = partial(torch.optim.Adam, lr=1e-4, weight_decay=1e-6)\n    # scheduler\n    scheduler = partial(torch.optim.lr_scheduler.ReduceLROnPlateau, mode='max', patience=2, factor=0.2)\n#    scheduler = partial(torch.optim.lr_scheduler.CosineAnnealingWarmRestarts, T_0=10, T_mult=1, eta_min=1e-6, last_epoch=-1)\n    step_scheduler = False  # do scheduler.step after optimizer.step\n    valid_scheduler = True  # do scheduler.step after validation stage loss\n    scheduler_update_by_loss = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(str(Path(data_dir) / 'train.csv'))\ntest_df = pd.read_csv(str(Path(data_dir) / 'sample_submission.csv'))\n\ntrain_df = train_df.sample(100).reset_index(drop=True) if debug else train_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def feature_engineering(df, config=GlobalConfig):\n    df_ = df.copy()\n    df_ = pd.get_dummies(df_, columns=['label'])\n    df_['label'] = df['label']\n    return df_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://github.com/knjcode/pytorch-finetuner\ndef custom_four_crop(img, size):\n    w, h = img.size\n    crop_h, crop_w = size\n    if crop_w > w or crop_h > h:\n        raise ValueError(\"Requested crop size {} is bigger than input size {}\".format(size, (h, w)))\n    \n    center = transforms.functional.center_crop(img, (crop_h, crop_w))\n    full = transforms.functional.resize(img, (crop_h, crop_w))\n\n    img_ = transforms.functional.hflip(img)\n\n    center_ = transforms.functional.center_crop(img_, (crop_h, crop_w))\n    full_ = transforms.functional.resize(img_, (crop_h, crop_w))\n\n    return (center, full, center_, full_)\n\nclass CustomFourCrop(object):\n    def __init__(self, size):\n        self.size = size\n        if isinstance(size, numbers.Number):\n            self.size = (int(size), int(size))\n        else:\n            assert len(size) == 2, \"Please provide only two dimensions (h, w) for size.\"\n            self.size = size\n\n    def __call__(self, img):\n        return custom_four_crop(img, self.size)\n\n    def __repr__(self):\n        return self.__class__.__name__ + '(size={0})'.format(self.size)\n\nclass RandomAugs(object):\n    def __init__(self, config, num_augs=3):\n        self.input_size = config.input_size\n        self.trans_params = config.trans_params\n        self.num_augs = num_augs\n\n    def __call__(self, image):\n        return self._random_augs(image)\n\n    def __repr__(self):\n        return self.__class__.__name__ + '(size={0})'.format(self.input_size)\n\n    def _random_augs(self, image):\n        images = []\n        for _ in range(self.num_augs):\n            images.append(self._transforms()(image))\n        return tuple(images)\n\n    def _transforms(self):\n        interpolation = getattr(Image, self.trans_params['interpolation'], 2)\n        data_transform = transforms.Compose([\n            transforms.RandomResizedCrop(size=self.input_size,\n                                         scale=self.trans_params['random_resized_crop_scale'],\n                                         ratio=self.trans_params['random_resized_crop_ratio'],\n                                         interpolation=interpolation),\n            transforms.RandomHorizontalFlip(),\n            transforms.RandomAffine(degrees=45., translate=(0.0625, 0.0625), scale=(0.9, 1.1), shear=10.),\n            transforms.ColorJitter(brightness=self.trans_params['brightness_limit'][1],\n                                   contrast=self.trans_params['contrast_limit'][1],\n                                   saturation=self.trans_params['sat_shift_limit'][1] / 255.,\n                                   hue=self.trans_params['hue_shift_limit'][1] / 255.),\n        ])\n        return data_transform","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_transforms(config):\n    trans_params = config.trans_params\n    data_transform = A.Compose([\n        A.RandomResizedCrop(width=config.input_size[0], height=config.input_size[1],\n                            scale=trans_params['random_resized_crop_scale'],\n                            ratio=trans_params['random_resized_crop_ratio']),\n        A.Transpose(p=0.5),\n        A.HorizontalFlip(p=0.5),\n        A.ShiftScaleRotate(p=0.5),\n        A.HueSaturationValue(hue_shift_limit=trans_params['hue_shift_limit'],\n                             sat_shift_limit=trans_params['sat_shift_limit'],\n                             val_shift_limit=trans_params['val_shift_limit'], p=0.5),\n        A.RandomBrightnessContrast(brightness_limit=trans_params['brightness_limit'],\n                                   contrast_limit=trans_params['contrast_limit'], p=0.5),\n        A.Normalize(mean=trans_params['rgb_mean'], std=trans_params['rgb_std'], max_pixel_value=255.0),\n        A.CoarseDropout(p=0.5),\n        A.Cutout(p=0.5),\n        ToTensorV2(),\n    ])\n    return data_transform\n\ndef valid_transforms(config):\n    trans_params = config.trans_params\n    data_transform = A.Compose([\n        A.Resize(width=config.scale_size[0], height=config.scale_size[1]),\n        A.CenterCrop(width=config.input_size[0], height=config.input_size[1]),\n        A.Normalize(mean=trans_params['rgb_mean'], std=trans_params['rgb_std'], max_pixel_value=255.0),\n        ToTensorV2(),\n    ])\n    return data_transform\n\ndef test_transforms(config):\n    trans_params = config.trans_params\n    data_transform = A.Compose([\n        A.Resize(width=config.scale_size[0], height=config.scale_size[1]),\n        A.CenterCrop(width=config.input_size[0], height=config.input_size[1]),\n        A.Normalize(mean=trans_params['rgb_mean'], std=trans_params['rgb_std'], max_pixel_value=255.0),\n        ToTensorV2(),\n    ])\n    return data_transform\n\ndef tta_transforms(config, num_augs=3, mode='normal'):\n    trans_params = config.trans_params\n    interpolation = getattr(Image, trans_params['interpolation'], 2)\n    if mode == 'heavy':\n        main_transforms = RandomAugs(config=config, num_augs=num_augs)\n    else:\n        main_transforms = CustomFourCrop(size=config.input_size)\n    data_transform = transforms.Compose([\n        main_transforms,\n        transforms.Lambda(lambda augs: torch.stack([\n            transforms.ToTensor()(aug) for aug in augs])),\n        transforms.Lambda(lambda augs: torch.stack([\n            transforms.Normalize(trans_params['rgb_mean'], trans_params['rgb_std'])(aug) for aug in augs])),\n    ])\n    return data_transform","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaDataset(Dataset):\n    def __init__(self, config, df, phase, transforms, indices=None):\n        self.image_ids = df.image_id.values\n        self.data_transform = transforms\n        self.phase = phase\n        if phase in ['train', 'valid']:\n            self.targets = df[config.onehot_cols].values.astype(np.float32)\n            images_dir = train_images_dir\n        else:\n            images_dir = test_images_dir\n        self.images_dir = images_dir\n        if indices is None:\n            indices = torch.from_numpy(df.index.values.astype(np.int))\n        self.indices = indices\n\n    def __len__(self):\n        return len(self.indices)\n\n    def __getitem__(self, index):\n        index = self.indices[index]\n        image_id = self.image_ids[index]\n        image_path = str(Path(self.images_dir) / image_id)\n        image = cv2.imread(image_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.phase in ['train', 'valid']:\n            image = self.data_transform(image=image)['image']\n            target = self.targets[index]\n            return image_id, image, target\n        elif self.phase == 'tta':\n            image = transforms.ToPILImage()(image)\n            tta_images = self.data_transform(image)\n            return image_id, tta_images\n        else:\n            image = self.data_transform(image=image)['image']\n            return image_id, image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def check_dataset(config=GlobalConfig, index=[0, 1, 2]):\n    train_df_ = train_df.copy()\n    train_df_ = feature_engineering(train_df_, config=config)\n    train_dataset = CassavaDataset(config=DataConfig, df=train_df_, phase='train',\n                                   transforms=train_transforms(DataConfig))\n    for i in index:\n        image_id, image, target = train_dataset.__getitem__(index=i)\n        image = np.clip(image.numpy().transpose((1, 2, 0)), 0, 1)\n        print(f\"image_id: {image_id}, target: {target}\")\n        plt.imshow(image)\n        plt.show()\n\ncheck_dataset()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DataLoaderMaker():\n\n    def __init__(self, train_df, test_df, config=DataConfig):\n        self.config = config\n        self.batch_size = config.batch_size\n        self.num_workers = config.num_workers\n        self.train_df = train_df\n        self.test_df = test_df\n\n    def _train_dataset(self, indices):\n        return CassavaDataset(df=self.train_df, phase='train', transforms=train_transforms(self.config),\n                              indices=indices, config=self.config)\n\n    def train_dataloader(self, train_index):\n        return DataLoader(self._train_dataset(train_index), batch_size=self.batch_size,\n                          num_workers=self.num_workers, drop_last=True, shuffle=True)\n\n    def _valid_dataset(self, indices):\n        return CassavaDataset(df=self.train_df, phase='valid', transforms=valid_transforms(self.config),\n                              indices=indices, config=self.config)\n\n    def valid_dataloader(self, valid_index):\n        return DataLoader(self._valid_dataset(valid_index), batch_size=self.batch_size,\n                          num_workers=self.num_workers, drop_last=False, shuffle=False)\n\n    def _test_dataset(self):\n        return CassavaDataset(df=self.test_df, phase='test', transforms=test_transforms(self.config),\n                              config=self.config)\n\n    def test_dataloader(self):\n        return DataLoader(self._test_dataset(), batch_size=self.batch_size,\n                          num_workers=self.num_workers, drop_last=False, shuffle=False)\n\n    def _tta_dataset(self, num_augs):\n        return CassavaDataset(df=self.test_df, phase='tta', transforms=tta_transforms(self.config, num_augs),\n                              config=self.config)\n\n    def tta_dataloader(self, num_augs=3):\n        return DataLoader(self._tta_dataset(num_augs), batch_size=self.batch_size,\n                          num_workers=self.num_workers, drop_last=False, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self._reset()\n\n    def _reset(self):\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.sum += val * n\n        self.count += n\n\n    @property\n    def avg(self):\n        return self.sum / self.count\n\nclass RocAucMeter(object):\n    def __init__(self, n_class=2):\n        self.n_class = n_class\n        self.reset()\n\n    def reset(self):\n        # to avoid sklearn method crushing, when batch has only one class\n        self.y_true = np.arange(self.n_class)\n        self.y_pred = np.full((self.n_class, self.n_class), 1. / self.n_class)\n        self.score = 0\n\n    def update(self, y_true, y_pred):\n        y_true = np.hstack(torch.argmax(y_true, dim=-1).cpu().numpy())\n        y_pred = nn.functional.softmax(y_pred, dim=-1).data.cpu().numpy()\n        self.y_true = np.concatenate([self.y_true, y_true], axis=0)\n        self.y_pred = np.concatenate([self.y_pred, y_pred], axis=0)\n        row_sums = np.sum(self.y_pred, 1)\n        row_sums = np.repeat(row_sums, self.n_class).reshape(-1, self.n_class)\n        self.y_pred = np.divide(self.y_pred , row_sums)\n        self.score = metrics.roc_auc_score(self.y_true, self.y_pred, multi_class=\"ovo\")\n\n    @property\n    def avg(self):\n        return self.score\n\nclass AccMeter(object):\n    def __init__(self, n_class):\n        self.n_class = n_class\n        self.reset()\n\n    def reset(self):\n        # to avoid sklearn method crushing, when batch has only one class\n        self.y_true = np.arange(self.n_class)\n        self.y_pred = np.arange(self.n_class)\n\n    def update(self, y_true, y_pred):\n        y_true = np.hstack(torch.argmax(y_true, dim=-1).cpu().numpy())\n        y_pred = np.hstack(torch.argmax(y_pred, dim=-1).data.cpu().numpy())\n        self.y_true = np.hstack((self.y_true, y_true))\n        self.y_pred = np.hstack((self.y_pred, y_pred))\n        self.correct_count = metrics.accuracy_score(self.y_true, self.y_pred, normalize=False)\n        self.score = (self.correct_count - self.n_class) / (self.y_true.shape[0] - self.n_class)\n\n    @property\n    def avg(self):\n        return self.score\n\nclass FoldLogger(object):\n    def __init__(self, save_fig=False):\n        self.save_fig = save_fig\n        self.train_loss = []\n        self.train_acc = []\n        self.valid_loss = []\n        self.valid_acc = []\n\n    def update_train(self, train_loss=None, train_acc=None):\n        self.train_loss.append(train_loss)\n        self.train_acc.append(train_acc)\n\n    def update_valid(self, valid_loss=None, valid_acc=None):\n        self.valid_loss.append(valid_loss)\n        self.valid_acc.append(valid_acc)\n\n    def get_graph(self, title=None, fig_title=None):\n        self.train_loss = [e for e in self.train_loss if e is not None]\n        self.train_acc = [e for e in self.train_acc if e is not None]\n        self.valid_loss = [e for e in self.valid_loss if e is not None]\n        self.valid_acc = [e for e in self.valid_acc if e is not None]\n\n        fig, (axis_l, axis_r) = plt.subplots(ncols=2, figsize=(12, 3))\n\n        if self.train_acc:\n            axis_l.plot(np.arange(0, len(self.train_acc)), self.train_acc,\n                        linestyle=\"solid\", label=\"train acc\", color='b')\n        if self.valid_acc:\n            axis_l.plot(np.arange(0, len(self.valid_acc)), self.valid_acc,\n                        linestyle=\"solid\", label=\"valid acc\", color='r')\n        axis_l.legend(bbox_to_anchor=(1, 0), loc='lower right', borderaxespad=1)\n        axis_l.set_xlabel(\"epochs\")\n        axis_l.set_ylabel(\"acc\")\n        axis_l.grid(True)\n\n        if self.train_loss:\n            axis_r.plot(np.arange(0, len(self.train_loss)), self.train_loss, linestyle=\"solid\", label=\"train loss\", color='b')\n        if self.valid_loss:\n            axis_r.plot(np.arange(0, len(self.valid_loss)), self.valid_loss, linestyle=\"solid\", label=\"valid loss\", color='r')\n        axis_r.legend(bbox_to_anchor=(1, 1), loc='upper right', borderaxespad=1)\n        axis_r.set_xlabel(\"epochs\")\n        axis_r.set_ylabel(\"loss\")\n        axis_r.grid(True)\n\n        if title is not None:\n            fig.suptitle(title)\n\n        if fig_title is not None and self.save_fig:\n            fig.savefig(fig_title)\n\n        plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaNet(nn.Module):\n    def __init__(self, config=ModelConfig):\n        super().__init__()\n        self.image_model = self._get_image_model(config)\n        self.image_model.classifier = nn.Linear(in_features=self.image_model.classifier.in_features,\n                                                out_features=config.num_targets)\n\n    def _get_image_model(self, config):\n        if kaggle:\n            image_model = timm.create_model(config.image_model_name, pretrained=False)\n            image_model.load_state_dict(torch.load(config.path))\n        else:\n            image_model = timm.create_model(config.image_model_name, pretrained=True)\n        return image_model\n\n    def forward(self, image):\n        x = self.image_model(image)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fix_model_state_dict(state_dict):\n    new_state_dict = OrderedDict()\n    for k, v in state_dict.items():\n        name = k\n        if name.startswith('module.'):\n            name = name[7:]  # remove 'module.' of dataparallel\n        new_state_dict[name] = v\n    return new_state_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Fitter(nn.Module):\n\n    def __init__(self, model, dataloader_maker, train_index=None, valid_index=None, config=FitterConfig):\n        super(Fitter, self).__init__()\n        self.device = config.device\n        self.train_criterion = config.train_criterion.to(config.device)\n        self.valid_criterion = config.valid_criterion.to(config.device)\n        self.train_index = torch.from_numpy(train_index)\n        self.valid_index = torch.from_numpy(valid_index)\n        self.best_loss = np.Inf\n        self.best_acc = -np.Inf\n        self.train_dataloader = dataloader_maker.train_dataloader(self.train_index)\n        self.valid_dataloader = dataloader_maker.valid_dataloader(self.valid_index)\n        self.test_dataloader = dataloader_maker.test_dataloader()\n        self.tta_dataloader = dataloader_maker.tta_dataloader(config.num_augs_tta)\n        if not kaggle and config.use_multi_gpus and torch.cuda.device_count() > 1:\n            model = nn.DataParallel(model, device_ids=config.device_ids)\n        self.model = model.to(config.device)\n        self.optimizer = config.optimizer(params=self.model.parameters())\n        self.scheduler = config.scheduler(optimizer=self.optimizer)\n        self.scaler = GradScaler()\n        self.num_epochs = config.num_epochs\n        self.batch_size = config.batch_size\n        self.early_stopping_start = config.early_stopping_start\n        self.early_stopping_rounds = config.early_stopping_rounds\n        self.num_targets = config.num_targets\n        self.step_scheduler = config.step_scheduler\n        self.valid_scheduler = config.valid_scheduler\n        self.scheduler_update_by_loss = config.scheduler_update_by_loss\n        self.iters_to_accumulate = config.iters_to_accumulate\n        self.onehot_cols = config.onehot_cols\n        self.change_device_config = config.change_device_config\n        self.use_tta = config.use_tta\n        self.weights_mode = config.weights_mode\n\n    def fit(self):\n        self.fold_logger = FoldLogger()\n        wait = 0\n        for epoch in range(self.num_epochs):\n            t = time.time()\n            loss, acc, roc_auc = self._train_one_epoch()\n            print(f\"[RESULT]: Train. Epoch: {epoch}, acc: {acc.avg:.9f}, roc_auc: {roc_auc.avg:.9f}\"\n                  + f\", loss: {loss.avg:.9f}, time: {(time.time() - t):.5f}\")\n            self.fold_logger.update_train(train_loss=loss.avg, train_acc=acc.avg)\n\n            t = time.time()\n            loss, acc, roc_auc, valid_preds_df = self._valid_one_epoch()\n            print(f\"[RESULT]: Valid. Epoch: {epoch}, acc: {acc.avg:.9f}, roc_auc: {roc_auc.avg:.9f}\"\n                  + f\", loss: {loss.avg:.9f}, time: {(time.time() - t):.5f}\")\n            self.fold_logger.update_valid(valid_loss=loss.avg, valid_acc=acc.avg)\n\n            if self.valid_scheduler:\n                if self.scheduler_update_by_loss:\n                    self.scheduler.step(loss.avg)\n                else:\n                    self.scheduler.step()\n\n            current = acc.avg\n            if current > self.best_acc:\n                print(f'Validation acc increased ({self.best_acc:.9f} --> {current:.9f}). Saving model ...')\n                wait = 0\n                self.best_acc = current\n                loss_at_best_acc = loss.avg\n                roc_auc_at_best_acc = roc_auc.avg\n                self.weights_at_best_acc = copy.deepcopy(self.model.state_dict())\n                if self.weights_mode == 'acc':\n                    self.best_weights = self.weights_at_best_acc\n\n            current = loss.avg\n            if current < self.best_loss:\n                print(f'Validation loss decreased ({self.best_loss:.9f} --> {current:.9f}). Saving model ...')\n                wait = 0\n                self.best_loss = current\n                acc_at_best_loss = acc.avg\n                roc_auc_at_best_loss = roc_auc.avg\n                self.weights_at_best_loss = copy.deepcopy(self.model.state_dict())\n                if self.weights_mode == 'loss':\n                    self.best_weights = self.weights_at_best_loss\n            elif epoch > self.early_stopping_start:\n                wait += 1\n                print(f'EarlyStopping counter: {wait} out of {self.early_stopping_rounds}')\n                if (self.early_stopping_rounds > 0) and (wait >= self.early_stopping_rounds):\n                    print('Epoch %05d: early stopping' % (epoch))\n                    break\n\n        if self.weights_mode == 'acc':\n            return loss_at_best_acc, self.best_acc, roc_auc_at_best_acc, valid_preds_df\n        else:\n            return self.best_loss, acc_at_best_loss, roc_auc_at_best_loss, valid_preds_df\n\n    # https://pytorch.org/docs/stable/notes/amp_examples.html\n    # https://www.kaggle.com/khyeh0719/pytorch-efficientnet-baseline-train-amp-aug\n    def _train_one_epoch(self):\n        self.model.train()\n        epoch_loss = AverageMeter()\n        epoch_acc = AccMeter(n_class=self.num_targets)\n        epoch_roc_auc = RocAucMeter(n_class=self.num_targets)\n        for step, (image_ids, images, targets) in enumerate(tqdm(self.train_dataloader)):\n            images = images.to(self.device, dtype=torch.float)\n            targets = targets.to(self.device, dtype=torch.float)\n            batch_size = len(image_ids)\n            with autocast():\n                preds = self.model(images)\n                loss = self.train_criterion(preds, targets)\n            self.scaler.scale(loss / self.iters_to_accumulate).backward()\n            if (step + 1) % self.iters_to_accumulate == 0 or step + 1 == len(self.train_dataloader):\n                self.scaler.step(self.optimizer)\n                self.scaler.update()\n                self.optimizer.zero_grad()\n            if self.step_scheduler:\n                self.scheduler.step()\n            epoch_loss.update(loss.item(), batch_size)\n            epoch_acc.update(targets, preds)\n            epoch_roc_auc.update(targets, preds)\n        return epoch_loss, epoch_acc, epoch_roc_auc\n\n    def _valid_one_epoch(self):\n        self.model.eval()\n        epoch_loss = AverageMeter()\n        epoch_acc = AccMeter(n_class=self.num_targets)\n        epoch_roc_auc = RocAucMeter(n_class=self.num_targets)\n        image_ids_list = []\n        valid_preds = np.empty((0, self.num_targets), dtype=np.float)\n        for image_ids, images, targets in tqdm(self.valid_dataloader):\n            images = images.to(self.device, dtype=torch.float)\n            targets = targets.to(self.device, dtype=torch.float)\n            batch_size = len(image_ids)\n            with torch.no_grad():\n                preds = self.model(images)\n                loss = self.valid_criterion(preds, targets)\n            epoch_loss.update(loss.item(), batch_size)\n            epoch_acc.update(targets, preds)\n            epoch_roc_auc.update(targets, preds)\n            image_ids_list += list(image_ids)\n            valid_preds = np.append(valid_preds, preds.data.cpu().numpy(), axis=0)\n        valid_preds_df = pd.concat([pd.DataFrame(image_ids_list, columns=['image_id']),\n                                    pd.DataFrame(valid_preds, columns=self.onehot_cols)], axis=1)\n        return epoch_loss, epoch_acc, epoch_roc_auc, valid_preds_df\n\n    def validation(self):\n        loss, acc, roc_auc, valid_preds_df = self._valid_one_epoch()\n        return loss.avg, acc.avg, roc_auc.avg, valid_preds_df\n\n    def _prediction(self):\n        self.set_model_weights(self.best_weights)\n        self.model.eval()\n        image_ids_list = []\n        test_preds = np.empty((0, self.num_targets), dtype=np.float)\n        for image_ids, images in tqdm(self.test_dataloader):\n            images = images.to(self.device, dtype=torch.float)\n            batch_size = len(image_ids)\n            with torch.no_grad():\n                preds = self.model(images)\n            image_ids_list += list(image_ids)\n            test_preds = np.append(test_preds, preds.data.cpu().numpy(), axis=0)\n        test_preds_df = pd.concat([pd.DataFrame(image_ids_list, columns=['image_id']),\n                                   pd.DataFrame(test_preds, columns=self.onehot_cols)], axis=1)\n        return test_preds_df\n\n    def _prediction_with_tta(self):\n        self.set_model_weights(self.best_weights)\n        self.model.eval()\n        image_ids_list = []\n        test_preds = np.empty((0, self.num_targets), dtype=np.float)\n        for image_ids, tta_images in tqdm(self.tta_dataloader):\n            tta_images = tta_images.to(self.device, dtype=torch.float).transpose(0, 1)\n            n_augs, batch_size, ch, height, width = tta_images.size()\n            with torch.no_grad():\n                preds_ = torch.zeros(batch_size, self.num_targets).to(self.device)\n                for i in range(n_augs):\n                    preds = self.model(tta_images[i])\n                    preds_ += preds.to(self.device)\n                preds_ /= n_augs\n            image_ids_list += list(image_ids)\n            test_preds = np.append(test_preds, preds_.data.cpu().numpy(), axis=0)\n        test_preds_df = pd.concat([pd.DataFrame(image_ids_list, columns=['image_id']),\n                                   pd.DataFrame(test_preds, columns=self.onehot_cols)], axis=1)\n        return test_preds_df\n\n    def prediction(self):\n        if self.use_tta:\n            test_preds_df = self._prediction_with_tta()\n        else:\n            test_preds_df = self._prediction()\n        return test_preds_df\n\n    def get_best_model_weights(self):\n        return self.best_weights\n\n    def set_model_weights(self, weights):\n        self.model.load_state_dict(weights)\n\n    def save_model_weights(self, path):\n        self.model.eval()\n        torch.save(self.model.state_dict(), path)\n\n    def save_best_model_weights(self, path):\n        torch.save(self.best_weights, path)\n\n    def load_model_weights(self, path):\n        if self.change_device_config:\n            self.best_weights = fix_model_state_dict(torch.load(path))\n        else:\n            self.best_weights = torch.load(path)\n        self.model.load_state_dict(self.best_weights)\n        print(f\"weights: {path} loaded.\")\n\n    def get_fold_log(self, fold=None):\n        if fold is not None:\n            title = f'convergence check of fold {fold}'\n            fig_title = f'fold{fold}.png'\n            self.fold_logger.get_graph(title=title, fig_title=fig_title)\n        else:\n            title = f'convergence check'\n            self.fold_logger.get_graph(title=title)\n\n    def delete(self):\n        del self.model, self.optimizer, self.scheduler, self.scaler","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_loss(preds, targets, config=FitterConfig):\n    preds = torch.from_numpy(preds).to(config.device).float()\n    targets = torch.from_numpy(targets).to(config.device).float()\n    with torch.no_grad():\n        loss = config.valid_criterion(preds, targets)\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_label(df, config=GlobalConfig):\n    return pd.concat([df, pd.DataFrame(np.argmax(df[config.onehot_cols].values, axis=-1), columns=['label'])], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_probability(df, config=GlobalConfig):\n    preds = torch.from_numpy(df[config.onehot_cols].values).to(config.device)\n    with torch.no_grad():\n        probs = nn.functional.softmax(preds, dim=-1).data.cpu().numpy()\n    df_ = pd.concat([df.drop(columns=config.onehot_cols), pd.DataFrame(probs, columns=config.onehot_cols)], axis=1)\n    return df_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_confusion_matrix(preds_list, targets_list):\n    cm = ConfusionMatrix(targets_list, preds_list)\n    cm.relabel(mapping={0: 'cbb', 1: 'cbsd', 2: 'cgm', 3: 'cmd', 4: 'healthy'})\n    cm.save_obj(os.path.join(cache_dir, 'cm'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://tech-blog.optim.co.jp/entry/2020/12/08/100000\ndef plot_cm(cm, normalize=False, title='Confusion matrix', annot=True, fmt='d', cmap='YlGnBu'):\n    data = cm.matrix\n    if normalize:\n        title += '(Normalized)'\n        data = cm.normalized_matrix\n        fmt = '.3f'\n    df = pd.DataFrame(data).T.fillna(0)\n    ax = sns.heatmap(df, annot=annot, cmap=cmap, fmt=fmt)\n    ax.set_title(title)\n    ax.set(xlabel='Predict', ylabel='Actual')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def evaluate_model_with_cv(train_df, test_df, config=GlobalConfig):\n\n    seed_everything(config.seed)\n\n    dataloader_maker = DataLoaderMaker(train_df, test_df, config=DataConfig)\n\n    skf = StratifiedKFold(n_splits=config.num_folds, random_state=config.seed, shuffle=config.shuffle)\n\n    oof_valid_preds_df_ = pd.DataFrame(index=[], columns=['image_id'] + config.onehot_cols)\n    fold_best_losses = []\n    for fold, (train_index, valid_index) in enumerate(skf.split(train_df.values, train_df.label.values)):\n\n        t = time.time()\n        print(f'----- fold {fold}/{config.num_folds - 1} Started. -----')\n\n        model = CassavaNet()\n\n        fitter = Fitter(model, dataloader_maker, train_index, valid_index)\n\n        if mode == 'training':\n            if config.load_weights:\n                file_path = Path(load_weights_dir) / f'{config.image_model_name}_{str(fold)}.pth'\n                fitter.load_model_weights(str(file_path))\n            file_path = Path(save_weights_dir) / f'{config.image_model_name}_{str(fold)}.pth'\n            fold_best_loss, fold_acc, fold_roc_auc, fold_valid_preds_df = fitter.fit()\n            fitter.save_best_model_weights(str(file_path))\n            fitter.get_fold_log(fold)\n        else:\n            file_path = Path(load_weights_dir) / f'{config.image_model_name}_{str(fold)}.pth'\n            fitter.load_model_weights(str(file_path))\n            fold_best_loss, fold_acc, fold_roc_auc, fold_valid_preds_df = fitter.validation()\n\n        print(f\"[fold {fold} best]: Valid. acc: {fold_acc:.9f}, roc_auc: {fold_roc_auc:.9f}\"\n              + f\", loss: {fold_best_loss:.9f}, time: {(time.time() - t):.5f}\")\n\n        fold_best_losses.append(fold_best_loss)\n\n        oof_valid_preds_df_ = pd.concat([oof_valid_preds_df_, fold_valid_preds_df],\n                                        sort=False, ignore_index=True)\n\n        fold_preds_df = fitter.prediction()\n\n        if fold == 0:\n            oof_preds_df = fold_preds_df.copy()\n        else:\n            oof_preds_df[config.onehot_cols] += fold_preds_df[config.onehot_cols]\n\n        fitter.delete()\n\n        del model, fitter\n        del fold_preds_df, fold_valid_preds_df\n        gc.collect()\n\n    oof_valid_preds_df = pd.merge(train_df.image_id, oof_valid_preds_df_, on='image_id', how='outer')\n    oof_preds_df[config.onehot_cols] = oof_preds_df[config.onehot_cols] / config.num_folds\n\n    oof_labels = train_df.label.values\n    oof_valid_preds = np.argmax(oof_valid_preds_df[config.onehot_cols].values, axis=-1)\n    oof_valid_accuracy = metrics.accuracy_score(oof_labels, oof_valid_preds, normalize=True)\n\n    get_confusion_matrix(preds_list=oof_valid_preds.tolist(), targets_list=oof_labels.tolist())\n\n    print(f\"[OOF]: acc: {oof_valid_accuracy:.9f}, loss (CV): {np.mean(fold_best_losses):.9f}\"\n          + f\", std of fold best: {np.std(fold_best_losses):.9f}\")\n\n    return oof_preds_df, oof_valid_preds_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_everything():\n\n    t = time.time()\n\n    print(\"device: {}\".format(GlobalConfig.device))\n\n    train_df_ = train_df.copy()\n    test_df_ = test_df.copy()\n    \n    train_df_ = feature_engineering(train_df_)\n\n    oof_preds_df, oof_valid_preds_df = evaluate_model_with_cv(train_df_, test_df_)\n\n    oof_valid_preds_df = get_label(oof_valid_preds_df)\n    oof_preds_df = get_label(oof_preds_df)\n    submission_df = oof_preds_df.drop(columns=GlobalConfig.onehot_cols).copy()\n\n    oof_valid_preds_df.to_csv(str(Path(cache_dir) / 'valid_preds.csv'), index=False)\n    oof_preds_df.to_csv(str(Path(cache_dir) / 'preds.csv'), index=False)\n    submission_df.to_csv(str(Path(cache_dir) / 'submission.csv'), index=False)\n\n    oof_probs_df = get_probability(oof_preds_df)\n    oof_probs_df.to_csv(str(Path(cache_dir) / 'probs.csv'), index=False)\n\n    print(f'All Completed. total time: {(time.time() - t):.5f}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if __name__ == '__main__':\n    run_everything()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_preds_df = pd.read_csv(str(Path(cache_dir) / 'valid_preds.csv'))\nvalid_preds_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cm = ConfusionMatrix(file=open(os.path.join(cache_dir, 'cm.obj'), 'r'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_cm(cm)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_cm(cm, normalize=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds_df = pd.read_csv(str(Path(cache_dir) / 'preds.csv'))\npreds_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"probs_df = pd.read_csv(str(Path(cache_dir) / 'probs.csv'))\nprobs_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.read_csv(str(Path(cache_dir) / 'submission.csv'))\nsubmission_df","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}