{"cells":[{"metadata":{},"cell_type":"markdown","source":"# GPU"},{"metadata":{"trusted":true},"cell_type":"code","source":"gpu_info = !nvidia-smi\ngpu_info = '\\n'.join(gpu_info)\nprint(gpu_info)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CFG"},{"metadata":{"trusted":true},"cell_type":"code","source":"CONFIG_NAME = 'stacking16.yml'\ndebug = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TTA_LIST = [\n    'CenterCrop-Normalize-ToTensorV2',\n    'CenterCrop-Transpose-Normalize-ToTensorV2',\n#     'CenterCrop-HorizontalFlip-Normalize-ToTensorV2',\n#     'CenterCrop-VerticalFlip-Normalize-ToTensorV2',\n    'Resize-Normalize-ToTensorV2',\n    'Resize-Transpose-Normalize-ToTensorV2',\n#     'Resize-VerticalFlip-Normalize-ToTensorV2',\n#     'Resize-HorizontalFlip-Normalize-ToTensorV2'\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nBefore running this cell, you will set hidden vars by \"add-ons\" menu. \n\"\"\"\n\nfrom kaggle_secrets import UserSecretsClient\n\ndef clone_repository(\n     ssh_keyval: \"str: BEGIN RSA PRIVATE KEY to END RSA PRIVATE KEY\"\n   , ssh_keyname: \"str: id_rsa...\"\n   , gitrepo: \"str: your private repository to clone\"\n   , uname_git: \"str: username of your Git\") -> None:\n   \"\"\"\n   receive hidden vars, then clone private repository to \"/kaggle/working/\"\n   \"\"\"\n\n   ssh_command = f\"ssh -l {uname_git} -i /kaggle/working/{ssh_keyname} -o StrictHostKeyChecking=no -F /dev/null\"\n\n   !rm -rf /kaggle/working/$gitrepo\n   !echo $ssh_keyval> $ssh_keyname\n   !chmod 600 /kaggle/working/$ssh_keyname\n   !git -c core.sshCommand=\"$ssh_command\" clone git@github.com:$uname_git/$gitrepo\n   !rm /kaggle/working/$ssh_keyname\n\nus = UserSecretsClient()\nclone_repository(ssh_keyval=us.get_secret(\"ssh_keyval\"), ssh_keyname=us.get_secret(\"ssh_keyname\")\n   , gitrepo=us.get_secret(\"gitrepo\"), uname_git=us.get_secret(\"uname_git\"))\ndel us\n\nimport sys\nsys.path.append('./kaggle-cassava')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from src.utils.envs.main import create_env\nenv_dict = create_env()\nenv_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# CFG\n# ====================================================\nimport yaml\n\nCONFIG_PATH = f'./kaggle-cassava/config/{CONFIG_NAME}'\nwith open(CONFIG_PATH) as f:\n    config = yaml.load(f)\n\nINFO = config['info']\nTAG = config['tag']\nCFG = config['cfg']\n\nDATA_PATH = env_dict[\"data_path\"]\nenv = env_dict[\"env\"]\nNOTEBOOK_PATH = env_dict[\"notebook_dir\"]\nOUTPUT_DIR = env_dict[\"output_dir\"]\n# TITLE = env_dict[\"title\"]\n\nCFG['train'] = True\nCFG['inference'] = False\n\nCFG['debug'] = debug\n\n# if CFG['debug']:\n#     CFG['epochs'] = 1\n\n# 環境変数\nimport os\nos.environ[\"GCLOUD_PROJECT\"] = INFO['PROJECT_ID']\n\n# 間違ったバージョンを実行しないかチェック\n# assert INFO['TITLE'] == TITLE, f'{TITLE}, {INFO[\"TITLE\"]}'\nTITLE = INFO[\"TITLE\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport glob\n\nmodel_dirs = []\noof_dirs = []\nfor stage1 in CFG['stage1_models']:\n    num = str(stage1).rjust(2, '0')\n    # model\n    model_dir_ = glob.glob(f'../input/{num}t*/')\n    assert len(model_dir_) == 1, model_dir_\n    model_dirs.append(model_dir_[0])\n    # oof\n    oof_dir_ = glob.glob(f'../input/{num}i*/')\n    assert len(oof_dir_) == 1, oof_dir_\n    oof_dirs.append(oof_dir_[0])\nmodel_dirs, oof_dirs","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Library"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\nimport sys\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\n\nimport datetime\nimport os\nimport math\nimport time\nimport random\nimport glob\nimport shutil\nfrom pathlib import Path\nfrom contextlib import contextmanager\nfrom collections import defaultdict, Counter\nimport yaml\n\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn import preprocessing\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import StratifiedKFold\n\nfrom tqdm.auto import tqdm\nfrom functools import partial\n\nimport cv2\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD\nimport torchvision.models as models\nfrom torch.nn.parameter import Parameter\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau\n\nfrom albumentations import (\n    Compose, OneOf, Normalize, Resize, RandomResizedCrop, RandomCrop, HorizontalFlip, VerticalFlip, \n    RandomBrightness, RandomContrast, RandomBrightnessContrast, Rotate, ShiftScaleRotate, Cutout, \n    IAAAdditiveGaussianNoise, Transpose, CenterCrop\n    )\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform\n\nimport timm\nimport mlflow\n\nimport warnings \nwarnings.filterwarnings('ignore')\n\nif CFG['debug']:\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nelse:\n    device = torch.device('cuda')\n\nfrom src.utils.logger import init_logger\nfrom src.utils.utils import seed_torch, EarlyStopping\nfrom src.utils.loss.bi_tempered_logistic_loss import bi_tempered_logistic_loss\nfrom src.utils.augments.randaugment import RandAugment\nfrom src.utils.augments.augmix import RandomAugMix\n\nstart_time = datetime.datetime.now()\nstart_time_str = start_time.strftime('%m%d%H%M')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Directory settings"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Directory settings\n# ====================================================\nif os.path.exists(OUTPUT_DIR):\n    shutil.rmtree(OUTPUT_DIR)\nif not os.path.exists(OUTPUT_DIR):\n    os.makedirs(OUTPUT_DIR)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# save basic files"},{"metadata":{"trusted":true},"cell_type":"code","source":"# with open(f'{OUTPUT_DIR}/{start_time_str}_TAG.json', 'w') as f:\n#     json.dump(TAG, f, indent=4)\n    \n# with open(f'{OUTPUT_DIR}/{start_time_str}_CFG.json', 'w') as f:\n#     json.dump(CFG, f, indent=4)\n\nif not os.path.isfile(NOTEBOOK_PATH):\n    NOTEBOOK_PATH = '__notebook_source__.ipynb'\n    \nimport shutil\nnotebook_path = f'{OUTPUT_DIR}/{start_time_str}_{TITLE}.ipynb'\nshutil.copy2(NOTEBOOK_PATH, notebook_path)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Loading"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(f'{DATA_PATH}/train.csv')\ntest = pd.read_csv(f'{DATA_PATH}/sample_submission.csv')\nlabel_map = pd.read_json(f'{DATA_PATH}/label_num_to_disease_map.json', \n                         orient='index')\n\nif CFG['debug']:\n    train = train.sample(n=1000, random_state=CFG['seed']).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_config_paths = []\nfor model_dir in model_dirs:\n    assert len(glob.glob(f'{model_dir}/*.yml'))==1\n    model_config_paths.append(glob.glob(f'{model_dir}/*.yml')[0])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Utils\n# ====================================================\ndef get_score(y_true, y_pred):\n    return accuracy_score(y_true, y_pred)\n\nlogger_path = OUTPUT_DIR+f'{start_time_str}_train.log'\nLOGGER = init_logger(logger_path)\nseed_torch(seed=CFG['seed'])\n\n\ndef remove_glob(pathname, recursive=True):\n    for p in glob.glob(pathname, recursive=recursive):\n        if os.path.isfile(p):\n            os.remove(p)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Utils\n# ====================================================\ndef get_score(y_true, y_pred):\n    return accuracy_score(y_true, y_pred)\n\n\n@contextmanager\ndef timer(name):\n    t0 = time.time()\n    LOGGER.info(f'[{name}] start')\n    yield\n    LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s.')\n\n\n# def init_logger(log_file=OUTPUT_DIR+'inference.log'):\n#     from logging import getLogger, INFO, FileHandler,  Formatter,  StreamHandler\n#     logger = getLogger(__name__)\n#     logger.setLevel(INFO)\n#     handler1 = StreamHandler()\n#     handler1.setFormatter(Formatter(\"%(message)s\"))\n#     handler2 = FileHandler(filename=log_file)\n#     handler2.setFormatter(Formatter(\"%(message)s\"))\n#     logger.addHandler(handler1)\n#     logger.addHandler(handler2)\n#     return logger\n\n#LOGGER = init_logger()\n\n\ndef seed_torch(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\n# seed_torch(seed=CFG['seed'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# train"},{"metadata":{},"cell_type":"markdown","source":"## Data loading"},{"metadata":{"trusted":true},"cell_type":"code","source":"check_cols = [\"image_id\", \"label\", \"fold\"]\n\ndf_ = None\noof_list = []\nfor oof_dir in oof_dirs:\n    tta_preds = glob.glob(os.path.join(oof_dir, \"*[!submission].csv\"))\n    tta_dict = {}\n    for tta_pred in tta_preds:\n        tta_name = tta_pred.split('/')[-1].split('.')[0]\n        df = pd.read_csv(tta_pred)\n        # 全てのoofでfoldが揃っているかチェック\n        if df_ is not None:\n            assert (df[check_cols] == df_[check_cols]).all().all(), tta_name\n        df_ = df.copy()\n        tta_dict[tta_name] = df\n    oof_list.append(tta_dict)\n    \n\nfirst_df = oof_list[0][TTA_LIST[0]]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"# X.shape = (N, Models, Labels, channel)\nX = np.zeros((len(first_df), len(model_dirs), CFG['target_size'], len(TTA_LIST)), dtype=np.float)\ny = first_df['label'].values\nfolds = first_df[check_cols].values\nfor model_idx, oof in enumerate(oof_list):\n    for channel_idx, tta_name in enumerate(TTA_LIST):\n        X[:, model_idx, :, channel_idx] = oof[tta_name][['0','1','2','3','4']].values\n\n# add Channel dim\n# X = X.reshape(len(oof_list[0]), len(model_dirs), CFG['target_size'], 1)\n# [N, Models, Labels, Channel] -> [N, Channel, Models, Labels]\nX = X.transpose(0, 3, 1, 2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X.shape, y.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class StackingDataset(Dataset):\n    def __init__(self, X: np.ndarray, y: np.ndarray):\n\n        self.X = X\n\n        self.y = y\n\n    def __len__(self):\n        return self.X.shape[0]\n\n    def __getitem__(self, idx):\n        return (\n            torch.tensor(self.X[idx], dtype=torch.float),\n            torch.tensor(self.y[idx], dtype=torch.long),\n        )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# class CNNStacking_(nn.Module):\n#     def __init__(self, n_features, n_labels):\n#         super(CNNStacking_, self).__init__()\n#         self.conv1 = nn.Conv2d(in_channels=1, out_channels=8, kernel_size=(3, 1), bias=False)\n#         self.conv2 = nn.Conv2d(in_channels=8, out_channels=16, kernel_size=(3, 1), bias=False)\n#         self.dense1 = nn.Linear(in_features=16* n_labels, out_features=4 * n_labels)\n#         self.dense2 = nn.Linear(in_features=4 * n_labels, out_features=n_labels)\n#         self.relu = nn.ReLU()\n        \n\n#     def forward(self, x):\n#         print(x.size())\n#         x = self.relu(self.conv1(x))\n#         print(x.size())\n#         x = self.relu(self.conv2(x))\n#         print(x.size())\n#         x = torch.flatten(x)\n#         print(x.size())\n#         x = self.relu(self.dense1(x))\n#         out = self.dense2(x)\n#         return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CNNStacking(nn.Module):\n    def __init__(self, n_labels):\n        super(CNNStacking, self).__init__()\n\n        self.sq = nn.Sequential(\n            nn.Conv2d(in_channels=4, out_channels=8, kernel_size=(3, 1), bias=False),\n            nn.ReLU(),\n            nn.Conv2d(in_channels=8, out_channels=16, kernel_size=(3, 1), bias=False),\n            nn.ReLU(),\n#             nn.Conv2d(in_channels=16, out_channels=32, kernel_size=(2, 1), bias=False),\n#             nn.ReLU(),\n            # nn.Conv2d(in_channels=32, out_channels=64, kernel_size=(3, 1), bias=False),\n            # nn.ReLU(),\n            nn.Flatten(),\n#             nn.Linear(in_features=32* n_labels, out_features=16 * n_labels),\n#             nn.ReLU(),\n            nn.Linear(in_features=16* n_labels, out_features=4 * n_labels),\n            nn.ReLU(),\n            nn.Linear(in_features=4 * n_labels, out_features=n_labels),\n        )\n\n    def forward(self, x):\n        return self.sq(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = CNNStacking(5)\ntrain_dataset = StackingDataset(X, y)\ntrain_loader = DataLoader(train_dataset, batch_size=4, shuffle=True,\n                          num_workers=4, pin_memory=True, drop_last=True)\n\nfor image, label in train_loader:\n    output = model(image)\n    print(output)\n    break","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## helper function"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Helper functions\n# ====================================================\nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\n\ndef asMinutes(s):\n    m = math.floor(s / 60)\n    s -= m * 60\n    return '%dm %ds' % (m, s)\n\n\ndef timeSince(since, percent):\n    now = time.time()\n    s = now - since\n    es = s / (percent)\n    rs = es - s\n    return '%s (remain %s)' % (asMinutes(s), asMinutes(rs))\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# loss\n# ====================================================\ndef get_loss(criterion, y_preds, labels):\n    if TAG['criterion']=='CrossEntropyLoss':\n        loss = criterion(y_preds, labels)\n    elif TAG['criterion'] == 'bi_tempered_logistic_loss':\n        loss = criterion(y_preds, labels, t1=CFG['bi_tempered_loss_t1'], t2=CFG['bi_tempered_loss_t2'])\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Helper functions\n# ====================================================\ndef train_fn(train_loader, model, criterion, optimizer, epoch, scheduler, device):\n    batch_time = AverageMeter()\n    data_time = AverageMeter()\n    losses = AverageMeter()\n    scores = AverageMeter()\n    # switch to train mode\n    model.train()\n    start = end = time.time()\n    global_step = 0\n    for step, (features, labels) in enumerate(train_loader):\n        # measure data loading time\n        data_time.update(time.time() - end)\n        features = features.to(device)\n        labels = labels.to(device)\n        batch_size = labels.size(0)\n        y_preds = model(features)\n        loss = get_loss(criterion, y_preds, labels)\n        # record loss\n        losses.update(loss.item(), batch_size)\n        if CFG['gradient_accumulation_steps'] > 1:\n            loss = loss / CFG['gradient_accumulation_steps']\n        if CFG['apex']:\n            with amp.scale_loss(loss, optimizer) as scaled_loss:\n                scaled_loss.backward()\n        else:\n            loss.backward()\n        # clear memory\n        del loss, y_preds\n        torch.cuda.empty_cache()\n        grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), CFG['max_grad_norm'])\n        if (step + 1) % CFG['gradient_accumulation_steps'] == 0:\n            optimizer.step()\n            optimizer.zero_grad()\n            global_step += 1\n        # measure elapsed time\n        batch_time.update(time.time() - end)\n        end = time.time()\n        if step % CFG['print_freq'] == 0 or step == (len(train_loader)-1):\n            print('Epoch: [{0}][{1}/{2}] '\n                  'Data {data_time.val:.3f} ({data_time.avg:.3f}) '\n                  'Elapsed {remain:s} '\n                  'Loss: {loss.val:.4f}({loss.avg:.4f}) '\n                  'Grad: {grad_norm:.4f}  '\n                  #'LR: {lr:.6f}  '\n                  .format(\n                   epoch+1, step, len(train_loader), batch_time=batch_time,\n                   data_time=data_time, loss=losses,\n                   remain=timeSince(start, float(step+1)/len(train_loader)),\n                   grad_norm=grad_norm,\n                   #lr=scheduler.get_lr()[0],\n                   ))\n    return losses.avg\n\n\ndef valid_fn(valid_loader, model, criterion, device):\n    batch_time = AverageMeter()\n    data_time = AverageMeter()\n    losses = AverageMeter()\n    scores = AverageMeter()\n    # switch to evaluation mode\n    model.eval()\n    preds = []\n    start = end = time.time()\n    for step, (features, labels) in enumerate(valid_loader):\n        # measure data loading time\n        data_time.update(time.time() - end)\n        features = features.to(device)\n        labels = labels.to(device)\n        batch_size = labels.size(0)\n        # compute loss\n        with torch.no_grad():\n            y_preds = model(features)\n        loss = get_loss(criterion, y_preds, labels)\n        losses.update(loss.item(), batch_size)\n        # record accuracy\n        preds.append(y_preds.softmax(1).to('cpu').numpy())\n        if CFG['gradient_accumulation_steps'] > 1:\n            loss = loss / CFG['gradient_accumulation_steps']\n        # measure elapsed time\n        batch_time.update(time.time() - end)\n        end = time.time()\n        if step % CFG['print_freq'] == 0 or step == (len(valid_loader)-1):\n            print('EVAL: [{0}/{1}] '\n                  'Data {data_time.val:.3f} ({data_time.avg:.3f}) '\n                  'Elapsed {remain:s} '\n                  'Loss: {loss.val:.4f}({loss.avg:.4f}) '\n                  .format(\n                   step, len(valid_loader), batch_time=batch_time,\n                   data_time=data_time, loss=losses,\n                   remain=timeSince(start, float(step+1)/len(valid_loader)),\n                   ))\n    predictions = np.concatenate(preds)\n    return losses.avg, predictions\n\n\ndef inference(model, states, test_loader, device):\n    model.to(device)\n    tk0 = tqdm(enumerate(test_loader), total=len(test_loader))\n    probs = []\n    for i, (features) in tk0:\n        features = features.to(device)\n        avg_preds = []\n        for state in states:\n            # model.load_state_dict(state['model'])\n            model.load_state_dict(state)\n            model.eval()\n            with torch.no_grad():\n                y_preds = model(features)\n            avg_preds.append(y_preds.softmax(1).to('cpu').numpy())\n        avg_preds = np.mean(avg_preds, axis=0)\n        probs.append(avg_preds)\n    probs = np.concatenate(probs)\n    return probs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# scheduler \n# ====================================================\ndef get_scheduler(optimizer):\n    if TAG['scheduler']=='ReduceLROnPlateau':\n        scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=CFG['factor'], patience=CFG['patience'], verbose=True, eps=CFG['eps'])\n    elif TAG['scheduler']=='CosineAnnealingLR':\n        scheduler = CosineAnnealingLR(optimizer, T_max=CFG['T_max'], eta_min=CFG['min_lr'], last_epoch=-1)\n    elif TAG['scheduler']=='CosineAnnealingWarmRestarts':\n        scheduler = CosineAnnealingWarmRestarts(optimizer, T_0=CFG['T_0'], T_mult=1, eta_min=CFG['min_lr'], last_epoch=-1)\n    return scheduler\n\n# ====================================================\n# criterion\n# ====================================================\ndef get_criterion():\n    if TAG['criterion']=='CrossEntropyLoss':\n        criterion = nn.CrossEntropyLoss()\n    elif TAG['criterion'] == 'bi_tempered_logistic_loss':\n        criterion = bi_tempered_logistic_loss\n    return criterion","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## training"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Train loop\n# ====================================================\ndef train_loop(X, y, folds, fold):\n\n    LOGGER.info(f\"========== fold: {fold} training ==========\")\n    if not CFG['debug']:\n        mlflow.set_tag('running.fold', str(fold))\n    # ====================================================\n    # loader\n    # ====================================================\n    trn_idx = folds[folds['fold'] != fold].index\n    val_idx = folds[folds['fold'] == fold].index\n\n    train_folds = folds.loc[trn_idx].reset_index(drop=True)\n    valid_folds = folds.loc[val_idx].reset_index(drop=True)\n    \n    X_train = X[trn_idx]\n    y_train = y[trn_idx]\n    X_valid = X[val_idx]\n    y_valid = y[val_idx]\n    \n    train_dataset = StackingDataset(X_train, y_train)\n    valid_dataset = StackingDataset(X_valid, y_valid)\n\n    train_loader = DataLoader(train_dataset, \n                              batch_size=CFG['batch_size'], \n                              shuffle=True, \n                              num_workers=CFG['num_workers'], pin_memory=True, drop_last=True)\n    valid_loader = DataLoader(valid_dataset, \n                              batch_size=CFG['batch_size'], \n                              shuffle=False, \n                              num_workers=CFG['num_workers'], pin_memory=True, drop_last=False)\n\n    # ====================================================\n    # model & optimizer & criterion\n    # ====================================================\n    best_model_path = OUTPUT_DIR+f'fold{fold}_best.pth'\n    latest_model_path = OUTPUT_DIR+f'fold{fold}_latest.pth'\n\n    model = CNNStacking(CFG['target_size'])\n    model.to(device)\n    # 学習途中の重みがあれば読み込み\n    if os.path.isfile(latest_model_path):\n        state_latest = torch.load(latest_model_path)\n        state_best = torch.load(best_model_path)\n        model.load_state_dict(state_latest['model'])\n        epoch_start = state_latest['epoch']+1\n        # er_best_score = state_latest['score']\n        er_counter = state_latest['counter']\n        er_best_score = state_best['best_score']\n        val_loss_history = state_latest['val_loss_history']\n\n        LOGGER.info(f'Load training model in epoch:{epoch_start}, best_score:{er_best_score:.3f}, counter:{er_counter}')\n\n    # 学習済みモデルを再学習する場合\n    elif os.path.isfile(best_model_path):\n        state_best = torch.load(best_model_path)\n        model.load_state_dict(state_best['model'])\n        epoch_start = 0 # epochは0からカウントしなおす\n        er_counter = 0\n        er_best_score = state_best['best_score']\n        val_loss_history = []   # 過去のval_lossも使用しない\n\n        LOGGER.info(f'Retrain model, best_score:{er_best_score:.3f}')\n    else:\n        epoch_start = 0\n        er_best_score = None\n        er_counter = 0\n        val_loss_history = []\n\n    optimizer = Adam(model.parameters(), lr=CFG['lr'], weight_decay=CFG['weight_decay'], amsgrad=False)\n    scheduler = get_scheduler(optimizer)\n    criterion = get_criterion()\n\n    # 再開時のepochまでschedulerを進める\n    assert len(range(epoch_start)) == len(val_loss_history)\n    for _, val_loss in zip(range(epoch_start), val_loss_history):\n        if isinstance(scheduler, ReduceLROnPlateau):\n            scheduler.step(val_loss)\n        elif isinstance(scheduler, CosineAnnealingLR):\n            scheduler.step()\n        elif isinstance(scheduler, CosineAnnealingWarmRestarts):\n            scheduler.step()\n\n    # ====================================================\n    # apex\n    # ====================================================\n    if CFG['apex']:\n        model, optimizer = amp.initialize(model, optimizer, opt_level='O1', verbosity=0)\n\n    # ====================================================\n    # loop\n    # ====================================================\n    # best_score = 0.\n    # best_loss = np.inf\n    early_stopping = EarlyStopping(\n                            patience=CFG['early_stopping_round'], \n                            verbose=True,\n                            save_path=best_model_path,\n                            counter=er_counter, best_score=er_best_score, \n                            save_latest_path=latest_model_path)\n    \n    for epoch in range(epoch_start, CFG['epochs']):\n        \n        start_time = time.time()\n        \n        # train\n        avg_loss = train_fn(train_loader, model, criterion, optimizer, epoch, scheduler, device)\n\n        # eval\n        avg_val_loss, preds = valid_fn(valid_loader, model, criterion, device)\n        valid_labels = valid_folds[CFG['target_col']].values\n\n        # scoring\n        score = get_score(valid_labels, preds.argmax(1))\n\n        # get learning rate\n        if hasattr(scheduler, 'get_last_lr'):\n            last_lr = scheduler.get_last_lr()[0]\n        else:\n            # ReduceLROnPlateauには関数get_last_lrがない\n            last_lr = optimizer.param_groups[0]['lr']\n        \n        # log mlflow\n        if not CFG['debug']:\n            mlflow.log_metric(f\"fold{fold} avg_train_loss\", avg_loss, step=epoch)\n            mlflow.log_metric(f\"fold{fold} avg_valid_loss\", avg_val_loss, step=epoch)\n            mlflow.log_metric(f\"fold{fold} score\", score, step=epoch)\n            mlflow.log_metric(f\"fold{fold} lr\", last_lr, step=epoch)\n        \n        # early stopping\n        early_stopping(avg_val_loss, model, preds, epoch)\n        if early_stopping.early_stop:\n            print(f'Epoch {epoch+1} - early stopping')\n            break\n        \n        if isinstance(scheduler, ReduceLROnPlateau):\n            scheduler.step(avg_val_loss)\n        elif isinstance(scheduler, CosineAnnealingLR):\n            scheduler.step()\n        elif isinstance(scheduler, CosineAnnealingWarmRestarts):\n            scheduler.step()\n\n        elapsed = time.time() - start_time\n\n        LOGGER.info(f'Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f}  avg_val_loss: {avg_val_loss:.4f}  time: {elapsed:.0f}s')\n        LOGGER.info(f'Epoch {epoch+1} - Accuracy: {score}')\n        \n        # log mlflow\n        if not CFG['debug']:\n            mlflow.log_artifact(best_model_path)\n            if os.path.isfile(latest_model_path):\n                mlflow.log_artifact(latest_model_path)\n    \n    check_point = torch.load(best_model_path)\n    valid_folds[[str(c) for c in range(5)]] = check_point['preds']\n    valid_folds['preds'] = check_point['preds'].argmax(1)\n\n    return valid_folds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_trained_fold_preds(folds, fold, best_model_path):\n    valid_folds = folds.loc[val_idx].reset_index(drop=True)\n    check_point = torch.load(best_model_path)\n    valid_folds[[str(c) for c in range(5)]] = check_point['preds']\n    valid_folds['preds'] = check_point['preds'].argmax(1)\n\n    return valid_folds\n\n\ndef save_confusion_matrix(oof):\n    from sklearn.metrics import confusion_matrix\n    cm_ = confusion_matrix(oof['label'], oof['preds'], labels=[0,1,2,3,4])\n    label_name = ['0 (CBB)', '1 (CBSD)', '2 (CGM)', '3 (CMD)', '4 (Healthy)']\n    cm = pd.DataFrame(cm_, index=label_name, columns=label_name)\n    cm.to_csv(OUTPUT_DIR+'oof_confusion_matrix.csv', index=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# main\n# ====================================================\ndef get_result(result_df):\n    preds = result_df['preds'].values\n    labels = result_df[CFG['target_col']].values\n    score = get_score(labels, preds)\n    LOGGER.info(f'Score: {score:<.5f}')\n    \n    return score\n        \n        \ndef get_oof_list(oof_dirs):\n    check_cols = [\"image_id\", \"label\", \"fold\"]\n\n    df_ = None\n    oof_list = []\n    for oof_dir in oof_dirs:\n        tta_preds = glob.glob(os.path.join(oof_dir, \"*[!submission].csv\"))\n        tta_dict = {}\n        for tta_pred in tta_preds:\n            tta_name = tta_pred.split('/')[-1].split('.')[0]\n            df = pd.read_csv(tta_pred)\n            # 全てのoofでfoldが揃っているかチェック\n            if df_ is not None:\n                assert (df[check_cols] == df_[check_cols]).all().all(), tta_name\n            df_ = df.copy()\n            tta_dict[tta_name] = df\n        oof_list.append(tta_dict)\n        \n    return oof_list\n\n    \n\ndef main():\n\n    oof_list = get_oof_list(oof_dirs)\n    first_df = oof_list[0][TTA_LIST[0]]\n    \n    data_num = len(first_df)\n    model_num = len(oof_dirs)\n    target_num = CFG['target_size']\n    channel_num = len(TTA_LIST)\n    \n    # [N, Models, Labels, Channel]\n    X = np.zeros((data_num, model_num, target_num, channel_num), dtype=np.float)\n    y = first_df['label'].values\n    folds = first_df[['image_id', 'label', 'fold']]\n    for model_idx, oof in enumerate(oof_list):\n        for channel_idx, tta_name in enumerate(TTA_LIST):\n            X[:, model_idx, :, channel_idx] = oof[tta_name][['0','1','2','3','4']].values\n    \n    # [N, Models, Labels, Channel] -> [N, Channel, Models, Labels]\n    X = X.transpose(0, 3, 1, 2)\n    \n    if CFG['train']:\n        # train \n        oof_df = pd.DataFrame()\n        for fold in range(CFG['n_fold']):\n            best_model_path = OUTPUT_DIR+f'fold{fold}_best.pth'\n            if fold in CFG['trn_fold']:\n                _oof_df = train_loop(X, y, folds, fold)\n            elif os.path.exists(best_model_path):\n                _oof_df = get_trained_fold_preds(folds, fold, best_model_path)\n            else:\n                _oof_df = None\n            if _oof_df is not None:\n                oof_df = pd.concat([oof_df, _oof_df])\n                LOGGER.info(f\"========== fold: {fold} result ==========\")\n                _ = get_result(_oof_df)\n        # CV result\n        LOGGER.info(f\"========== CV ==========\")\n        score = get_result(oof_df)\n        # save result\n        oof_df.to_csv(OUTPUT_DIR+'oof_df.csv', index=False)\n        save_confusion_matrix(oof_df)\n        # save tta\n        pd.Series(TTA_LIST, name=\"tta_list\").to_csv(OUTPUT_DIR+'tta_list.csv', index=False)\n        # log mlflow\n        if not CFG['debug']:\n            mlflow.log_metric('oof score', score)\n            mlflow.delete_tag('running.fold')\n            mlflow.log_artifact(OUTPUT_DIR+'oof_df.csv')\n            mlflow.log_artifact(OUTPUT_DIR+'tta_list.csv')\n    \n    if CFG['inference']:\n        pass\n#         # inference\n#         model = CustomModel(TAG['model_name'], pretrained=False)\n#         states = [torch.load(OUTPUT_DIR+f'{TAG[\"model_name\"]}_fold{fold}_best.pth') for fold in CFG['trn_fold']]\n#         test_dataset = TestDataset(test, transform=get_transforms(data='valid'))\n#         test_loader = DataLoader(test_dataset, batch_size=CFG['batch_size'], shuffle=False, \n#                                  num_workers=CFG['num_workers'], pin_memory=True)\n#         predictions = inference(model, states, test_loader, device)\n#         # submission\n#         test['label'] = predictions.argmax(1)\n#         test[['image_id', 'label']].to_csv(OUTPUT_DIR+'submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _load_save_point(run_id):\n    # どこで中断したか取得\n    stop_fold = int(mlflow.get_run(run_id=run_id).to_dictionary()['data']['tags']['running.fold'])\n    # 学習対象のfoldを変更\n    CFG['trn_fold'] = [fold for fold in CFG['trn_fold'] if fold>=stop_fold]\n    # 学習済みモデルがあれば.pthファイルを取得(学習中も含む)\n    client = mlflow.tracking.MlflowClient()\n    artifacts = [artifact for artifact in client.list_artifacts(run_id) if \".pth\" in artifact.path]\n    for artifact in artifacts:\n        client.download_artifacts(run_id, artifact.path, OUTPUT_DIR)\n\n\ndef check_have_run():\n    results = mlflow.search_runs(INFO['EXPERIMENT_ID'])\n    if 'tags.mlflow.runName' in results.columns:\n        run_id_list = results[results['tags.mlflow.runName']==TITLE]['run_id'].tolist()\n    else:\n        print(f'No results in experiment_id=={INFO[\"EXPERIMENT_ID\"]}')\n        run_id_list = []\n    # 初めて実行する場合\n    if len(run_id_list) == 0:\n        run_id = None\n    # 既に実行されている場合\n    else:\n        assert len(run_id_list)==1\n        run_id = run_id_list[0]\n        _load_save_point(run_id)\n\n    return run_id\n\n\ndef push_github():\n    ! cp {NOTEBOOK_PATH} kaggle-cassava/notebook/{TITLE}.ipynb\n    !git config --global user.email \"raijin.1059@gmail.com\"\n    ! git config --global user.name \"Raijin Shibata\"\n    !cd kaggle-cassava ;git add .; git commit -m {TITLE}; git remote set-url origin https://{user_name}:{password}@github.com/raijin0704/kaggle-cassava.git; git push origin master","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if __name__ == '__main__':\n    if CFG['debug']:\n        main()\n    else:\n        mlflow.set_tracking_uri(INFO['TRACKING_URI'])\n        mlflow.set_experiment('stacking')\n        # 既に実行済みの場合は続きから実行する\n        run_id = check_have_run()\n        with mlflow.start_run(run_id=run_id, run_name=TITLE):\n            if run_id is None:\n                mlflow.log_artifact(CONFIG_PATH)\n                mlflow.log_param('device', device)\n                mlflow.set_tag('env', env)\n                mlflow.set_tags(TAG)\n                mlflow.log_params(CFG)\n            mlflow.log_artifact(notebook_path)\n            main()\n            mlflow.log_artifacts(OUTPUT_DIR)\n            remove_glob(f'{OUTPUT_DIR}/*latest.pth')\n            push_github()\n            if env==\"kaggle\":\n                shutil.copy2(CONFIG_PATH, f'{OUTPUT_DIR}/{CONFIG_NAME}')\n                ! rm -r kaggle-cassava\n            elif env==\"colab\":\n                shutil.copytree(OUTPUT_DIR, f'{INFO[\"SHARE_DRIVE_PATH\"]}/{TITLE}')\n                shutil.copy2(CONFIG_PATH, f'{INFO[\"SHARE_DRIVE_PATH\"]}/{TITLE}/{CONFIG_NAME}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%debug","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}