{"cells":[{"metadata":{},"cell_type":"markdown","source":"# About this notebook\nI share an example how to use annotated images to improve score.  \nIf this notebook is helpful, feel free to upvote :)\n## Training strategy\n- [1st-stage training](https://www.kaggle.com/yasufuminakama/ranzcr-resnet200d-3-stage-training-step1)\n    - teacher model training for annotated image\n        - data: annotated data\n        - pretrained weight: imagenet weight\n        - `BCEWithLogitsLoss(y_preds, labels)`\n        - `y_preds: teacher model predictions for annotated image`\n- [2nd-stage training](https://www.kaggle.com/yasufuminakama/ranzcr-resnet200d-3-stage-training-step2)\n    - student model training with teacher model features\n        - data: annotated data\n        - student model pretrained weight: imagenet weight\n        - teacher model pretrained weight: 1st-stage weight\n        - `BCEWithLogitsLoss(y_preds, labels) + w * MSELoss(student_features, teacher_features)`\n        - `y_preds: student model predictions for normal image`\n        - `student_features: student model features for normal image`\n        - `teacher_features: teacher model features for annotated image`\n- [3rd-stage training](https://www.kaggle.com/yasufuminakama/ranzcr-resnet200d-3-stage-training-step3)\n    - model training\n        - data: all data\n        - pretrained weight: 2nd-stage weight\n        - `BCEWithLogitsLoss(y_preds, labels)`\n        - `y_preds: student model predictions for normal image`\n- [inference notebook](https://www.kaggle.com/yasufuminakama/ranzcr-resnet200d-3-stage-training-sub)"},{"metadata":{"papermill":{"duration":0.027734,"end_time":"2020-12-23T19:04:31.666166","exception":false,"start_time":"2020-12-23T19:04:31.638432","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Directory settings"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-23T19:04:31.731971Z","iopub.status.busy":"2020-12-23T19:04:31.730987Z","iopub.status.idle":"2020-12-23T19:04:31.734248Z","shell.execute_reply":"2020-12-23T19:04:31.733486Z"},"papermill":{"duration":0.039089,"end_time":"2020-12-23T19:04:31.734384","exception":false,"start_time":"2020-12-23T19:04:31.695295","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# ====================================================\n# Directory settings\n# ====================================================\nimport os\n\nOUTPUT_DIR = './'\nif not os.path.exists(OUTPUT_DIR):\n    os.makedirs(OUTPUT_DIR)\n\nTRAIN_PATH = '../input/ranzcr-clip-catheter-line-classification/train'","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.027722,"end_time":"2020-12-23T19:04:31.792603","exception":false,"start_time":"2020-12-23T19:04:31.764881","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# CFG"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-23T19:04:31.864601Z","iopub.status.busy":"2020-12-23T19:04:31.863568Z","iopub.status.idle":"2020-12-23T19:04:31.867028Z","shell.execute_reply":"2020-12-23T19:04:31.866253Z"},"papermill":{"duration":0.045622,"end_time":"2020-12-23T19:04:31.867167","exception":false,"start_time":"2020-12-23T19:04:31.821545","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# ====================================================\n# CFG\n# ====================================================\nclass CFG:\n    debug=False\n    device='TPU' # ['TPU', 'GPU']\n    nprocs=1 # [1, 8]\n    print_freq=100\n    num_workers=4\n    model_name='resnet200d_320'\n    size=640\n    scheduler='CosineAnnealingLR' # ['ReduceLROnPlateau', 'CosineAnnealingLR', 'CosineAnnealingWarmRestarts']\n    epochs=5\n    #factor=0.2 # ReduceLROnPlateau\n    #patience=4 # ReduceLROnPlateau\n    #eps=1e-6 # ReduceLROnPlateau\n    T_max=4 # CosineAnnealingLR\n    #T_0=4 # CosineAnnealingWarmRestarts\n    lr=5e-4 # 1e-4\n    min_lr=1e-6\n    batch_size=16 # 64\n    weight_decay=1e-6\n    gradient_accumulation_steps=1\n    max_grad_norm=1000\n    seed=416\n    target_size=11\n    target_cols=['ETT - Abnormal', 'ETT - Borderline', 'ETT - Normal',\n                 'NGT - Abnormal', 'NGT - Borderline', 'NGT - Incompletely Imaged', 'NGT - Normal', \n                 'CVC - Abnormal', 'CVC - Borderline', 'CVC - Normal',\n                 'Swan Ganz Catheter Present']\n    n_fold=5\n    trn_fold= [2]  #[0, 1, 2, 3, 4]\n    train=True\n    \nif CFG.debug:\n    CFG.epochs = 1","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-23T19:04:31.938548Z","iopub.status.busy":"2020-12-23T19:04:31.937742Z","iopub.status.idle":"2020-12-23T19:06:15.173769Z","shell.execute_reply":"2020-12-23T19:06:15.172808Z"},"papermill":{"duration":103.277198,"end_time":"2020-12-23T19:06:15.173939","exception":false,"start_time":"2020-12-23T19:04:31.896741","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"if CFG.device == 'TPU':\n    import os\n    os.system('curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py')\n    os.system('python pytorch-xla-env-setup.py --version nightly --apt-packages libomp5 libopenblas-dev')\n    os.system('export XLA_USE_BF16=1')\n    import torch_xla.core.xla_model as xm\n    import torch_xla.distributed.parallel_loader as pl\n    import torch_xla.distributed.xla_multiprocessing as xmp\n    CFG.lr = CFG.lr * CFG.nprocs\n    CFG.batch_size = CFG.batch_size // CFG.nprocs","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.031204,"end_time":"2020-12-23T19:06:15.235441","exception":false,"start_time":"2020-12-23T19:06:15.204237","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Library"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-23T19:06:15.331972Z","iopub.status.busy":"2020-12-23T19:06:15.33012Z","iopub.status.idle":"2020-12-23T19:06:17.540544Z","shell.execute_reply":"2020-12-23T19:06:17.53983Z"},"papermill":{"duration":2.270679,"end_time":"2020-12-23T19:06:17.540705","exception":false,"start_time":"2020-12-23T19:06:15.270026","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\nimport sys\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\n\nimport os\nimport ast\nimport copy\nimport math\nimport time\nimport random\nimport shutil\nfrom pathlib import Path\nfrom contextlib import contextmanager\nfrom collections import defaultdict, Counter\n\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn import preprocessing\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.utils import check_random_state\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\n\nfrom tqdm.auto import tqdm\nfrom functools import partial\n\nimport cv2\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD\nimport torchvision.models as models\nfrom torch.nn.parameter import Parameter\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau\n\nfrom albumentations import (\n    Compose, OneOf, Normalize, Resize, RandomResizedCrop, RandomCrop, HorizontalFlip, VerticalFlip, \n    RandomBrightness, RandomContrast, RandomBrightnessContrast, Rotate, ShiftScaleRotate, Cutout, \n    IAAAdditiveGaussianNoise, Transpose, HueSaturationValue, CoarseDropout\n    )\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform\n\nimport timm\n\nif CFG.device == 'TPU':\n    import ignite.distributed as idist\nelif CFG.device == 'GPU':\n    from torch.cuda.amp import autocast, GradScaler\n\nimport warnings \nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.028493,"end_time":"2020-12-23T19:06:17.598011","exception":false,"start_time":"2020-12-23T19:06:17.569518","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Utils"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-23T19:06:17.675794Z","iopub.status.busy":"2020-12-23T19:06:17.674877Z","iopub.status.idle":"2020-12-23T19:06:17.681689Z","shell.execute_reply":"2020-12-23T19:06:17.680811Z"},"papermill":{"duration":0.055059,"end_time":"2020-12-23T19:06:17.681837","exception":false,"start_time":"2020-12-23T19:06:17.626778","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# ====================================================\n# Utils\n# ====================================================\ndef get_score(y_true, y_pred):\n    scores = []\n    for i in range(y_true.shape[1]):\n        score = roc_auc_score(y_true[:,i], y_pred[:,i])\n        scores.append(score)\n    avg_score = np.mean(scores)\n    return avg_score, scores\n\n\n@contextmanager\ndef timer(name):\n    t0 = time.time()\n    LOGGER.info(f'[{name}] start')\n    yield\n    LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s.')\n\n\ndef init_logger(log_file=OUTPUT_DIR+'train.log'):\n    from logging import getLogger, INFO, FileHandler,  Formatter,  StreamHandler\n    logger = getLogger(__name__)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=log_file)\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    return logger\n\nLOGGER = init_logger()\n\n\ndef seed_torch(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_torch(seed=CFG.seed)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Loading"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/ranzcr-clip-catheter-line-classification/train.csv')\nfolds = pd.read_csv('../input/ranzcr-folds/train_folds.csv')\ntrain_annotations = pd.read_csv('../input/ranzcr-clip-catheter-line-classification/train_annotations.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"folds=folds.rename(columns = {'kfold': 'fold'})\nfolds.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.57005638787237813934531972491254580369', 'CVC - Borderline'] = 0\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.57005638787237813934531972491254580369', 'NGT - Borderline'] = 1\n\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.93345761486297843389996628528592497280', 'ETT - Abnormal'] = 0\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.93345761486297843389996628528592497280', 'CVC - Abnormal'] = 1\n\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.50891603479257167332052859560303996365', 'NGT - Normal'] = 0\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.50891603479257167332052859560303996365', 'CVC - Normal'] = 1\n\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.32665013930528750130301395098139968929', 'NGT - Borderline'] = 0\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.32665013930528750130301395098139968929', 'CVC - Borderline'] = 1\n\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.47822809495672253227315400926882161159', 'NGT - Borderline'] = 0\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.47822809495672253227315400926882161159', 'CVC - Borderline'] = 1\n\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.55171965195784371324650309161724846475', 'NGT - Borderline'] = 0\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.55171965195784371324650309161724846475', 'CVC - Borderline'] = 1\n\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.29639870594803047496855371142714987539', 'ETT - Normal'] = 0\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.29639870594803047496855371142714987539', 'CVC - Normal'] = 1\n\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.52422864792637441690285442425747003963', 'NGT - Normal'] = 0\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.52422864792637441690285442425747003963', 'ETT - Normal'] = 1\n\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.51277351337858188519077141427236143108', 'NGT - Normal'] = 0\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.51277351337858188519077141427236143108', 'CVC - Normal'] = 1\n\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.33011244702337270174558484639492100815', 'CVC - Normal'] = 0\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.33011244702337270174558484639492100815', 'NGT - Normal'] = 1\n\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.10505287747515183956922280117689383476', 'NGT - Normal'] = 0\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.10505287747515183956922280117689383476', 'CVC - Normal'] = 1\n\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.43340424479611237895060478106689360500', 'NGT - Normal'] = 0\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.43340424479611237895060478106689360500', 'CVC - Normal'] = 1\n\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.12545979153892772426852721449004507757', 'NGT - Abnormal'] = 0\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.12545979153892772426852721449004507757', 'CVC - Abnormal'] = 1\n\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.83700037297895094021306651705503600111', 'NGT - Normal'] = 0\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.83700037297895094021306651705503600111', 'ETT - Normal'] = 1\n\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.35772244095675958072394978496245125294', 'NGT - Normal'] = 0\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.35772244095675958072394978496245125294', 'ETT - Normal'] = 1\n\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.96130195933728659348647733812659169362', 'CVC - Abnormal'] = 0\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.96130195933728659348647733812659169362', 'NGT - Abnormal'] = 1\n\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.75269816256944932004789976844599885553', 'NGT - Abnormal'] = 0\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.75269816256944932004789976844599885553', 'CVC - Abnormal'] = 1\n\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.11935284122896798228836385959451625327', 'NGT - Abnormal'] = 0\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.11935284122896798228836385959451625327', 'CVC - Abnormal'] = 1\n\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.83574817573978660270935463700320068005', 'NGT - Abnormal'] = 0\ntrain.loc[train.StudyInstanceUID=='1.2.826.0.1.3680043.8.498.83574817573978660270935463700320068005', 'CVC - Abnormal'] = 1\n\ntrain_annotations.loc[4344, 'label'] = 'CVC - Abnormal'\ntrain_annotations.loc[3589, 'label'] = 'NGT - Borderline'\ntrain_annotations.loc[6294, 'label'] = 'CVC - Normal'\ntrain_annotations.loc[7558, 'label'] = 'CVC - Borderline'\ntrain_annotations.loc[8457, 'label'] = 'CVC - Borderline'\ntrain_annotations.loc[8586, 'label'] = 'CVC - Borderline'\ntrain_annotations.loc[8589, 'label'] = 'CVC - Normal'\ntrain_annotations.loc[9908, 'label'] = 'ETT - Normal'\ntrain_annotations.loc[10889, 'label'] = 'CVC - Normal'\ntrain_annotations.loc[10963, 'label'] = 'NGT - Normal'\ntrain_annotations.loc[11902, 'label'] = 'CVC - Normal'\ntrain_annotations.loc[12041, 'label'] = 'CVC - Normal'\ntrain_annotations.loc[12782, 'label'] = 'CVC - Abnormal'\ntrain_annotations.loc[13513, 'label'] = 'ETT - Normal'\ntrain_annotations.loc[14226, 'label'] = 'ETT - Normal'\ntrain_annotations.loc[15750, 'label'] = 'NGT - Abnormal'\ntrain_annotations.loc[15779, 'label'] = 'CVC - Abnormal'\ntrain_annotations.loc[16629, 'label'] = 'CVC - Abnormal'\ntrain_annotations.loc[17501, 'label'] = 'CVC - Abnormal'","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.030724,"end_time":"2020-12-23T19:06:18.847705","exception":false,"start_time":"2020-12-23T19:06:18.816981","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Dataset\n# ====================================================\nCOLOR_MAP = {'ETT - Abnormal': (255, 0, 0),\n             'ETT - Borderline': (0, 255, 0),\n             'ETT - Normal': (0, 0, 255),\n             'NGT - Abnormal': (255, 255, 0),\n             'NGT - Borderline': (255, 0, 255),\n             'NGT - Incompletely Imaged': (0, 255, 255),\n             'NGT - Normal': (128, 0, 0),\n             'CVC - Abnormal': (0, 128, 0),\n             'CVC - Borderline': (0, 0, 128),\n             'CVC - Normal': (128, 128, 0),\n             'Swan Ganz Catheter Present': (128, 0, 128),\n            }\n\n\nclass TrainDataset(Dataset):\n    def __init__(self, df, df_annotations, annot_size=50, transform=None):\n        self.df = df\n        self.df_annotations = df_annotations\n        self.annot_size = annot_size\n        self.file_names = df['StudyInstanceUID'].values\n        self.labels = df[CFG.target_cols].values\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{TRAIN_PATH}/{file_name}.jpg'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        query_string = f\"StudyInstanceUID == '{file_name}'\"\n        df = self.df_annotations.query(query_string)\n        for i, row in df.iterrows():\n            label = row[\"label\"]\n            data = np.array(ast.literal_eval(row[\"data\"]))\n            for d in data:\n                image[d[1]-self.annot_size//2:d[1]+self.annot_size//2,\n                      d[0]-self.annot_size//2:d[0]+self.annot_size//2,\n                      :] = COLOR_MAP[label]\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        label = torch.tensor(self.labels[idx]).float()\n        return image, label","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.031645,"end_time":"2020-12-23T19:06:19.086874","exception":false,"start_time":"2020-12-23T19:06:19.055229","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Transforms"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-23T19:06:19.163947Z","iopub.status.busy":"2020-12-23T19:06:19.161721Z","iopub.status.idle":"2020-12-23T19:06:19.167297Z","shell.execute_reply":"2020-12-23T19:06:19.168067Z"},"papermill":{"duration":0.048275,"end_time":"2020-12-23T19:06:19.168252","exception":false,"start_time":"2020-12-23T19:06:19.119977","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# ====================================================\n# Transforms\n# ====================================================\ndef get_transforms(*, data):\n    \n    if data == 'train':\n        return Compose([\n            #Resize(CFG.size, CFG.size),\n            RandomResizedCrop(CFG.size, CFG.size, scale=(0.85, 1.0)),\n            HorizontalFlip(p=0.5),\n            RandomBrightnessContrast(p=0.2, brightness_limit=(-0.2, 0.2), contrast_limit=(-0.2, 0.2)),\n            HueSaturationValue(p=0.2, hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2),\n            ShiftScaleRotate(p=0.2, shift_limit=0.0625, scale_limit=0.2, rotate_limit=20),\n            CoarseDropout(p=0.2),\n            Cutout(p=0.2, max_h_size=16, max_w_size=16, fill_value=(0., 0., 0.), num_holes=16),\n            Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n                p=1.0\n            ),\n            ToTensorV2(),\n        ])\n\n    elif data == 'valid':\n        return Compose([\n            Resize(CFG.size, CFG.size),\n            Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n                p=1.0\n            ),\n            ToTensorV2(),\n        ])","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-23T19:06:19.244888Z","iopub.status.busy":"2020-12-23T19:06:19.238449Z","iopub.status.idle":"2020-12-23T19:06:20.905143Z","shell.execute_reply":"2020-12-23T19:06:20.906145Z"},"papermill":{"duration":1.705488,"end_time":"2020-12-23T19:06:20.906438","exception":false,"start_time":"2020-12-23T19:06:19.20095","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from matplotlib import pyplot as plt\n\ntrain_dataset = TrainDataset(folds[folds['StudyInstanceUID'].isin(train_annotations['StudyInstanceUID'].unique())].reset_index(drop=True),\n                             train_annotations, transform=None)\n\nfor i in range(5):\n    image, label = train_dataset[i]\n    plt.imshow(image)\n    plt.title(f'label: {label}')\n    plt.show() ","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.062385,"end_time":"2020-12-23T19:06:21.079214","exception":false,"start_time":"2020-12-23T19:06:21.016829","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# MODEL"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-23T19:06:21.216578Z","iopub.status.busy":"2020-12-23T19:06:21.215376Z","iopub.status.idle":"2020-12-23T19:06:21.225711Z","shell.execute_reply":"2020-12-23T19:06:21.226511Z"},"papermill":{"duration":0.088216,"end_time":"2020-12-23T19:06:21.226699","exception":false,"start_time":"2020-12-23T19:06:21.138483","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# ====================================================\n# MODEL\n# ====================================================\nclass CustomResNet200D(nn.Module):\n    def __init__(self, model_name='resnet200d_320', pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=False)\n        if pretrained:\n            pretrained_path = '../input/resnet200d-pretrained-weight/resnet200d_ra2-bdba9bf9.pth'\n            self.model.load_state_dict(torch.load(pretrained_path))\n            print(f'load {model_name} pretrained model')\n        n_features = self.model.fc.in_features\n        self.model.global_pool = nn.Identity()\n        self.model.fc = nn.Identity()\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Linear(n_features, CFG.target_size)\n\n    def forward(self, x):\n        bs = x.size(0)\n        features = self.model(x)\n        pooled_features = self.pooling(features).view(bs, -1)\n        output = self.fc(pooled_features)\n        return features, pooled_features, output","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.063942,"end_time":"2020-12-23T19:06:21.347849","exception":false,"start_time":"2020-12-23T19:06:21.283907","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Helper functions"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-23T19:06:21.467293Z","iopub.status.busy":"2020-12-23T19:06:21.465046Z","iopub.status.idle":"2020-12-23T19:06:21.543666Z","shell.execute_reply":"2020-12-23T19:06:21.544562Z"},"papermill":{"duration":0.141122,"end_time":"2020-12-23T19:06:21.5448","exception":false,"start_time":"2020-12-23T19:06:21.403678","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# ====================================================\n# Helper functions\n# ====================================================\nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\n\ndef asMinutes(s):\n    m = math.floor(s / 60)\n    s -= m * 60\n    return '%dm %ds' % (m, s)\n\n\ndef timeSince(since, percent):\n    now = time.time()\n    s = now - since\n    es = s / (percent)\n    rs = es - s\n    return '%s (remain %s)' % (asMinutes(s), asMinutes(rs))\n\n\ndef train_fn(train_loader, model, criterion, optimizer, epoch, scheduler, device):\n    if CFG.device == 'GPU':\n        scaler = GradScaler()\n    batch_time = AverageMeter()\n    data_time = AverageMeter()\n    losses = AverageMeter()\n    scores = AverageMeter()\n    # switch to train mode\n    model.train()\n    start = end = time.time()\n    global_step = 0\n    for step, (images, labels) in enumerate(train_loader):\n        # measure data loading time\n        data_time.update(time.time() - end)\n        images = images.to(device)\n        labels = labels.to(device)\n        batch_size = labels.size(0)\n        if CFG.device == 'GPU':\n            with autocast():\n                _, _, y_preds = model(images)\n                loss = criterion(y_preds, labels)\n                # record loss\n                losses.update(loss.item(), batch_size)\n                if CFG.gradient_accumulation_steps > 1:\n                    loss = loss / CFG.gradient_accumulation_steps\n                scaler.scale(loss).backward()\n                grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), CFG.max_grad_norm)\n                if (step + 1) % CFG.gradient_accumulation_steps == 0:\n                    scaler.step(optimizer)\n                    scaler.update()\n                    optimizer.zero_grad()\n                    global_step += 1\n        elif CFG.device == 'TPU':\n            _, _, y_preds = model(images)\n            loss = criterion(y_preds, labels)\n            # record loss\n            losses.update(loss.item(), batch_size)\n            if CFG.gradient_accumulation_steps > 1:\n                loss = loss / CFG.gradient_accumulation_steps\n            loss.backward()\n            grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), CFG.max_grad_norm)\n            if (step + 1) % CFG.gradient_accumulation_steps == 0:\n                xm.optimizer_step(optimizer, barrier=True)\n                optimizer.zero_grad()\n                global_step += 1\n        # measure elapsed time\n        batch_time.update(time.time() - end)\n        end = time.time()\n        if CFG.device == 'GPU':\n            if step % CFG.print_freq == 0 or step == (len(train_loader)-1):\n                print('Epoch: [{0}][{1}/{2}] '\n                      'Data {data_time.val:.3f} ({data_time.avg:.3f}) '\n                      'Elapsed {remain:s} '\n                      'Loss: {loss.val:.4f}({loss.avg:.4f}) '\n                      'Grad: {grad_norm:.4f}  '\n                      #'LR: {lr:.6f}  '\n                      .format(\n                       epoch+1, step, len(train_loader), batch_time=batch_time,\n                       data_time=data_time, loss=losses,\n                       remain=timeSince(start, float(step+1)/len(train_loader)),\n                       grad_norm=grad_norm,\n                       #lr=scheduler.get_lr()[0],\n                       ))\n        elif CFG.device == 'TPU':\n            if step % CFG.print_freq == 0 or step == (len(train_loader)-1):\n                xm.master_print('Epoch: [{0}][{1}/{2}] '\n                                'Data {data_time.val:.3f} ({data_time.avg:.3f}) '\n                                'Elapsed {remain:s} '\n                                'Loss: {loss.val:.4f}({loss.avg:.4f}) '\n                                'Grad: {grad_norm:.4f}  '\n                                #'LR: {lr:.6f}  '\n                                .format(\n                                epoch+1, step, len(train_loader), batch_time=batch_time,\n                                data_time=data_time, loss=losses,\n                                remain=timeSince(start, float(step+1)/len(train_loader)),\n                                grad_norm=grad_norm,\n                                #lr=scheduler.get_lr()[0],\n                                ))\n    return losses.avg\n\n\ndef valid_fn(valid_loader, model, criterion, device):\n    batch_time = AverageMeter()\n    data_time = AverageMeter()\n    losses = AverageMeter()\n    scores = AverageMeter()\n    # switch to evaluation mode\n    model.eval()\n    trues = []\n    preds = []\n    start = end = time.time()\n    for step, (images, labels) in enumerate(valid_loader):\n        # measure data loading time\n        data_time.update(time.time() - end)\n        images = images.to(device)\n        labels = labels.to(device)\n        batch_size = labels.size(0)\n        # compute loss\n        with torch.no_grad():\n            _, _, y_preds = model(images)\n        loss = criterion(y_preds, labels)\n        losses.update(loss.item(), batch_size)\n        # record accuracy\n        trues.append(labels.to('cpu').numpy())\n        preds.append(y_preds.sigmoid().to('cpu').numpy())\n        if CFG.gradient_accumulation_steps > 1:\n            loss = loss / CFG.gradient_accumulation_steps\n        # measure elapsed time\n        batch_time.update(time.time() - end)\n        end = time.time()\n        if CFG.device == 'GPU':\n            if step % CFG.print_freq == 0 or step == (len(valid_loader)-1):\n                print('EVAL: [{0}/{1}] '\n                      'Data {data_time.val:.3f} ({data_time.avg:.3f}) '\n                      'Elapsed {remain:s} '\n                      'Loss: {loss.val:.4f}({loss.avg:.4f}) '\n                      .format(\n                       step, len(valid_loader), batch_time=batch_time,\n                       data_time=data_time, loss=losses,\n                       remain=timeSince(start, float(step+1)/len(valid_loader)),\n                       ))\n        elif CFG.device == 'TPU':\n            if step % CFG.print_freq == 0 or step == (len(valid_loader)-1):\n                xm.master_print('EVAL: [{0}/{1}] '\n                                'Data {data_time.val:.3f} ({data_time.avg:.3f}) '\n                                'Elapsed {remain:s} '\n                                'Loss: {loss.val:.4f}({loss.avg:.4f}) '\n                                .format(\n                                step, len(valid_loader), batch_time=batch_time,\n                                data_time=data_time, loss=losses,\n                                remain=timeSince(start, float(step+1)/len(valid_loader)),\n                                ))\n    trues = np.concatenate(trues)\n    predictions = np.concatenate(preds)\n    return losses.avg, predictions, trues","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.070036,"end_time":"2020-12-23T19:06:21.680816","exception":false,"start_time":"2020-12-23T19:06:21.61078","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Train loop"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2020-12-23T19:06:21.836318Z","iopub.status.busy":"2020-12-23T19:06:21.798625Z","iopub.status.idle":"2020-12-23T19:06:21.839883Z","shell.execute_reply":"2020-12-23T19:06:21.839093Z"},"papermill":{"duration":0.115843,"end_time":"2020-12-23T19:06:21.840016","exception":false,"start_time":"2020-12-23T19:06:21.724173","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# ====================================================\n# Train loop\n# ====================================================\ndef train_loop(folds, fold):\n\n    if CFG.device == 'GPU':\n        LOGGER.info(f\"========== fold: {fold} training ==========\")\n    elif CFG.device == 'TPU':\n        if CFG.nprocs == 1:\n            LOGGER.info(f\"========== fold: {fold} training ==========\")\n        elif CFG.nprocs == 8:\n            xm.master_print(f\"========== fold: {fold} training ==========\")\n\n    # ====================================================\n    # loader\n    # ====================================================\n    trn_idx = folds[folds['fold'] != fold].index\n    val_idx = folds[folds['fold'] == fold].index\n\n    train_folds = folds.loc[trn_idx].reset_index(drop=True)\n    valid_folds = folds.loc[val_idx].reset_index(drop=True)\n    \n    train_folds = train_folds[train_folds['StudyInstanceUID'].isin(train_annotations['StudyInstanceUID'].unique())].reset_index(drop=True)\n    valid_folds = valid_folds[valid_folds['StudyInstanceUID'].isin(train_annotations['StudyInstanceUID'].unique())].reset_index(drop=True)\n    \n    valid_labels = valid_folds[CFG.target_cols].values\n\n    train_dataset = TrainDataset(train_folds, train_annotations,\n                                 transform=get_transforms(data='train'))\n    valid_dataset = TrainDataset(valid_folds, train_annotations,\n                                 transform=get_transforms(data='valid'))\n\n    if CFG.device == 'GPU':\n        train_loader = DataLoader(train_dataset, \n                                  batch_size=CFG.batch_size, \n                                  shuffle=True, \n                                  num_workers=CFG.num_workers, pin_memory=True, drop_last=True)\n        valid_loader = DataLoader(valid_dataset, \n                                  batch_size=CFG.batch_size * 2, \n                                  shuffle=False, \n                                  num_workers=CFG.num_workers, pin_memory=True, drop_last=False)\n    elif CFG.device == 'TPU':\n        train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset,\n                                                                        num_replicas=xm.xrt_world_size(),\n                                                                        rank=xm.get_ordinal(),\n                                                                        shuffle=True)\n        train_loader = torch.utils.data.DataLoader(train_dataset,\n                                                   batch_size=CFG.batch_size,\n                                                   sampler=train_sampler,\n                                                   drop_last=True,\n                                                   num_workers=CFG.num_workers)\n        valid_sampler = torch.utils.data.distributed.DistributedSampler(valid_dataset,\n                                                                        num_replicas=xm.xrt_world_size(),\n                                                                        rank=xm.get_ordinal(),\n                                                                        shuffle=False)\n        valid_loader = torch.utils.data.DataLoader(valid_dataset,\n                                                   batch_size=CFG.batch_size * 2,\n                                                   sampler=valid_sampler,\n                                                   drop_last=False,\n                                                   num_workers=CFG.num_workers)\n\n    # ====================================================\n    # scheduler \n    # ====================================================\n    def get_scheduler(optimizer):\n        if CFG.scheduler=='ReduceLROnPlateau':\n            scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=CFG.factor, patience=CFG.patience, verbose=True, eps=CFG.eps)\n        elif CFG.scheduler=='CosineAnnealingLR':\n            scheduler = CosineAnnealingLR(optimizer, T_max=CFG.T_max, eta_min=CFG.min_lr, last_epoch=-1)\n        elif CFG.scheduler=='CosineAnnealingWarmRestarts':\n            scheduler = CosineAnnealingWarmRestarts(optimizer, T_0=CFG.T_0, T_mult=1, eta_min=CFG.min_lr, last_epoch=-1)\n        return scheduler\n\n    # ====================================================\n    # model & optimizer\n    # ====================================================\n    if CFG.device == 'TPU':\n        device = xm.xla_device()\n    elif CFG.device == 'GPU':\n        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    \n    model = CustomResNet200D(CFG.model_name, pretrained=True)\n    model.to(device)\n\n    optimizer = Adam(model.parameters(), lr=CFG.lr, weight_decay=CFG.weight_decay, amsgrad=False)\n    scheduler = get_scheduler(optimizer)\n\n    # ====================================================\n    # loop\n    # ====================================================\n    criterion = nn.BCEWithLogitsLoss()\n\n    best_score = 0.\n    best_loss = np.inf\n    \n    for epoch in range(CFG.epochs):\n        \n        start_time = time.time()\n        \n        # train\n        if CFG.device == 'TPU':\n            if CFG.nprocs == 1:\n                avg_loss = train_fn(train_loader, model, criterion, optimizer, epoch, scheduler, device)\n            elif CFG.nprocs == 8:\n                para_train_loader = pl.ParallelLoader(train_loader, [device])\n                avg_loss = train_fn(para_train_loader.per_device_loader(device), model, criterion, optimizer, epoch, scheduler, device)\n        elif CFG.device == 'GPU':\n            avg_loss = train_fn(train_loader, model, criterion, optimizer, epoch, scheduler, device)\n                \n        # eval\n        if CFG.device == 'TPU':\n            if CFG.nprocs == 1:\n                avg_val_loss, preds, _ = valid_fn(valid_loader, model, criterion, device)\n            elif CFG.nprocs == 8:\n                para_valid_loader = pl.ParallelLoader(valid_loader, [device])\n                avg_val_loss, preds, valid_labels = valid_fn(para_valid_loader.per_device_loader(device), model, criterion, device)\n                preds = idist.all_gather(torch.tensor(preds)).to('cpu').numpy()\n                valid_labels = idist.all_gather(torch.tensor(valid_labels)).to('cpu').numpy()\n        elif CFG.device == 'GPU':\n            avg_val_loss, preds, _ = valid_fn(valid_loader, model, criterion, device)\n        \n        if isinstance(scheduler, ReduceLROnPlateau):\n            scheduler.step(avg_val_loss)\n        elif isinstance(scheduler, CosineAnnealingLR):\n            scheduler.step()\n        elif isinstance(scheduler, CosineAnnealingWarmRestarts):\n            scheduler.step()\n\n        # scoring\n        score, scores = get_score(valid_labels, preds)\n\n        elapsed = time.time() - start_time\n\n        if CFG.device == 'GPU':\n            LOGGER.info(f'Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f}  avg_val_loss: {avg_val_loss:.4f}  time: {elapsed:.0f}s')\n            LOGGER.info(f'Epoch {epoch+1} - Score: {score:.4f}  Scores: {np.round(scores, decimals=4)}')\n        elif CFG.device == 'TPU':\n            if CFG.nprocs == 1:\n                LOGGER.info(f'Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f}  avg_val_loss: {avg_val_loss:.4f}  time: {elapsed:.0f}s')\n                LOGGER.info(f'Epoch {epoch+1} - Score: {score:.4f}  Scores: {np.round(scores, decimals=4)}')\n            elif CFG.nprocs == 8:\n                xm.master_print(f'Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f}  avg_val_loss: {avg_val_loss:.4f}  time: {elapsed:.0f}s')\n                xm.master_print(f'Epoch {epoch+1} - Score: {score:.4f}  Scores: {np.round(scores, decimals=4)}')\n        \n        if score > best_score:\n            best_score = score\n            if CFG.device == 'GPU':\n                LOGGER.info(f'Epoch {epoch+1} - Save Best Score: {best_score:.4f} Model')\n                torch.save({'model': model.state_dict(), \n                            'preds': preds},\n                           OUTPUT_DIR+f'{CFG.model_name}_fold{fold}_best_score.pth')\n            elif CFG.device == 'TPU':\n                if CFG.nprocs == 1:\n                    LOGGER.info(f'Epoch {epoch+1} - Save Best Score: {best_score:.4f} Model')\n                elif CFG.nprocs == 8:\n                    xm.master_print(f'Epoch {epoch+1} - Save Best Score: {best_score:.4f} Model')\n                xm.save({'model': model, \n                         'preds': preds}, \n                        OUTPUT_DIR+f'{CFG.model_name}_fold{fold}_best_score.pth')\n        \n        if avg_val_loss < best_loss:\n            best_loss = avg_val_loss\n            if CFG.device == 'GPU':\n                LOGGER.info(f'Epoch {epoch+1} - Save Best Loss: {best_loss:.4f} Model')\n                torch.save({'model': model.state_dict(), \n                            'preds': preds},\n                           OUTPUT_DIR+f'{CFG.model_name}_fold{fold}_best_loss.pth')\n            elif CFG.device == 'TPU':\n                if CFG.nprocs == 1:\n                    LOGGER.info(f'Epoch {epoch+1} - Save Best Loss: {best_loss:.4f} Model')\n                elif CFG.nprocs == 8:\n                    xm.master_print(f'Epoch {epoch+1} - Save Best Loss: {best_loss:.4f} Model')\n                xm.save({'model': model, \n                         'preds': preds}, \n                        OUTPUT_DIR+f'{CFG.model_name}_fold{fold}_best_loss.pth')\n    \n    if CFG.nprocs != 8:\n        check_point = torch.load(OUTPUT_DIR+f'{CFG.model_name}_fold{fold}_best_score.pth')\n        for c in [f'pred_{c}' for c in CFG.target_cols]:\n            valid_folds[c] = np.nan\n        valid_folds[[f'pred_{c}' for c in CFG.target_cols]] = check_point['preds']\n\n    return valid_folds","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-23T19:06:21.949276Z","iopub.status.busy":"2020-12-23T19:06:21.937789Z","iopub.status.idle":"2020-12-23T19:06:21.956761Z","shell.execute_reply":"2020-12-23T19:06:21.955543Z"},"papermill":{"duration":0.071384,"end_time":"2020-12-23T19:06:21.95697","exception":false,"start_time":"2020-12-23T19:06:21.885586","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# ====================================================\n# main\n# ====================================================\ndef main():\n\n    \"\"\"\n    Prepare: 1.train  2.folds\n    \"\"\"\n\n    def get_result(result_df):\n        preds = result_df[[f'pred_{c}' for c in CFG.target_cols]].values\n        labels = result_df[CFG.target_cols].values\n        score, scores = get_score(labels, preds)\n        LOGGER.info(f'Score: {score:<.4f}  Scores: {np.round(scores, decimals=4)}')\n    \n    if CFG.train:\n        # train \n        oof_df = pd.DataFrame()\n        for fold in range(CFG.n_fold):\n            if fold in CFG.trn_fold:\n                _oof_df = train_loop(folds, fold)\n                oof_df = pd.concat([oof_df, _oof_df])\n                if CFG.nprocs != 8:\n                    LOGGER.info(f\"========== fold: {fold} result ==========\")\n                    get_result(_oof_df)\n                    \n        if CFG.nprocs != 8:\n            # CV result\n            LOGGER.info(f\"========== CV ==========\")\n            get_result(oof_df)\n            # save result\n            oof_df.to_csv(OUTPUT_DIR+'oof_df.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.execute_input":"2020-12-23T19:06:22.059092Z","iopub.status.busy":"2020-12-23T19:06:22.057932Z","iopub.status.idle":"2020-12-23T19:57:36.57877Z","shell.execute_reply":"2020-12-23T19:57:36.580062Z"},"papermill":{"duration":3074.575173,"end_time":"2020-12-23T19:57:36.580679","exception":false,"start_time":"2020-12-23T19:06:22.005506","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"if __name__ == '__main__':\n    if CFG.device == 'TPU':\n        def _mp_fn(rank, flags):\n            torch.set_default_tensor_type('torch.FloatTensor')\n            a = main()\n        FLAGS = {}\n        xmp.spawn(_mp_fn, args=(FLAGS,), nprocs=CFG.nprocs, start_method='fork')\n    elif CFG.device == 'GPU':\n        main()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-23T19:57:37.175743Z","iopub.status.busy":"2020-12-23T19:57:37.174833Z","iopub.status.idle":"2020-12-23T19:57:56.066285Z","shell.execute_reply":"2020-12-23T19:57:56.065385Z"},"papermill":{"duration":19.418317,"end_time":"2020-12-23T19:57:56.066443","exception":false,"start_time":"2020-12-23T19:57:36.648126","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# save as cpu\nif CFG.device == 'TPU':\n    for fold in range(CFG.n_fold):\n        if fold in CFG.trn_fold:\n            # best score\n            state = torch.load(OUTPUT_DIR+f'{CFG.model_name}_fold{fold}_best_score.pth')\n            torch.save({'model': state['model'].to('cpu').state_dict(), \n                        'preds': state['preds']}, \n                        OUTPUT_DIR+f'{CFG.model_name}_fold{fold}_best_score_cpu.pth')\n            # best loss\n            state = torch.load(OUTPUT_DIR+f'{CFG.model_name}_fold{fold}_best_loss.pth')\n            torch.save({'model': state['model'].to('cpu').state_dict(), \n                        'preds': state['preds']}, \n                        OUTPUT_DIR+f'{CFG.model_name}_fold{fold}_best_loss_cpu.pth')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}