{"cells":[{"metadata":{},"cell_type":"markdown","source":"# v12 (continue training)    \n\nname : melanoma_tachyon_v12  \nabout : check qishen ha model.  \nmodel : Enet (b3) batchnorm freeze  \nimg_size : 384x384  \nbatch : 16  \nepoch : 4  \ncriterion : BCEWithLogitsLoss  \noptimizer : RAdam  \ninit_lr : 2e-5  \nscheduler: CosineAnnealingLR  \ndata : melanoma-merged-external-data-512x512-jpeg-shades  \npreprocess : Shades  \ntrain_test_split : 5 of 3 folds(Triple StratifiedKFold, k=5)  \ndata augmentation : hairaug,flip,crop,jitter,cutout  \ntta : 3 times(same condition of train DA)  ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"local = False\nDEBUG = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#!pip install efficientnet_pytorch torchtoolbox","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#!pip install git+https://github.com/ildoonet/pytorch-gradual-warmup-lr.git","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#before import process\nimport sys\nif local == True:\n    package_path = 'Q:/kaggle/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master'\nelse:\n    package_path = '../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master'\nsys.path.append(package_path)\n\n#imports\nimport os, warnings, random, time\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport cv2\nfrom sklearn.metrics import accuracy_score, roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom IPython.display import display\nfrom tqdm import tqdm_notebook as tqdm\nimport torch\nfrom torch import nn, optim\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom torch.functional import F \nfrom torch.utils.data import Dataset, DataLoader\nfrom efficientnet_pytorch import model as enet\nimport albumentations as A\n#import torchtoolbox.transform as transforms\n#from warmup_scheduler import GradualWarmupScheduler\n%matplotlib inline\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SEED = 32 #69\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(SEED)\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(device)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#params\nenet_type = 'efficientnet-b3'\nmodel_name = 'v12'\nn_epochs = 5 if DEBUG else 4\ncosine_t = 6\nn_fold = 5\n\nbatch_size = 32\nimage_size = 300\n\nnum_workers = 2\n\ninit_lr = 2e-5\nTTA = 3\n\nProgress_Bar = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if local:\n    pass\nelse:\n    train_df = pd.read_csv('../input/melanoma-merged-external-data-512x512-jpeg/folds_13062020.csv')\n    test_df = pd.read_csv('../input/siim-isic-melanoma-classification/test.csv')\n    train_images = '../input/melanoma-merged-external-data-512x512-jpeg/512x512-dataset-melanoma/512x512-dataset-melanoma'\n    test_images = '../input/melanoma-merged-external-data-512x512-jpeg/512x512-test/512x512-test'\n    model_dir = '../input/melanoma-tachyon-v12-models'\n\nif DEBUG:\n    train_df = train_df[:100]\n    test_df = test_df[:50]\n    \n# One-hot encoding of anatom_site_general_challenge feature\nconcat = pd.concat([train_df['anatom_site_general_challenge'], test_df['anatom_site_general_challenge']], ignore_index=True)\ndummies = pd.get_dummies(concat, dummy_na=True, dtype=np.uint8, prefix='site')\ntrain_df = pd.concat([train_df, dummies.iloc[:train_df.shape[0]]], axis=1)\ntest_df = pd.concat([test_df, dummies.iloc[train_df.shape[0]:].reset_index(drop=True)], axis=1)\n\n# Sex features\ntrain_df['sex'] = train_df['sex'].map({'male': 1, 'female': 0})\ntest_df['sex'] = test_df['sex'].map({'male': 1, 'female': 0})\ntrain_df['sex'] = train_df['sex'].fillna(-1)\ntest_df['sex'] = test_df['sex'].fillna(-1)\n\n# Age features\ntrain_df['age_approx'] /= train_df['age_approx'].max()\ntest_df['age_approx'] /= test_df['age_approx'].max()\ntrain_df['age_approx'] = train_df['age_approx'].fillna(0)\ntest_df['age_approx'] = test_df['age_approx'].fillna(0)\n\ntrain_df['patient_id'] = train_df['patient_id'].fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"train_df.reset_index(inplace=True)\nskf = StratifiedKFold(n_fold, shuffle=True, random_state=SEED)\ntrain_df['fold'] = -1\nfor i, (train_idx, valid_idx) in enumerate(skf.split(train_df, train_df['target'])):\n    train_df.loc[valid_idx, 'fold'] = i\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"meta_features = ['sex', 'age_approx'] + [col for col in train_df.columns if 'site_' in col]\nmeta_features.remove('anatom_site_general_challenge')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if local:\n    pretrained_model = {\n        'efficientnet-b0': 'Q:/kaggle/efficientnet-pytorch/efficientnet-b0-08094119.pth'\n        \n    }\n    model_save_path = 'Q:/kaggle/melanoma-trained-models'\nelse:\n    pretrained_model = {\n        'efficientnet-b0': '../input/efficientnet-pytorch/efficientnet-b0-08094119.pth',\n        'efficientnet-b1': '../input/efficientnet-pytorch/efficientnet-b1-dbc7070a.pth',\n        'efficientnet-b2': '../input/efficientnet-pytorch/efficientnet-b2-27687264.pth',\n        'efficientnet-b3': '../input/efficientnet-pytorch/efficientnet-b3-c8376fa2.pth',\n        'efficientnet-b4': '../input/efficientnet-pytorch/efficientnet-b4-e116e8b3.pth',\n        'efficientnet-b5': '../input/efficientnet-pytorch/efficientnet-b5-586e6cc6.pth',\n        \n    }\n    model_save_path = False\n\nclass enetv2(nn.Module):\n    def __init__(self, backbone, out_dim, n_meta_features):\n        super(enetv2, self).__init__()\n        if out_dim != 500:\n            out_dim = 500\n            print(\"out_dim size is changed\")\n        \n        self.enet = enet.EfficientNet.from_name(backbone)\n        self.enet.load_state_dict(torch.load(pretrained_model[backbone]))\n\n        self.myfc = nn.Linear(self.enet._fc.in_features, out_dim)\n        self.enet._fc = nn.Identity()\n        self.meta = nn.Sequential(nn.Linear(n_meta_features, 500),\n                                  nn.BatchNorm1d(500),\n                                  nn.ReLU(),\n                                  nn.Dropout(p=0.2),\n                                  nn.Linear(500, 250),  # FC layer output will have 250 features\n                                  nn.BatchNorm1d(250),\n                                  nn.ReLU(),\n                                  nn.Dropout(p=0.2))\n        self.sigmoid = nn.Sigmoid()\n        self.ouput = nn.Linear(500 + 250, 1)\n        \n    def extract(self, x):\n        return self.enet(x)\n\n    def forward(self, inputs):\n        x,meta = inputs\n        \n        x = self.extract(x)\n        x = self.myfc(x) #500\n        \n        meta = self.meta(meta) #250\n        \n        features = torch.cat((x, meta), dim=1)\n        output = self.ouput(features)\n        #output = self.sigmoid(output) #no sigmoid(if BCE loss)\n        return output \ndef load_models(model_files):\n    models = []\n    for model_f in model_files:\n        model_f = os.path.join(model_dir, model_f)\n        model = enetv2(enet_type, out_dim=500,n_meta_features = len(meta_features))\n        model.load_state_dict(torch.load(model_f, map_location=lambda storage, loc: storage), strict=True)\n        model.eval()\n        model.to(device)\n        models.append(model)\n        print(f'{model_f} loaded!')\n    return models\n\n\nmodel_files = [\n    'v12_efficientnet-b3_f0_final.pt'\n]\n\npre_models = load_models(model_files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class AdvancedHairAugmentation:\n    \"\"\"\n    Impose an image of a hair to the target image\n\n    Args:\n        hairs (int): maximum number of hairs to impose\n        hairs_folder (str): path to the folder with hairs images\n    \"\"\"\n\n    def __init__(self, hairs: int = 5, hairs_folder: str = \"\"):\n        self.hairs = hairs\n        self.hairs_folder = hairs_folder\n\n    def __call__(self, img):\n        \"\"\"\n        Args:\n            img (PIL Image): Image to draw hairs on.\n\n        Returns:\n            PIL Image: Image with drawn hairs.\n        \"\"\"\n        n_hairs = random.randint(0, self.hairs)\n        \n        if not n_hairs:\n            return img\n        \n        height, width, _ = img.shape  # target image width and height\n        hair_images = [im for im in os.listdir(self.hairs_folder) if 'png' in im]\n        \n        for _ in range(n_hairs):\n            hair = cv2.imread(os.path.join(self.hairs_folder, random.choice(hair_images)))\n            hair = cv2.flip(hair, random.choice([-1, 0, 1]))\n            hair = cv2.rotate(hair, random.choice([0, 1, 2]))\n\n            h_height, h_width, _ = hair.shape  # hair image width and height\n            roi_ho = random.randint(0, img.shape[0] - hair.shape[0])\n            roi_wo = random.randint(0, img.shape[1] - hair.shape[1])\n            roi = img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width]\n\n            # Creating a mask and inverse mask\n            img2gray = cv2.cvtColor(hair, cv2.COLOR_BGR2GRAY)\n            ret, mask = cv2.threshold(img2gray, 10, 255, cv2.THRESH_BINARY)\n            mask_inv = cv2.bitwise_not(mask)\n\n            # Now black-out the area of hair in ROI\n            img_bg = cv2.bitwise_and(roi, roi, mask=mask_inv)\n\n            # Take only region of hair from hair image.\n            hair_fg = cv2.bitwise_and(hair, hair, mask=mask)\n\n            # Put hair in ROI and modify the target image\n            dst = cv2.add(img_bg, hair_fg)\n\n            img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width] = dst\n                \n        return img\n\n    def __repr__(self):\n        return f'{self.__class__.__name__}(hairs={self.hairs}, hairs_folder=\"{self.hairs_folder}\")'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Microscope:\n    \"\"\"\n    Cutting out the edges around the center circle of the image\n    Imitating a picture, taken through the microscope\n\n    Args:\n        p (float): probability of applying an augmentation\n    \"\"\"\n\n    def __init__(self, p: float = 0.5):\n        self.p = p\n\n    def __call__(self, img):\n        \"\"\"\n        Args:\n            img (PIL Image): Image to apply transformation to.\n\n        Returns:\n            PIL Image: Image with transformation.\n        \"\"\"\n        if random.random() < self.p:\n            circle = cv2.circle((np.ones(img.shape) * 255).astype(np.uint8), # image placeholder\n                        (img.shape[0]//2, img.shape[1]//2), # center point of circle\n                        random.randint(img.shape[0]//2 - 3, img.shape[0]//2 + 15), # radius\n                        (0, 0, 0), # color\n                        -1)\n\n            mask = circle - 255\n            img = np.multiply(img, mask)\n        \n        return img\n\n    def __repr__(self):\n        return f'{self.__class__.__name__}(p={self.p})'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Cutout(object):\n    def __init__(self, length=16):\n        self.length = length\n\n    def __call__(self, img):\n        img = np.array(img)\n\n        mask_val = img.mean()\n\n        top = np.random.randint(0 - self.length//2, img.shape[0] - self.length)\n        left = np.random.randint(0 - self.length//2, img.shape[1] - self.length)\n        bottom = top + self.length\n        right = left + self.length\n\n        top = 0 if top < 0 else top\n        left = 0 if left < 0 else top\n\n        img[top:bottom, left:right, :] = mask_val\n\n        #img = Image.fromarray(img)\n\n        return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transforms_train = A.Compose([\n    A.Transpose(p=0.5),\n    A.VerticalFlip(p=0.5),\n    A.HorizontalFlip(p=0.5),\n    A.RandomBrightness(limit=0.2, p=0.75),\n    A.RandomContrast(limit=0.2, p=0.75),\n    A.OneOf([\n        A.MotionBlur(blur_limit=5),\n        A.MedianBlur(blur_limit=5),\n        A.GaussianBlur(blur_limit=5),\n        A.GaussNoise(var_limit=(5.0, 30.0)),\n    ], p=0.7),\n\n    A.OneOf([\n        A.OpticalDistortion(distort_limit=1.0),\n        A.GridDistortion(num_steps=5, distort_limit=1.),\n        A.ElasticTransform(alpha=3),\n    ], p=0.7),\n\n    A.CLAHE(clip_limit=4.0, p=0.7),\n    A.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=20, val_shift_limit=10, p=0.5),\n    A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.1, rotate_limit=15, border_mode=0, p=0.85),\n    A.Resize(image_size, image_size),\n    A.Cutout(max_h_size=int(image_size * 0.375), max_w_size=int(image_size * 0.375), num_holes=1, p=0.7),    \n    A.Normalize()\n])\n\ntransforms_val = A.Compose([\n    A.Resize(image_size, image_size),\n    A.Normalize()\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nclass MelanomaDataset(Dataset):\n    def __init__(self, df: pd.DataFrame, imfolder: str, train: bool = True, transforms = None, meta_features = None):\n        \"\"\"\n        Class initialization\n        Args:\n            df (pd.DataFrame): DataFrame with data description\n            imfolder (str): folder with images\n            train (bool): flag of whether a training dataset is being initialized or testing one\n            transforms: image transformation method to be applied\n            meta_features (list): list of features with meta information, such as sex and age\n            \n        \"\"\"\n        self.df = df\n        self.imfolder = imfolder\n        self.transforms = transforms\n        self.train = train\n        self.meta_features = meta_features\n        \n    def __getitem__(self, index):\n        if self.train:\n            im_path = os.path.join(self.imfolder, self.df.iloc[index]['image_id'] + '.jpg')\n        else:\n            im_path = os.path.join(self.imfolder, self.df.iloc[index]['image_name'] + '.jpg')\n        if not os.path.exists(im_path):\n            raise Exception(f\"ファイルパスが存在しません：{im_path}\")\n        x = cv2.imread(im_path)\n        meta = np.array(self.df.iloc[index][self.meta_features].values, dtype=np.float32)\n\n        if self.transforms:\n            x = self.transforms(image = x)\n            x = x['image'].astype(np.float32)\n        x = x.transpose(2, 0, 1) #channel first\n        if self.train:\n            y = self.df.iloc[index]['target']\n            return (x, meta), y\n        else:\n            return (x, meta)\n    \n    def __len__(self):\n        return len(self.df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset_show = MelanomaDataset(train_df, train_images, train=True,transforms = transforms_train,  meta_features=meta_features)\nfrom pylab import rcParams\nrcParams['figure.figsize'] = 20,10\nfor i in range(2):\n    f, axarr = plt.subplots(1,5)\n    for p in range(5):\n        idx = np.random.randint(0, len(dataset_show))\n        img, label = dataset_show[idx]\n        img, meta = img[0],img[1]\n        img = np.asarray(img)\n        img = img.transpose(1,2,0)\n        #img = img.astype(np.uint8)\n        #img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) #rgb→bgr\n        axarr[p].imshow(img) \n        axarr[p].set_title(str(label))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import math\nimport torch\nfrom torch.optim.optimizer import Optimizer, required\n\nclass RAdam(Optimizer):\n\n    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0):\n        defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)\n        self.buffer = [[None, None, None] for ind in range(10)]\n        super(RAdam, self).__init__(params, defaults)\n\n    def __setstate__(self, state):\n        super(RAdam, self).__setstate__(state)\n\n    def step(self, closure=None):\n\n        loss = None\n        if closure is not None:\n            loss = closure()\n\n        for group in self.param_groups:\n\n            for p in group['params']:\n                if p.grad is None:\n                    continue\n                grad = p.grad.data.float()\n                if grad.is_sparse:\n                    raise RuntimeError('RAdam does not support sparse gradients')\n\n                p_data_fp32 = p.data.float()\n\n                state = self.state[p]\n\n                if len(state) == 0:\n                    state['step'] = 0\n                    state['exp_avg'] = torch.zeros_like(p_data_fp32)\n                    state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)\n                else:\n                    state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)\n                    state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)\n\n                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']\n                beta1, beta2 = group['betas']\n\n                exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)\n                exp_avg.mul_(beta1).add_(1 - beta1, grad)\n\n                state['step'] += 1\n                buffered = self.buffer[int(state['step'] % 10)]\n                if state['step'] == buffered[0]:\n                    N_sma, step_size = buffered[1], buffered[2]\n                else:\n                    buffered[0] = state['step']\n                    beta2_t = beta2 ** state['step']\n                    N_sma_max = 2 / (1 - beta2) - 1\n                    N_sma = N_sma_max - 2 * state['step'] * beta2_t / (1 - beta2_t)\n                    buffered[1] = N_sma\n\n                    # more conservative since it's an approximated value\n                    if N_sma >= 5:\n                        step_size = group['lr'] * math.sqrt((1 - beta2_t) * (N_sma - 4) / (N_sma_max - 4) * (N_sma - 2) / N_sma * N_sma_max / (N_sma_max - 2)) / (1 - beta1 ** state['step'])\n                    else:\n                        step_size = group['lr'] / (1 - beta1 ** state['step'])\n                    buffered[2] = step_size\n\n                if group['weight_decay'] != 0:\n                    p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)\n\n                # more conservative since it's an approximated value\n                if N_sma >= 5:            \n                    denom = exp_avg_sq.sqrt().add_(group['eps'])\n                    p_data_fp32.addcdiv_(-step_size, exp_avg, denom)\n                else:\n                    p_data_fp32.add_(-step_size, exp_avg)\n\n                p.data.copy_(p_data_fp32)\n\n        return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def set_bn_eval(module):\n    if isinstance(module, torch.nn.modules.batchnorm._BatchNorm):\n        module.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(model, iterator, optimizer, criterion, device):\n    \n    epoch_loss = 0\n    model.train()\n    model.apply(set_bn_eval)\n    bar = tqdm(iterator) if Progress_Bar else iterator\n    \n    for (x, y) in bar:\n        #x[0]:image deta, x[1]:table data(meta data)\n        x[0] = torch.tensor(x[0], device=device, dtype=torch.float32)\n        x[1] = torch.tensor(x[1], device=device, dtype=torch.float32)\n        y = torch.tensor(y, device=device, dtype=torch.float32)\n        \n        optimizer.zero_grad()\n        y_pred = model(x)\n        loss = criterion(y_pred, y.unsqueeze(1)) #ここで、yにy.unsqeeze()次元拡張を挟む必要がある？(BCEだったら、だと思われる)\n        loss.backward()\n        optimizer.step()\n        loss_np = loss.detach().cpu().numpy()\n        epoch_loss += loss_np\n        if Progress_Bar:\n            bar.set_description('Training loss: %.5f' % (loss_np))\n        \n    return epoch_loss/len(iterator)\n\ndef evaluate(model, iterator, criterion, device):\n    \n    epoch_loss = 0\n    preds = []\n    preds = np.array(preds)\n    targets = []\n    targets = np.array(targets)\n    model.eval()\n    bar = tqdm(iterator) if Progress_Bar else iterator\n    \n    with torch.no_grad():\n        \n        for (x, y) in bar:\n        \n            x[0] = torch.tensor(x[0], device=device, dtype=torch.float32)\n            x[1] = torch.tensor(x[1], device=device, dtype=torch.float32)\n            y = torch.tensor(y, device=device, dtype=torch.float32)\n            \n            y_pred = model(x)\n            loss = criterion(y_pred, y.unsqueeze(1))\n            loss_np = loss.detach().cpu().numpy()\n            epoch_loss += loss_np\n            y_pred = torch.sigmoid(y_pred)\n            preds = np.append(preds, y_pred.detach().cpu().numpy())\n            targets = np.append(targets, y.detach().cpu().numpy())\n            \n            if Progress_Bar:\n                bar.set_description('Validation loss: %.5f' % (loss_np))\n    \n    val_acc = accuracy_score(targets, np.round(preds))\n    try:\n       val_roc = roc_auc_score(targets, preds)\n    except ValueError:\n       val_roc = -1\n    \n            \n    return epoch_loss/len(iterator), val_acc,val_roc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fit_model(model, model_name, train_iterator, valid_iterator, optimizer, loss_criterion, device, epochs):\n    \"\"\" Fits a dataset to model\"\"\"\n    best_valid_score = float('inf')\n    \n    train_losses = []\n    valid_losses = []\n    valid_roc_scores = []\n    \n    for epoch in range(epochs):\n        scheduler.step(epoch)\n        start_time = time.time()\n    \n        train_loss = train(model, train_iterator, optimizer, loss_criterion, device)\n        valid_loss, valid_acc_score, valid_roc_score = evaluate(model, valid_iterator, loss_criterion, device)\n        \n        train_losses.append(train_loss)\n        valid_losses.append(valid_loss)\n        valid_roc_scores.append(valid_roc_score)\n\n        if valid_roc_score < best_valid_score:\n            best_valid_score = valid_roc_score\n            if model_save_path:\n                torch.save(model.state_dict(), os.path.join(model_save_path,f'{model_name}.pt'))\n            else:\n                torch.save(model.state_dict(), f'{model_name}_best.pt')\n        \n        #schedulerの処理 cosineannealingは別\n        #if scheduler != None:\n        #    scheduler.step(valid_loss)\n        end_time = time.time()\n\n        epoch_mins, epoch_secs = (end_time-start_time)//60,round((end_time-start_time)%60)\n    \n        print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n        print(f'Train Loss: {train_loss:.3f}')\n        print(f'Val. Loss: {valid_loss:.3f} | Val. ACC Score: {valid_acc_score:.3f} | Val. Metric Score: {valid_roc_score:.4f}')\n        print(f'lr:{optimizer.param_groups[0][\"lr\"]:.7f}')\n        \n        torch.save(model.state_dict(), f'{model_name}_final.pt')\n        \n    return train_losses, valid_losses, valid_roc_scores","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_loss=[]\nval_loss=[]\nval_roc=[]\nmodels = []\nfor fold in range(1): #now 5 folds\n    print(f\"Fitting on Fold {fold+1}\")\n    #Make Train and Valid DataFrame from fold\n    train_df_fold = train_df[train_df['fold'] != fold].reset_index(drop=True)\n    valid_df_fold = train_df[train_df['fold'] == fold].reset_index(drop=True)\n    \n    #Build and load Dataset\n    train_data = MelanomaDataset(train_df_fold, train_images, train=True, transforms = transforms_train, meta_features=meta_features) \n    valid_data = MelanomaDataset(valid_df_fold, train_images, train=True, transforms = transforms_val, meta_features=meta_features) \n    \n    train_iterator = DataLoader(train_data, shuffle=True, batch_size=batch_size, num_workers=num_workers)\n    valid_iterator = DataLoader(valid_data, shuffle=False, batch_size=16, num_workers=num_workers)\n    \n    model = pre_models[0].to(device)\n    loss_criterion = nn.BCEWithLogitsLoss()\n    opt= RAdam(model.parameters(),lr=init_lr)\n    scheduler = CosineAnnealingLR(opt, cosine_t)\n    name = model_name + \"_\" + enet_type + \"_f\" + str(fold)\n    \n    temp_tr_loss, temp_val_loss, temp_val_roc = fit_model(model, name, train_iterator, valid_iterator, opt, loss_criterion, device, epochs=n_epochs)\n    \n    tr_loss+=temp_tr_loss\n    val_loss+=temp_val_loss\n    val_roc+=temp_val_roc\n    \n    models.append(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig,ax = plt.subplots(nrows=1, ncols=2, figsize=(20,5))\nax[0].plot(tr_loss)\nax[0].set_title('Training and Validation Loss')\nax[0].plot(val_loss)\nax[0].set_xlabel('Epoch')\n\nax[1].plot(val_roc)\nax[1].set_title('Val ROC Score')\nax[1].set_xlabel('Epoch')\n\n\nax[0].legend();\nax[1].legend();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# submit section from here  ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test = MelanomaDataset(df=test_df,\n                       imfolder=test_images, \n                       train=False,\n                       transforms=transforms_train,  # if TTA. change here\n                       meta_features=meta_features)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#(model数*TTA数)回すので注意\ndef get_predictions(model, iterator, device):\n    \n    preds = np.array([0.]*len(test_df))\n    model.eval()\n    bar = tqdm(iterator) if Progress_Bar else iterator\n    \n    with torch.no_grad():\n        for tta in range(TTA):\n            res = np.array([])\n            for x in bar:\n                x[0] = torch.tensor(x[0], device=device, dtype=torch.float32)\n                x[1] = torch.tensor(x[1], device=device, dtype=torch.float32)\n                y_pred = model(x)\n                y_pred = torch.sigmoid(y_pred)\n                res = np.append(res, y_pred.detach().cpu().numpy())\n            preds += res\n    preds /= TTA\n    return preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction = np.array([0.]*len(test_df))\nfor i in range(len(models)):\n    test_iterator = DataLoader(dataset=test, batch_size=16, shuffle=False, num_workers=num_workers)\n    preds = get_predictions(models[i], test_iterator, device)\n    prediction += preds\nprediction /= len(models)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df = pd.read_csv('../input/siim-isic-melanoma-classification/sample_submission.csv')\nsub_df = sub_df[:50] if DEBUG else sub_df\nsub_df['target'] = prediction\n\nsub_df.to_csv('submission.csv', index=False)\nsub_df.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}