{"cells":[{"metadata":{},"cell_type":"markdown","source":"# 1st Place Solution Training & Testing Code\n\nHi, all,\n\nWe're very exciting to writing this notebook and the summary of our solution here.\n\nThis is a small version (b3 w/ input size 256 training for only 5 epochs) of our training & testing pipeline only for demonstrating our ideas and methods.\n\nTo get a similar result of a single model to ours, you only need to do:\n\n* set `DEBUG` to `False`\n* train for 15 epochs for each fold\n* using apex\n* change the model type to effnet-b4/b5/b6/b7\n* modify the input size to 384 or 512 or 640 (do remember to use bigger dataset as well)\n* keep batch size >= 36 (may have to use multi-gpus)\n\nOur brief summary of winning solution:\nhttps://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/175412\n\n\n# Main Ideas\n\n* apex amp (install apex into kaggle kernel is hard, so I skipped it)\n* use meta data\n* optimized augmentation methods\n* 9 classes w/ CrossEntropyLoss\n* warmup + cosine scheduler\n* rank per fold before ensemble\n\n# Thanks!","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"DEBUG = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip -q install git+https://github.com/ildoonet/pytorch-gradual-warmup-lr.git\n!pip -q install geffnet","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport sys\nimport time\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport PIL.Image\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nfrom tqdm.notebook import tqdm\nfrom sklearn.metrics import roc_auc_score\nimport torch\nfrom torch.utils.data import TensorDataset, DataLoader,Dataset\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\nimport torchvision.transforms as transforms\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data.sampler import SubsetRandomSampler, RandomSampler, SequentialSampler\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom warmup_scheduler import GradualWarmupScheduler  # https://github.com/ildoonet/pytorch-gradual-warmup-lr\nimport albumentations as A\nimport geffnet\n\ndevice = torch.device('cuda')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Config","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"kernel_type = 'effnetb3_256_meta_9c_ext_5epo'\nimage_size = 256\nuse_amp = False\ndata_dir = '../input/jpeg-melanoma-256x256'\ndata_dir2 = '../input/jpeg-isic2019-256x256'\nenet_type = 'efficientnet-b3'\nbatch_size = 64\nnum_workers = 4\ninit_lr = 3e-5\nout_dim = 9\n\nfreeze_epo = 0\nwarmup_epo = 1\ncosine_epo = 4\nn_epochs = freeze_epo + warmup_epo + cosine_epo\n\nuse_external = '_ext' in kernel_type\nuse_meta = 'meta' in kernel_type","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Read CSV & Target Preprocess","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = pd.read_csv(os.path.join(data_dir, 'test.csv'))\ndf_test['filepath'] = df_test['image_name'].apply(lambda x: os.path.join(data_dir, 'test', f'{x}.jpg'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))\ndf_train = df_train[df_train['tfrecord'] != -1].reset_index(drop=True)\n# df_train['fold'] = df_train['tfrecord'] % 5\ntfrecord2fold = {\n    2:0, 4:0, 5:0,\n    1:1, 10:1, 13:1,\n    0:2, 9:2, 12:2,\n    3:3, 8:3, 11:3,\n    6:4, 7:4, 14:4,\n}\ndf_train['fold'] = df_train['tfrecord'].map(tfrecord2fold)\ndf_train['is_ext'] = 0\ndf_train['filepath'] = df_train['image_name'].apply(lambda x: os.path.join(data_dir, 'train', f'{x}.jpg'))\n\n\ndf_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('seborrheic keratosis', 'BKL'))\ndf_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('lichenoid keratosis', 'BKL'))\ndf_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('solar lentigo', 'BKL'))\ndf_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('lentigo NOS', 'BKL'))\ndf_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('cafe-au-lait macule', 'unknown'))\ndf_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('atypical melanocytic proliferation', 'unknown'))\n\ndf_train['diagnosis'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if use_external:\n    df_train2 = pd.read_csv(os.path.join(data_dir2, 'train.csv'))\n    df_train2 = df_train2[df_train2['tfrecord'] >= 0].reset_index(drop=True)\n    df_train2['fold'] = df_train2['tfrecord'] % 5\n    df_train2['is_ext'] = 1\n    df_train2['filepath'] = df_train2['image_name'].apply(lambda x: os.path.join(data_dir2, 'train', f'{x}.jpg'))\n\n    df_train2['diagnosis'] = df_train2['diagnosis'].apply(lambda x: x.replace('NV', 'nevus'))\n    df_train2['diagnosis'] = df_train2['diagnosis'].apply(lambda x: x.replace('MEL', 'melanoma'))\n    df_train = pd.concat([df_train, df_train2]).reset_index(drop=True)\n\ndiagnosis2idx = {d: idx for idx, d in enumerate(sorted(df_train.diagnosis.unique()))}\ndf_train['target'] = df_train['diagnosis'].map(diagnosis2idx)\nmel_idx = diagnosis2idx['melanoma']\ndiagnosis2idx","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['target'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preprocess Meta Data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.filepath","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if use_meta:\n    # One-hot encoding of anatom_site_general_challenge feature\n    concat = pd.concat([df_train['anatom_site_general_challenge'], df_test['anatom_site_general_challenge']], ignore_index=True)\n    dummies = pd.get_dummies(concat, dummy_na=True, dtype=np.uint8, prefix='site')\n    df_train = pd.concat([df_train, dummies.iloc[:df_train.shape[0]]], axis=1)\n    df_test = pd.concat([df_test, dummies.iloc[df_train.shape[0]:].reset_index(drop=True)], axis=1)\n    # Sex features\n    df_train['sex'] = df_train['sex'].map({'male': 1, 'female': 0})\n    df_test['sex'] = df_test['sex'].map({'male': 1, 'female': 0})\n    df_train['sex'] = df_train['sex'].fillna(-1)\n    df_test['sex'] = df_test['sex'].fillna(-1)\n    # Age features\n    df_train['age_approx'] /= 90\n    df_test['age_approx'] /= 90\n    df_train['age_approx'] = df_train['age_approx'].fillna(0)\n    df_test['age_approx'] = df_test['age_approx'].fillna(0)\n    df_train['patient_id'] = df_train['patient_id'].fillna(0)\n    # n_image per user\n    df_train['n_images'] = df_train.patient_id.map(df_train.groupby(['patient_id']).image_name.count())\n    df_test['n_images'] = df_test.patient_id.map(df_test.groupby(['patient_id']).image_name.count())\n    df_train.loc[df_train['patient_id'] == -1, 'n_images'] = 1\n    df_train['n_images'] = np.log1p(df_train['n_images'].values)\n    df_test['n_images'] = np.log1p(df_test['n_images'].values)\n    # image size\n    train_images = df_train['filepath'].values\n    train_sizes = np.zeros(train_images.shape[0])\n    for i, img_path in enumerate(tqdm(train_images)):\n        train_sizes[i] = os.path.getsize(img_path)\n    df_train['image_size'] = np.log(train_sizes)\n    test_images = df_test['filepath'].values\n    test_sizes = np.zeros(test_images.shape[0])\n    for i, img_path in enumerate(tqdm(test_images)):\n        test_sizes[i] = os.path.getsize(img_path)\n    df_test['image_size'] = np.log(test_sizes)\n    meta_features = ['sex', 'age_approx', 'n_images', 'image_size'] + [col for col in df_train.columns if col.startswith('site_')]\n    n_meta_features = len(meta_features)\nelse:\n    n_meta_features = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_meta_features","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Define Dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class SIIMISICDataset(Dataset):\n    def __init__(self, csv, split, mode, transform=None):\n\n        self.csv = csv.reset_index(drop=True)\n        self.split = split\n        self.mode = mode\n        self.transform = transform\n\n    def __len__(self):\n        return self.csv.shape[0]\n\n    def __getitem__(self, index):\n        row = self.csv.iloc[index]\n        \n        image = cv2.imread(row.filepath)\n        image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)\n\n        if self.transform is not None:\n            res = self.transform(image=image)\n            image = res['image'].astype(np.float32)\n        else:\n            image = image.astype(np.float32)\n\n        image = image.transpose(2, 0, 1)\n\n        if use_meta:\n            data = (torch.tensor(image).float(), torch.tensor(self.csv.iloc[index][meta_features]).float())\n        else:\n            data = torch.tensor(image).float()\n\n        if self.mode == 'test':\n            return data\n        else:\n            return data, torch.tensor(self.csv.iloc[index].target).long()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Augmentations","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"transforms_train = A.Compose([\n    A.Transpose(p=0.5),\n    A.VerticalFlip(p=0.5),\n    A.HorizontalFlip(p=0.5),\n    A.RandomBrightness(limit=0.2, p=0.75),\n    A.RandomContrast(limit=0.2, p=0.75),\n    A.OneOf([\n        A.MotionBlur(blur_limit=5),\n        A.MedianBlur(blur_limit=5),\n        A.GaussianBlur(blur_limit=5),\n        A.GaussNoise(var_limit=(5.0, 30.0)),\n    ], p=0.7),\n\n    A.OneOf([\n        A.OpticalDistortion(distort_limit=1.0),\n        A.GridDistortion(num_steps=5, distort_limit=1.),\n        A.ElasticTransform(alpha=3),\n    ], p=0.7),\n\n    A.CLAHE(clip_limit=4.0, p=0.7),\n    A.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=20, val_shift_limit=10, p=0.5),\n    A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.1, rotate_limit=15, border_mode=0, p=0.85),\n    A.Resize(image_size, image_size),\n    A.Cutout(max_h_size=int(image_size * 0.375), max_w_size=int(image_size * 0.375), num_holes=1, p=0.7),    \n    A.Normalize()\n])\n\ntransforms_val = A.Compose([\n    A.Resize(image_size, image_size),\n    A.Normalize()\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_show = df_train.sample(1000)\ndataset_show = SIIMISICDataset(df_show, 'train', 'train', transform=transforms_train)\n\nfrom pylab import rcParams\nrcParams['figure.figsize'] = 20,10\nfor i in range(2):\n    f, axarr = plt.subplots(1,5)\n    for p in range(5):\n        idx = np.random.randint(0, len(dataset_show))\n        img, label = dataset_show[idx]\n        if use_meta:\n            img = img[0]\n        axarr[p].imshow(img.transpose(0, 1).transpose(1,2).squeeze())\n        axarr[p].set_title(str(label))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sigmoid = torch.nn.Sigmoid()\n\nclass Swish(torch.autograd.Function):\n    @staticmethod\n    def forward(ctx, i):\n        result = i * sigmoid(i)\n        ctx.save_for_backward(i)\n        return result\n    @staticmethod\n    def backward(ctx, grad_output):\n        i = ctx.saved_variables[0]\n        sigmoid_i = sigmoid(i)\n        return grad_output * (sigmoid_i * (1 + i * (1 - sigmoid_i)))\nswish = Swish.apply\n\nclass Swish_module(nn.Module):\n    def forward(self, x):\n        return swish(x)\nswish_layer = Swish_module()\n\nclass enetv2(nn.Module):\n    def __init__(self, backbone, out_dim, n_meta_features=0, load_pretrained=True):\n\n        super(enetv2, self).__init__()\n        self.n_meta_features = n_meta_features\n        self.enet = geffnet.create_model(enet_type.replace('-', '_'), pretrained=True)\n        self.dropout = nn.Dropout(0.5)\n        in_ch = self.enet.classifier.in_features\n        if n_meta_features > 0:\n            self.meta = nn.Sequential(\n                nn.Linear(n_meta_features, 512),\n                nn.BatchNorm1d(512),\n                Swish_module(),\n                nn.Dropout(p=0.3),\n                nn.Linear(512, 128),\n                nn.BatchNorm1d(128),\n                Swish_module(),\n            )\n            in_ch += 128\n        self.myfc = nn.Linear(in_ch, out_dim)\n        self.enet.classifier = nn.Identity()\n\n    def extract(self, x):\n        x = self.enet(x)\n        return x\n\n    def forward(self, x, x_meta=None):\n        x = self.extract(x).squeeze(-1).squeeze(-1)\n        if self.n_meta_features > 0:\n            x_meta = self.meta(x_meta)\n            x = torch.cat((x, x_meta), dim=1)\n            x = self.myfc(self.dropout(x))\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loss Function","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train & Valid Function","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_epoch(model, loader, optimizer):\n    model.train()\n    train_loss = []\n    bar = tqdm(loader)\n    for (data, target) in bar:\n        optimizer.zero_grad()\n        if use_meta:\n            data, meta = data\n            data, meta, target = data.to(device), meta.to(device), target.to(device)\n            logits = model(data, meta)\n        else:\n            data, target = data.to(device), target.to(device)\n            logits = model(data)\n        loss = criterion(logits, target)\n\n        if not use_amp:\n            loss.backward()\n        else:\n            with amp.scale_loss(loss, optimizer) as scaled_loss:\n                scaled_loss.backward()\n\n        optimizer.step()\n        loss_np = loss.detach().cpu().numpy()\n        train_loss.append(loss_np)\n        smooth_loss = sum(train_loss[-100:]) / min(len(train_loss), 100)\n        bar.set_description('loss: %.5f, smth: %.5f' % (loss_np, smooth_loss))\n    return train_loss\n\n\ndef get_trans(img, I):\n    if I >= 4:\n        img = img.transpose(2,3)\n    if I % 4 == 0:\n        return img\n    elif I % 4 == 1:\n        return img.flip(2)\n    elif I % 4 == 2:\n        return img.flip(3)\n    elif I % 4 == 3:\n        return img.flip(2).flip(3)\n\n    \ndef val_epoch(model, loader, is_ext=None, n_test=1, get_output=False):\n    model.eval()\n    val_loss = []\n    LOGITS = []\n    PROBS = []\n    TARGETS = []\n    with torch.no_grad():\n        for (data, target) in tqdm(loader):\n            \n            if use_meta:\n                data, meta = data\n                data, meta, target = data.to(device), meta.to(device), target.to(device)\n                logits = torch.zeros((data.shape[0], out_dim)).to(device)\n                probs = torch.zeros((data.shape[0], out_dim)).to(device)\n                for I in range(n_test):\n                    l = model(get_trans(data, I), meta)\n                    logits += l\n                    probs += l.softmax(1)\n            else:\n                data, target = data.to(device), target.to(device)\n                logits = torch.zeros((data.shape[0], out_dim)).to(device)\n                probs = torch.zeros((data.shape[0], out_dim)).to(device)\n                for I in range(n_test):\n                    l = model(get_trans(data, I))\n                    logits += l\n                    probs += l.softmax(1)\n            logits /= n_test\n            probs /= n_test\n\n            LOGITS.append(logits.detach().cpu())\n            PROBS.append(probs.detach().cpu())\n            TARGETS.append(target.detach().cpu())\n\n            loss = criterion(logits, target)\n            val_loss.append(loss.detach().cpu().numpy())\n\n    val_loss = np.mean(val_loss)\n    LOGITS = torch.cat(LOGITS).numpy()\n    PROBS = torch.cat(PROBS).numpy()\n    TARGETS = torch.cat(TARGETS).numpy()\n\n    if get_output:\n        return PROBS\n    else:\n        acc = (PROBS.argmax(1) == TARGETS).mean() * 100.\n        auc = roc_auc_score((TARGETS==mel_idx).astype(float), PROBS[:, mel_idx])\n        auc_20 = roc_auc_score((TARGETS[is_ext==0]==mel_idx).astype(float), PROBS[is_ext==0, mel_idx])\n        return val_loss, acc, auc, auc_20","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Fix Warmup Bug\nclass GradualWarmupSchedulerV2(GradualWarmupScheduler):\n    def __init__(self, optimizer, multiplier, total_epoch, after_scheduler=None):\n        super(GradualWarmupSchedulerV2, self).__init__(optimizer, multiplier, total_epoch, after_scheduler)\n    def get_lr(self):\n        if self.last_epoch > self.total_epoch:\n            if self.after_scheduler:\n                if not self.finished:\n                    self.after_scheduler.base_lrs = [base_lr * self.multiplier for base_lr in self.base_lrs]\n                    self.finished = True\n                return self.after_scheduler.get_lr()\n            return [base_lr * self.multiplier for base_lr in self.base_lrs]\n        if self.multiplier == 1.0:\n            return [base_lr * (float(self.last_epoch) / self.total_epoch) for base_lr in self.base_lrs]\n        else:\n            return [base_lr * ((self.multiplier - 1.) * self.last_epoch / self.total_epoch + 1.) for base_lr in self.base_lrs]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run(fold):\n    \n    i_fold = fold\n\n    if DEBUG:\n        df_this = df_train[df_train['fold'] != i_fold].sample(batch_size * 3)\n        df_valid = df_train[df_train['fold'] == i_fold].sample(batch_size * 3)\n    else:\n        df_this = df_train[df_train['fold'] != i_fold]\n        df_valid = df_train[df_train['fold'] == i_fold]\n\n    dataset_train = SIIMISICDataset(df_this,  'train', 'train', transform=transforms_train)\n    dataset_valid = SIIMISICDataset(df_valid, 'train', 'val', transform=transforms_val)\n    train_loader = torch.utils.data.DataLoader(dataset_train, batch_size=batch_size, sampler=RandomSampler(dataset_train), num_workers=num_workers)\n    valid_loader = torch.utils.data.DataLoader(dataset_valid, batch_size=batch_size, num_workers=num_workers)\n\n    model = enetv2(enet_type, n_meta_features=n_meta_features, out_dim=out_dim)\n    model = model.to(device)\n\n    auc_max = 0.\n    auc_20_max = 0.\n    model_file = f'{kernel_type}_best_fold{i_fold}.pth'\n    model_file2 = f'{kernel_type}_best_o_fold{i_fold}.pth'\n\n    optimizer = optim.Adam(model.parameters(), lr=init_lr)\n    if use_amp:\n        model, optimizer = amp.initialize(model, optimizer, opt_level=\"O1\")\n    scheduler_cosine = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, cosine_epo)\n    scheduler_warmup = GradualWarmupSchedulerV2(optimizer, multiplier=10, total_epoch=warmup_epo, after_scheduler=scheduler_cosine)\n\n    print(len(dataset_train), len(dataset_valid))\n\n    for epoch in range(1, n_epochs+1):\n        print(time.ctime(), 'Epoch:', epoch)\n        scheduler_warmup.step(epoch-1)\n\n        train_loss = train_epoch(model, train_loader, optimizer)\n        val_loss, acc, auc, auc_20 = val_epoch(model, valid_loader, is_ext=df_valid['is_ext'].values)\n\n        content = time.ctime() + ' ' + f'Fold {fold}, Epoch {epoch}, lr: {optimizer.param_groups[0][\"lr\"]:.7f}, train loss: {np.mean(train_loss):.5f}, valid loss: {(val_loss):.5f}, acc: {(acc):.4f}, auc: {(auc):.6f}, auc_20: {(auc_20):.6f}.'\n        print(content)\n        with open(f'log_{kernel_type}.txt', 'a') as appender:\n            appender.write(content + '\\n')\n\n        if auc > auc_max:\n            print('auc_max ({:.6f} --> {:.6f}). Saving model ...'.format(auc_max, auc))\n            torch.save(model.state_dict(), model_file)\n            auc_max = auc\n        if auc_20 > auc_20_max:\n            print('auc_20_max ({:.6f} --> {:.6f}). Saving model ...'.format(auc_20_max, auc_20))\n            torch.save(model.state_dict(), model_file2)\n            auc_20_max = auc_20\n\n    scores.append(auc_max)\n    scores_20.append(auc_20_max)\n    torch.save(model.state_dict(), os.path.join(f'{kernel_type}_model_fold{i_fold}.pth'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Run 5-Fold Training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"scores = []\nscores_20 = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for fold in range(5):\n    run(fold)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(scores)\nprint(scores_20)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Get OOF and CV score","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"PROBS = []\ndfs = []\n\nfor fold in range(5):\n    i_fold = fold\n\n    df_valid = df_train[df_train['fold'] == i_fold]\n    dataset_valid = SIIMISICDataset(df_valid, 'train', 'val', transform=transforms_val)\n    valid_loader = torch.utils.data.DataLoader(dataset_valid, batch_size=batch_size, num_workers=num_workers)\n\n    model = enetv2(enet_type, n_meta_features=n_meta_features, out_dim=out_dim)\n    model = model.to(device)\n    model_file = f'{kernel_type}_best_fold{i_fold}.pth'\n    model.load_state_dict(torch.load(model_file), strict=True)\n    model.eval()\n\n    this_PROBS = val_epoch(model, valid_loader, is_ext=df_valid['is_ext'].values, n_test=8, get_output=True)\n    PROBS.append(this_PROBS)\n    dfs.append(df_valid)\n\ndfs = pd.concat(dfs).reset_index(drop=True)\ndfs['pred'] = np.concatenate(PROBS).squeeze()[:, mel_idx]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Raw auc_all\nroc_auc_score(dfs['target'] == mel_idx, dfs['pred'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Rank per fold auc_all\ndfs2 = dfs.copy()\nfor i in range(5):\n    dfs2.loc[dfs2['fold']==i, 'pred'] = dfs2.loc[dfs2['fold']==i, 'pred'].rank(pct=True)\nroc_auc_score(dfs2['target'] == mel_idx, dfs2['pred'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Raw auc_2020\nroc_auc_score(dfs[dfs['is_ext']==0]['target']==mel_idx, dfs[dfs['is_ext']==0]['pred'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Rank per fold auc_2020\ndfs2 = dfs[dfs.is_ext==0].copy()\nfor i in range(5):\n    dfs2.loc[dfs2['fold']==i, 'pred'] = dfs2.loc[dfs2['fold']==i, 'pred'].rank(pct=True)\nroc_auc_score(dfs2['target'] == mel_idx, dfs2['pred'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predict","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"n_test = 8\ndataset_test = SIIMISICDataset(df_test, 'test', 'test', transform=transforms_val)\ntest_loader = torch.utils.data.DataLoader(dataset_test, batch_size=batch_size, num_workers=num_workers)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"OUTPUTS = []\n\nfor fold in range(5):\n    model = enetv2(enet_type, n_meta_features=n_meta_features, out_dim=out_dim)\n    model = model.to(device)\n    model.load_state_dict(torch.load(os.path.join('%s_best_fold%s.pth' % (kernel_type, fold))), strict=True)\n    model.eval()\n    \n    LOGITS = []\n    PROBS = []\n\n    with torch.no_grad():\n        for (data) in tqdm(test_loader):\n            \n            if use_meta:\n                data, meta = data\n                data, meta = data.to(device), meta.to(device)\n                logits = torch.zeros((data.shape[0], out_dim)).to(device)\n                probs = torch.zeros((data.shape[0], out_dim)).to(device)\n                for I in range(n_test):\n                    l = model(get_trans(data, I), meta)\n                    logits += l\n                    probs += l.softmax(1)\n            else:\n                data = data.to(device)\n                logits = torch.zeros((data.shape[0], out_dim)).to(device)\n                probs = torch.zeros((data.shape[0], out_dim)).to(device)\n                for I in range(n_test):\n                    l = model(get_trans(data, I))\n                    logits += l\n                    probs += l.softmax(1)\n            logits /= n_test\n            probs /= n_test\n    \n            LOGITS.append(logits.detach().cpu())\n            PROBS.append(probs.detach().cpu())\n\n    LOGITS = torch.cat(LOGITS).numpy()\n    PROBS = torch.cat(PROBS).numpy()\n    \n    OUTPUTS.append(PROBS[:, mel_idx])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Rank per fold\npred = np.zeros(OUTPUTS[0].shape[0])\nfor probs in OUTPUTS:\n    pred += pd.Series(probs).rank(pct=True).values\npred /= len(OUTPUTS)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submit to Kaggle","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test['target'] = pred\ndf_test[['image_name', 'target']].to_csv(f'submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}