{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n'''\nimport os\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))'''\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"! pip install pretrainedmodels\n! pip install wtfml","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"! pip install efficientnet_pytorch torchtoolbox\n! curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n! python pytorch-xla-env-setup.py --version nightly --apt-packages libomp5 libopenblas-dev","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import gc\nimport torch\nimport torchvision\nimport albumentations\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader, Subset\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom sklearn.metrics import accuracy_score, roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\nfrom tqdm import tqdm_notebook, tqdm\nimport pretrainedmodels\nfrom wtfml.utils import EarlyStopping\nfrom efficientnet_pytorch import EfficientNet\n#device = torch.device('cuda') if torch.cuda.is_available() else 'cpu'\n#print(\"Imported required packages. Using device: {}\".format(device))\nimport torch_xla.core.xla_model as xm\nimport torch_xla.distributed.parallel_loader as pl\nimport torch_xla.distributed.xla_multiprocessing as xmp","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Hair Removal technique","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\nimport matplotlib.pyplot as plt\nimport pandas as pd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BASE_PATH = '../input/siim-isic-melanoma-classification'\nhair_images =['ISIC_0078712','ISIC_0080817','ISIC_0082348','ISIC_0109869','ISIC_0155012','ISIC_0159568','ISIC_0164145','ISIC_0194550','ISIC_0194914','ISIC_0202023']\nwithout_hair_images = ['ISIC_0015719','ISIC_0074268','ISIC_0075914','ISIC_0084395','ISIC_0085718','ISIC_0081956']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"'''fig = plt.figure(figsize=(20,30))\nl = len(hair_images)\n# Plot different stages of transformation\nfor i, image_name in enumerate(hair_images):\n    image = cv2.imread(BASE_PATH+'/jpeg/train/'+image_name+'.jpg')\n    resized_img = cv2.resize(image, (512,512))\n    #original image\n    plt.subplot(l, 5, (i*5)+1)\n    plt.imshow(cv2.cvtColor(resized_img, cv2.COLOR_BGR2RGB))\n    plt.axis('off')\n    plt.title('Original Image')\n    \n    # gray image\n    plt.subplot(l, 5, (i*5)+2)\n    gray_image = cv2.cvtColor(resized_img, cv2.COLOR_BGR2GRAY)\n    plt.imshow(gray_image)\n    plt.axis('off')\n    plt.title('Gray Image')\n    \n    # blackhat\n    kernel = cv2.getStructuringElement(1, (17,17))\n    plt.subplot(l, 5, (i*5)+3)\n    black_hat = cv2.morphologyEx(gray_image, cv2.MORPH_BLACKHAT, kernel)\n    plt.imshow(black_hat)\n    plt.axis('off')\n    plt.title('Blackhat Image')\n    \n    # Intensify the hair contours\n    plt.subplot(l, 5, (i*5)+4)\n    retval, intense_hair = cv2.threshold(black_hat, 10, 255, cv2.THRESH_BINARY)\n    plt.imshow(intense_hair)\n    plt.axis('off')\n    plt.title('Intense hair Image')\n    \n    # Inpaint the hair region with neighbouring pixels\n    plt.subplot(l, 5, (i*5)+5)\n    hair_removed = cv2.inpaint(resized_img, intense_hair, 1, cv2.INPAINT_TELEA)\n    plt.imshow(cv2.cvtColor(hair_removed, cv2.COLOR_BGR2RGB))\n    plt.axis('off')\n    plt.title('Hair removed image')'''","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Wrap the above in a function","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def remove_hair(image):\n    #fig = plt.figure(figsize=(20,30))\n    #l = len(hair_images)\n    # Plot different stages of transformation\n    #transformed_images = []\n    #image = cv2.imread(BASE_PATH+'/jpeg/train/'+image+'.jpg')\n    #resized_img = cv2.resize(image, (128,128))\n\n    gray_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n\n    kernel = cv2.getStructuringElement(1, (17,17))\n    black_hat = cv2.morphologyEx(gray_image, cv2.MORPH_BLACKHAT, kernel)\n\n    # Intensify the hair contours\n    retval, intense_hair = cv2.threshold(black_hat, 10, 255, cv2.THRESH_BINARY)\n\n    # Inpaint the hair region with neighbouring pixels\n    hair_removed = cv2.inpaint(image, intense_hair, 1, cv2.INPAINT_TELEA)\n    #transformed_images.append(hair_removed)\n    return hair_removed","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''%%time\nhair_image = 'ISIC_0078712'\nhair_removed_images = remove_hair(hair_image)'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(hair_removed_images[0].shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# To use TPU\ntry:\n    import torch_xla.core.xla_model as xm \n    import torch_xla.distributed.parallel_loader as pl\n    _xla_available = True\nexcept ImportError:\n    _xla_available = False\nprint('TPU available: ',_xla_available)\n#print(\"Imported required packages. Using device: {}\".format(device))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''def reduce_fn(vals):\n    return sum(vals) / len(vals)'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"warnings.simplefilter('ignore')\ntorch.manual_seed(42)\nnp.random.seed(42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BASE_DIR = \"../input/siim-isic-melanoma-classification/\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls -lrt \"../input/siimisic-melanoma-resized-images\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"npy_data = np.load(\"../input/siimisic-melanoma-resized-images/x_train_96.npy\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataloader Class","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class MelanomaDataLoader(Dataset):\n    '''Dataloader class'''\n    def __init__(self, npy_data, targets, augmentations=None):\n        self.npy_data = npy_data\n        self.targets = targets\n        self.augmentations = augmentations\n        \n    def __len__(self):\n        return len(self.npy_data)\n    \n    def __getitem__(self, idx):\n        \n        np_img = self.npy_data[idx]\n        np_img = remove_hair(np_img)\n        target = self.targets[idx]\n        if self.augmentations:\n            augmented = self.augmentations(image=np_img)\n            image_data = augmented['image']\n        else:\n            image_data = torch.from_numpy(np_img)\n        image_data = np.transpose(image_data, (2,0,1)).astype(np.float32)\n        return {\n            'images': torch.tensor(image_data, dtype=torch.float),\n            'targets': torch.tensor(target, dtype=torch.long)\n        }","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Class","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class SEResnext50_32x4d(nn.Module):\n    '''This is network class'''\n    def __init__(self, pretrained='imagenet', wp = None):\n        super(SEResnext50_32x4d, self).__init__()\n        \n        self.base_model = pretrainedmodels.__dict__['se_resnext50_32x4d'](pretrained=None)\n        #print(self.base_model)\n        if pretrained is not None:\n            self.base_model.load_state_dict(\n            torch.load('../input/pretrained-model-weights-pytorch/se_resnext50_32x4d-a260b3a4.pth')\n            )\n        '''for params in self.base_model.parameters():\n            params.requires_grad = False'''\n            \n        self.l0 = nn.Linear(2048, 1)\n        if wp is not None:\n            self.criterion = nn.BCEWithLogitsLoss(pos_weight=wp)\n        else:\n            self.criterion = nn.BCEWithLogitsLoss()\n        \n    def forward(self, images, targets):\n        batch_size = images.shape[0]\n        \n        x = self.base_model.features(images)\n        x = F.adaptive_avg_pool2d(x, 1).reshape(batch_size, -1)\n        yhat = self.l0(x)\n        #loss = nn.BCEWithLogitsLoss(pos_weight=wp)(yhat, targets.view(-1, 1).type_as(x))\n        loss = self.criterion(yhat, targets.view(-1, 1).type_as(x))\n        return yhat, loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class EfNet(nn.Module):\n    '''This is network class'''\n    def __init__(self, pretrained='imagenet', wp = None):\n        super(EfNet, self).__init__()\n        \n        self.base_model = EfficientNet.from_pretrained('efficientnet-b0')\n        self.base_model._fc = nn.Linear(1280, 1, bias=True)\n        \n        '''self.meta = nn.Sequential(\n                        nn.BatchNorm1d(500),\n                        nn.ReLU(),\n                        nn.Dropout(0.4),\n                        nn.Linear(500,100, bias=True),\n                        nn.BatchNorm1d(100),\n                        nn.ReLU(),\n                        nn.Dropout(0.4),\n                        nn.Linear(100,1, bias=True))'''\n        if wp is not None:\n            self.criterion = nn.BCEWithLogitsLoss(pos_weight=wp)\n        else:\n            self.criterion = nn.BCEWithLogitsLoss()\n        \n    def forward(self, images, targets):\n        batch_size = images.shape[0]\n        \n        yhat = self.base_model(images)\n        #x = F.adaptive_avg_pool2d(x, 1).reshape(batch_size, -1)\n        #yhat = self.l0(x)\n        #loss = nn.BCEWithLogitsLoss(pos_weight=wp)(yhat, targets.view(-1, 1).type_as(x))\n        #yhat = self.meta(x)\n        loss = self.criterion(yhat, targets.view(-1, 1).type_as(yhat))\n        return yhat, loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"efnet = EfNet(pretrained='Imagenet',wp=torch.tensor(0))\n#efnet._fc = nn.Linear(1280, 1)\nprint(efnet)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for param in efnet.parameters():\n    if param.requires_grad:\n        print(param.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls -lrt ../input/siim-isic-melanoma-classification/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create folds\ndf = pd.read_csv(BASE_DIR+'train.csv')\ndf['fold'] = -1\n#df = df.sample(frac=1).reset_index(drop=True)\ny = df.target.values\n\nkfolds = StratifiedKFold(n_splits=5)\n\nfor fold, (t_, v_) in enumerate(kfolds.split(X=df, y=y)):\n    df.loc[v_, 'fold'] = fold\n    \ndf.to_csv('./train_new.csv', index=False)\nprint(df.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_new = pd.read_csv('./train_new.csv')\nprint(train_new.head())\ntrain_new.fold.value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create Dataloaders","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fold = 2\ntrain_indices = train_new[train_new.fold != fold].index.to_numpy()\nval_indices = train_new[train_new.fold == fold].index.to_numpy()\nprint(len(train_indices), len(val_indices))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_indices[:5])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(train_indices)+len(val_indices))\nprint(len(npy_data))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(set(train_indices).intersection(val_indices))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fold=1\ntrain_npy = npy_data[train_indices]\nval_npy = npy_data[val_indices]\ntrain_targets = train_new[train_new.fold != fold]['target'].to_numpy()\nval_targets = train_new[train_new.fold == fold]['target'].to_numpy()\nprint(sum(train_targets==0))\nprint(sum(train_targets==1))\nprint(len(train_targets))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#fold=0\ntrain_unique, train_counts = np.unique(train_targets, return_counts=True)\nval_unique, val_counts = np.unique(val_targets, return_counts=True)\nprint(f\"Train counts: {train_unique} {train_counts} ********* Val counts: {val_unique} {val_counts}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#fold=1\ntrain_unique, train_counts = np.unique(train_targets, return_counts=True)\nval_unique, val_counts = np.unique(val_targets, return_counts=True)\nprint(f\"Train counts: {train_unique} {train_counts} ********* Val counts: {val_unique} {val_counts}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"There are {len(train_npy)} train data and {len(train_targets)} train targets. val count: {len(val_npy)}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#mean = (0.485, 0.456, 0.406)\n#std = (0.229, 0.224, 0.225)\nmean = (0.5,0.5,0.5)\nstd = (0.5,0.5,0.5)\ntrain_aug = albumentations.Compose([\n    albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True),\n    albumentations.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.1, rotate_limit=15),\n    albumentations.Flip(p=0.5)\n])\nvalid_aug = albumentations.Compose([\n    albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True)\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data = MelanomaDataLoader(train_npy, train_targets, augmentations=train_aug)\nval_data = MelanomaDataLoader(val_npy, val_targets, augmentations=valid_aug)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_loader = DataLoader(train_data, batch_size=4, shuffle=True)\nval_loader = DataLoader(val_data, batch_size=4, shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(train_loader.dataset))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Basic Visualization","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_batch = next(iter(train_loader))\nprint(train_batch['images'].shape, train_batch['targets'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def imshow(img, title):\n    plt.figure(figsize=(10,5))\n    np_img = img.numpy() / 2 + 0.5\n    plt.axis('off')\n    plt.imshow(np.transpose(np_img, (1,2,0)))\n    plt.title(title)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_image_batches(data_loader):\n    batch = next(iter(data_loader))\n    imgs, labels = batch['images'], batch['targets']\n    print(\"img shape: \",batch['images'].shape)\n    imgs = torchvision.utils.make_grid(imgs)\n    title = labels.numpy().tolist()\n    imshow(imgs, title)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_image_batches(train_loader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_image_batches(val_loader)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataloader time check","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"BS = 16\ntrain_loader = DataLoader(train_data, batch_size=BS, shuffle=True)\nval_loader = DataLoader(val_data, batch_size=BS, shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nbatch = next(iter(train_loader))\nprint(batch['images'].shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(fold, use_tpu=False, net='se_resnext'):\n    epochs = 25\n    BS = 64\n    lr = 0.0001\n    device = xm.xla_device()\n    if use_tpu:\n        device = xm.xla_device()\n    #else:\n    #    device = 'cuda' if torch.cuda.is_available() else 'cpu'\n    #Dataloader prep steps\n    train_indices = train_new[train_new.fold != fold].index.to_numpy()\n    val_indices = train_new[train_new.fold == fold].index.to_numpy()\n    train_npy = npy_data[train_indices]\n    val_npy = npy_data[val_indices]\n    train_targets = train_new[train_new.fold != fold]['target'].to_numpy()\n    val_targets = train_new[train_new.fold == fold]['target'].to_numpy()\n    \n    #let's check target distribution in this fold\n    train_unique, train_counts = np.unique(train_targets, return_counts=True)\n    val_unique, val_counts = np.unique(val_targets, return_counts=True)\n    print(f\"Train counts: {train_unique} {train_counts} ********* Val counts: {val_unique} {val_counts}\")\n    \n    mean = (0.485, 0.456, 0.406)\n    std = (0.229, 0.224, 0.225)\n    train_aug = albumentations.Compose([\n        albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True),\n        albumentations.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.1, rotate_limit=15),\n        albumentations.Flip(p=0.5)\n    ])\n    valid_aug = albumentations.Compose([\n        albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True)\n    ])\n    train_data = MelanomaDataLoader(train_npy, train_targets, augmentations=train_aug)\n    val_data = MelanomaDataLoader(val_npy, val_targets, augmentations=valid_aug)\n    train_loader = DataLoader(train_data, batch_size=BS, shuffle=True)\n    val_loader = DataLoader(val_data, batch_size=BS, shuffle=False)\n    \n    wp = sum(train_targets==0) / sum(train_targets)\n    fold_wp = torch.tensor(wp, dtype=torch.float)\n    #modelling\n    \n    if 'ef' in net:\n        model = EfNet(pretrained='imagenet', wp=fold_wp)\n    else:\n        model = SEResnext50_32x4d(pretrained='imagenet', wp=fold_wp)\n    \n    model.to(device)\n    \n    '''for param in model.parameters():\n        if param.requires_grad:\n            print(param.shape)'''\n    \n    optimizr = torch.optim.Adam(model.parameters(), lr=lr)\n    schedulr = torch.optim.lr_scheduler.ReduceLROnPlateau(\n        optimizr, patience=3, threshold=0.001, mode=\"max\"\n        )\n    es = EarlyStopping(patience=5, mode='max')\n    best_auc = 0\n    losses = []\n    n_iter = len(train_indices) // BS\n    \n    for epoch in range(epochs):\n        model.train()\n        if use_tpu:\n            pl_loader = pl.ParallelLoader(train_loader, [device])\n            tk0 = tqdm(\n                pl_loader.per_device_loader(device),\n                total=len(train_loader))\n        else:\n            tk0 = tqdm(train_loader, total=len(train_loader))\n                    \n        for i, data in enumerate(tk0, 1):\n            images, targets = data['images'], data['targets']\n            images, targets = images.to(device), targets.to(device)\n            \n            optimizr.zero_grad()\n            #batch_wp = sum(targets==0) / sum(targets)\n            out, loss = model(images, targets)\n            \n            loss.backward()\n            \n            if use_tpu:\n                xm.optimizer_step(optimizr)\n            else:\n                optimizr.step()\n            \n            optimizr.zero_grad()\n            #train_unique, train_counts = np.unique(targets.cpu().numpy(), return_counts=True)\n            #print(f\"Train counts: {train_unique} {train_counts}\")\n            #print(\"Loss for batch: {} is {}\".format(i+1, loss.item()))\n            \n            torch.cuda.empty_cache()\n            \n            #if i%50 == 0:\n            #print(f\"Batch {i} contains {sum(targets)} positive labels\")\n            #print(\"Evaluating model...\")\n            #print(\"Epoch: %d ******* Iter: %d/%d ******* Loss: %0.2f VAL_AUC: %0.2f\"%(epoch, i, n_iter, loss.item(), val_auc))\n            '''if val_auc > best_auc:\n                print(\"Max AUC attained, saving model..\")\n                torch.save(model.state_dict(), './siimModel_{}.pth'.format(fold))\n                best_auc = val_auc'''\n            \n            del images, targets\n            \n        \n        val_auc = evaluate(val_loader, val_targets, model, device, use_tpu)\n        print(\"Epoch: %d ******* VAL_AUC: %0.2f\"%(epoch, val_auc))\n        schedulr.step(val_auc)\n        es(val_auc, model, model_path=f\"./melanoma_fold_{fold}.bin\")\n        \n        '''if val_auc > best_auc:\n            print(\"Max AUC attained, saving model..\")\n            torch.save(model.state_dict(), './siimModel_{}.pth'.format(fold))\n            best_auc = val_auc'''\n            \n        if es.early_stop:\n            print(\"Early Stopping..\")\n            break\n        gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def evaluate(data_loader, val_targets, model, device, use_tpu=False):\n    model = model.to(device)\n    model.eval()\n    final_preds = []\n    with torch.no_grad():\n        if use_tpu:\n            pl_loader = pl.ParallelLoader(data_loader, [device])\n            tk0 = tqdm(pl_loader.per_device_loader(device), total = len(data_loader))\n        else:\n            tk0 = tqdm(data_loader, total=len(data_loader))\n        for i, data in enumerate(tk0):\n            images, targets = data['images'], data['targets']\n            images, targets = images.to(device), targets.to(device)\n            batch_wp = sum(targets==0) / sum(targets)\n            \n            preds, _ = model(images, targets)\n            final_preds.append(preds.cpu())\n    predictions = np.vstack((final_preds)).ravel()\n    print('val_targets: ',val_targets[:5])\n    print('predictions: ',predictions[:5])\n    \n    auc = roc_auc_score(val_targets, predictions)\n    return auc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''def _mp_fn(rank, flags):\n    torch.set_default_tensor_type('torch.FloatTensor')\n    a = train(fold=0, use_tpu=True)'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''FLAGS={}\nxmp.spawn(_mp_fn, args=(FLAGS,), nprocs=8, start_method='fork')'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train(0, use_tpu=True, net='efnet')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train(1, use_tpu=True, net='efnet')\ntrain(2, use_tpu=True, net='efnet')\ntrain(3, use_tpu=True, net='efnet')\ntrain(4, use_tpu=True, net='efnet')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls -lrt ../input/melanoma-pytorch-starter/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"npy_test = np.load(\"../input/siimisic-melanoma-resized-images/x_test_64.npy\")\nprint(f\"There are {len(npy_test)} images in test set\")\nprint(npy_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict(fold):\n    BS = 4\n    mean = (0.485, 0.456, 0.406)\n    std = (0.229, 0.224, 0.225)\n    test_aug = albumentations.Compose([\n        albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True)\n    ])\n    #just for the sake\n    test_targets = np.zeros(len(npy_test))\n    test_wp = torch.tensor(1, dtype=torch.float)\n    \n    test_data = MelanomaDataLoader(npy_test, test_targets, augmentations=test_aug)\n    test_loader = DataLoader(test_data, batch_size=BS, shuffle=False)\n    \n    if 'ef' in net:\n        model = EfNet(pretrained=None, wp=fold_wp)\n    else:\n        model = SEResnext50_32x4d(pretrained=None, wp=fold_wp)\n    \n    #model = SEResnext50_32x4d(pretrained=None, wp=test_wp)\n    print(f\"Loading from model: melanoma_fold_{fold}.bin\")\n    model.load_state_dict(torch.load(f\"../input/melanoma-pytorch-starter/melanoma_fold_{fold}.bin\"))\n    model = model.to(device)\n    model.eval()\n    \n    test_preds = []\n    #tk1 = tqdm(test_loader, total = len(test_loader))\n    with torch.no_grad():\n        for batch, data in enumerate(test_loader, 1):\n            torch.cuda.empty_cache()\n            images, targets = data['images'], data['targets']\n            images, targets = images.to(device), targets.to(device)\n            out, _ = model(images, targets)\n            #test_preds.append(out)\n            test_preds.append(out.cpu())\n            del images, targets\n    predictions = np.vstack(test_preds).ravel()\n    return predictions        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"p1 = predict(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(type(p1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"p0 = predict(0)\np2 = predict(2)\np3 = predict(3)\np4 = predict(4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#submission\npredictions = (p0 + p1 + p2 + p3 + p4) / 5\nsubmission_df = pd.read_csv('../input/siim-isic-melanoma-classification/sample_submission.csv')\nsubmission_df.loc[:, 'target'] = predictions\nprint(submission_df.head())\nsubmission_df.to_csv('submission_file.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Check roc_auc\ntargets = np.zeros(10)\ntargets[8] = 1\nprint(targets)\npreds = (np.random.rand(10)*0.1).ravel()\npreds[8] = -0.1\nprint(preds)\nauc = roc_auc_score(targets, preds)\nprint(auc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# check loss\nout = np.array([-2.9445, -3.8510, -8.5114, 3.1692, 1.6949, -5.5680, -9.3456, -6.9603, -5.7006, -9.9718])\n#out = (np.random.rand(10)*-10)\nout = torch.from_numpy(out)\n#out = out.view(-1,1)\nprint((out))\ntargets = torch.zeros(10)\ntargets[9] = 1\n#targets = targets.view(-1,1)\nprint((targets))\nprint(targets.shape, out.shape)\nwp = torch.tensor(9/1, dtype=torch.float)\nloss = nn.BCEWithLogitsLoss(pos_weight=wp)(out, targets)\nprint(loss)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# check loss\nout = np.array([-2.9445, -3.8510, -8.5114, -3.1692, -1.6949, -5.5680, -9.3456, -6.9603, -5.7006, -5.9718])\n#out = (np.random.rand(10)*-10)\nout = torch.from_numpy(out)\n#out = out.view(-1,1)\nprint((out))\ntargets = torch.zeros(10)\ntargets[9] = 1\n#targets = targets.view(-1,1)\nprint((targets))\nprint(targets.shape, out.shape)\nloss = nn.BCEWithLogitsLoss()(out, targets)\nprint(loss)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}