{"cells":[{"metadata":{},"cell_type":"markdown","source":"Due to Limited resources i am doing inference only, model training code is there you can uncomment the calling of train function and run it.\n\nThis model includes EfficientNet with GridMask.\nMost of the code taken from https://www.kaggle.com/abhishek/melanoma-detection-with-pytorch/\nif you find it useful please upvote.\nThanks to Abhishek Thakur for awesome kernel.","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install wtfml\n!pip install pretrainedmodels","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!pip install --user --upgrade efficientnet-pytorch\n!pip install --user --upgrade albumentations","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport torch \nimport albumentations\n\nimport numpy as np\nimport pandas as pd\n\nimport torch.nn as nn\nfrom sklearn import metrics\nfrom sklearn import model_selection\nfrom torch.nn import functional as F\n\nfrom wtfml.utils import EarlyStopping\nfrom wtfml.engine import Engine\nfrom wtfml.data_loaders.image import ClassificationLoader\n\nimport pretrainedmodels\nfrom efficientnet_pytorch import EfficientNet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class EfficientNetB1(nn.Module):\n    def __init__(self, pretrained):\n        super(EfficientNetB1, self).__init__()\n\n        if pretrained is True:\n            self.base_model = EfficientNet.from_pretrained(\"efficientnet-b7\")\n        else:\n            self.base_model = EfficientNet.from_name(\"efficientnet-b7\")\n        \n        self.l0 = nn.Linear(2560, 1)\n\n    def forward(self, image, targets):\n        batch_size, _, _, _ = image.shape\n        x = self.base_model.extract_features(image)\n        x = F.adaptive_avg_pool2d(x,1).reshape(batch_size,-1)\n        out = self.l0(x)\n        loss = nn.BCEWithLogitsLoss()(out, targets.view(-1, 1).type_as(x))\n        return out, loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\n\nimport torch\n\nfrom PIL import Image\nfrom albumentations.core.transforms_interface import ImageOnlyTransform\nfrom albumentations.augmentations import functional as F1\nfrom PIL import Image, ImageOps, ImageEnhance\n\nfrom albumentations.core.transforms_interface import DualTransform\n\nclass GridMask(DualTransform):\n    \"\"\"GridMask augmentation for image classification and object detection.\n\n    Args:\n        num_grid (int): number of grid in a row or column.\n        fill_value (int, float, lisf of int, list of float): value for dropped pixels.\n        rotate ((int, int) or int): range from which a random angle is picked. If rotate is a single int\n            an angle is picked from (-rotate, rotate). Default: (-90, 90)\n        mode (int):\n            0 - cropout a quarter of the square of each grid (left top)\n            1 - reserve a quarter of the square of each grid (left top)\n            2 - cropout 2 quarter of the square of each grid (left top & right bottom)\n\n    Targets:\n        image, mask\n\n    Image types:\n        uint8, float32\n\n    Reference:\n    |  https://arxiv.org/abs/2001.04086\n    |  https://github.com/akuxcw/GridMask\n    \"\"\"\n\n    def __init__(self, num_grid=3, fill_value=0, rotate=0, mode=0, always_apply=False, p=0.5):\n        super(GridMask, self).__init__(always_apply, p)\n        if isinstance(num_grid, int):\n            num_grid = (num_grid, num_grid)\n        if isinstance(rotate, int):\n            rotate = (-rotate, rotate)\n        self.num_grid = num_grid\n        self.fill_value = fill_value\n        self.rotate = rotate\n        self.mode = mode\n        self.masks = None\n        self.rand_h_max = []\n        self.rand_w_max = []\n\n    def init_masks(self, height, width):\n        if self.masks is None:\n            self.masks = []\n            n_masks = self.num_grid[1] - self.num_grid[0] + 1\n            for n, n_g in enumerate(range(self.num_grid[0], self.num_grid[1] + 1, 1)):\n                grid_h = height / n_g\n                grid_w = width / n_g\n                this_mask = np.ones((int((n_g + 1) * grid_h), int((n_g + 1) * grid_w))).astype(np.uint8)\n                for i in range(n_g + 1):\n                    for j in range(n_g + 1):\n                        this_mask[\n                             int(i * grid_h) : int(i * grid_h + grid_h / 2),\n                             int(j * grid_w) : int(j * grid_w + grid_w / 2)\n                        ] = self.fill_value\n                        if self.mode == 2:\n                            this_mask[\n                                 int(i * grid_h + grid_h / 2) : int(i * grid_h + grid_h),\n                                 int(j * grid_w + grid_w / 2) : int(j * grid_w + grid_w)\n                            ] = self.fill_value\n                \n                if self.mode == 1:\n                    this_mask = 1 - this_mask\n\n                self.masks.append(this_mask)\n                self.rand_h_max.append(grid_h)\n                self.rand_w_max.append(grid_w)\n\n    def apply(self, image, mask, rand_h, rand_w, angle, **params):\n        h, w = image.shape[:2]\n        mask = F1.rotate(mask, angle) if self.rotate[1] > 0 else mask\n        mask = mask[:,:,np.newaxis] if image.ndim == 3 else mask\n        image *= mask[rand_h:rand_h+h, rand_w:rand_w+w].astype(image.dtype)\n        return image\n\n    def get_params_dependent_on_targets(self, params):\n        img = params['image']\n        height, width = img.shape[:2]\n        self.init_masks(height, width)\n\n        mid = np.random.randint(len(self.masks))\n        mask = self.masks[mid]\n        rand_h = np.random.randint(self.rand_h_max[mid])\n        rand_w = np.random.randint(self.rand_w_max[mid])\n        angle = np.random.randint(self.rotate[0], self.rotate[1]) if self.rotate[1] > 0 else 0\n\n        return {'mask': mask, 'rand_h': rand_h, 'rand_w': rand_w, 'angle': angle}\n\n    @property\n    def targets_as_params(self):\n        return ['image']\n\n    def get_transform_init_args_names(self):\n        return ('num_grid', 'fill_value', 'rotate', 'mode')\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# create train folds\ndf = pd.read_csv(\"../input/siim-isic-melanoma-classification/train.csv\")\ndf[\"kfold\"] = -1\ndf = df.sample(frac=1).reset_index(drop=True)\ny = df.target.values\nkf = model_selection.StratifiedKFold(n_splits=5)\nfor f, (t_, v_) in enumerate(kf.split(X=df, y=y)):\n    df.loc[v_, 'kfold'] = f\n\ndf.to_csv(\"train_folds.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(fold):\n    training_data_path = \"../input/siic-isic-224x224-images/train/\"\n    df = pd.read_csv(\"/kaggle/working/train_folds.csv\")\n    device = \"cuda\"\n    epochs = 1  #increase it to 50\n    train_bs = 8\n    valid_bs = 4\n    df_train = df[df.kfold != fold].reset_index(drop=True)\n    df_valid = df[df.kfold == fold].reset_index(drop=True)\n    model = EfficientNetB1(pretrained=True)\n    model.to(device)\n    mean = (0.485, 0.456, 0.406)\n    std = (0.229, 0.224, 0.225)\n    train_aug = albumentations.Compose(\n    [\n        albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True),\n        albumentations.ShiftScaleRotate(shift_limit=0.0625,\n                                        scale_limit=0.1, rotate_limit=15),\n        albumentations.Flip(p=0.5),\n        albumentations.OneOf([\n                    GridMask(num_grid=3, mode=0, rotate=15),\n                    GridMask(num_grid=3, mode=2, rotate=15),\n                ], p=0.75)\n    ]\n    )\n    \n    valid_aug = albumentations.Compose(\n    [\n        albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True)\n        \n    ]\n    )\n    train_images = df_train.image_name.values.tolist()\n    train_images = [os.path.join(training_data_path, i+\".png\" ) for i in train_images]\n    train_targets = df_train.target.values\n    valid_images = df_valid.image_name.values.tolist()\n    valid_images = [os.path.join(training_data_path, i+\".png\") for i in valid_images]\n    valid_targets = df_valid.target.values\n    \n    train_dataset = ClassificationLoader(\n            image_paths = train_images,\n            targets = train_targets,\n            resize = None,\n            augmentations = train_aug,\n    )\n    train_loader = torch.utils.data.DataLoader(\n        train_dataset, batch_size=train_bs, shuffle=True, num_workers=4\n    )\n    \n    valid_dataset = ClassificationLoader(\n        image_paths = valid_images,\n        targets = valid_targets,\n        resize = None,\n        augmentations = valid_aug,\n    )\n    \n    valid_loader = torch.utils.data.DataLoader(\n        valid_dataset, batch_size = valid_bs, shuffle=False, num_workers=4\n    )\n    \n    optimizer = torch.optim.Adam(model.parameters(),lr=1e-4)\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n                optimizer,\n                patience=3,\n                threshold=0.001,\n                mode=\"max\",\n    )\n    \n    es = EarlyStopping(patience=5, mode=\"max\")\n    for epoch in range(epochs):\n        train_loss = Engine.train(train_loader, model, optimizer, device=device)\n        predictions, valid_loss = Engine.evaluate(valid_loader, model, device=device)\n        predictions = np.vstack((predictions)).ravel()\n        auc = metrics.roc_auc_score(valid_targets, predictions)\n        print(f\"Epoch = {epoch}, AUC = {auc}\")\n        scheduler.step(auc)\n        es(auc, model,model_path=f\"model_fold_{fold}.bin\")\n        if es.early_stop:\n            print(\"Early Stopping\")\n            break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict(fold):\n    test_data_path = \"../input/siic-isic-224x224-images/test/\"\n    df = pd.read_csv(\"../input/siim-isic-melanoma-classification/test.csv\")\n    device = \"cuda\"\n    if fold == 1:\n        model_path = f\"../input/pytorch-efficientnet-gridmask/model_fold_{fold}.bin\"\n    else:\n        model_path = f\"../input/modelsmelanoma/model_fold_{fold}.bin\"\n    mean = (0.485, 0.456, 0.406)\n    std = (0.229, 0.224, 0.225)\n    aug = albumentations.Compose(\n        [\n            albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True)\n        ]\n    )\n    images = df.image_name.values.tolist()\n    images = [os.path.join(test_data_path, i+\".png\") for i in images]\n    targets = np.zeros(len(images))\n    test_dataset = ClassificationLoader(\n        image_paths = images,\n        targets = targets,\n        resize = None,\n        augmentations = aug,\n    )\n    test_loader = torch.utils.data.DataLoader(\n            test_dataset, batch_size=16, shuffle=False,\n            num_workers=4\n    )\n    model = EfficientNetB1(pretrained=False)\n    model.load_state_dict(torch.load(model_path),strict=False)\n    model.to(device)\n    \n    predictions = Engine.predict(test_loader, model, device=device)\n    predictions = np.vstack((predictions)).ravel()\n    return predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train(0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Uncomment the below calling of train function.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# train(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train(4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"p1 = predict(0)\np2 = predict(1)\np3 = predict(2)\np4 = predict(3)\np5 = predict(4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = (p1+p2+p3+p4+p5)/5 \nsample = pd.read_csv(\"../input/siim-isic-melanoma-classification/sample_submission.csv\")\nsample.loc[:,\"target\"] = predictions\nsample.to_csv(\"submission.cv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}