{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"!pip install -U git+https://github.com/albumentations-team/albumentations > /dev/null\n!pip install timm > /dev/null\n!pip install pytorch_toolbelt > /dev/null\n!pip install tensorboardX > /dev/null\n!pip install catalyst==20.4.2 > /dev/null\n\n!pip install torch==1.6.0+cu101 torchvision==0.7.0+cu101 -f https://download.pytorch.org/whl/torch_stable.html > /dev/null","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"IMAGE_SIZE = \"384x384\"\n\nimport pandas as pd\n\ntrain_csv = pd.read_csv(f\"../input/melanomaprocesseddf/train_c30_v1.csv\")\ntrain_2019_csv = pd.read_csv(f\"../input/melanomaprocesseddf/train2019_c30_v1.csv\")\nmalignant_csv = pd.read_csv(f\"../input/melanomaprocesseddf/malignant_c30_v1.csv\")\ntest_csv = pd.read_csv(f\"../input/melanomaprocesseddf/test_c30_v1.csv\")\n\ntrain_image_path = f\"../input/jpeg-melanoma-{IMAGE_SIZE}/train\"\ntest_image_path = f\"../input/jpeg-melanoma-{IMAGE_SIZE}/test\"\ntrain_2019_path = f\"../input/jpeg-isic2019-{IMAGE_SIZE}/train\"\nmalignant_path = f\"../input/malignant-v2-{IMAGE_SIZE}/jpeg384\"\n\ntrain_csv[\"image_path\"] = train_csv[\"image_name\"].apply(lambda x: f\"{train_image_path}/{x}.jpg\") \ntest_csv[\"image_path\"] = test_csv[\"image_name\"].apply(lambda x: f\"{test_image_path}/{x}.jpg\")\ntrain_2019_csv[\"image_path\"] = train_2019_csv[\"image_name\"].apply(lambda x: f\"{train_2019_path}/{x}.jpg\") \nmalignant_csv[\"image_path\"] = malignant_csv[\"image_name\"].apply(lambda x: f\"{malignant_path}/{x}.jpg\")\n\ntrain_csv.to_csv(\"train_csv.csv\", index=False)\ntrain_2019_csv.to_csv(\"train_2019_csv.csv\", index=False)\nmalignant_csv.to_csv(\"malignat_csv.csv\", index=False)\ntest_csv.to_csv(\"test_csv.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"%%writefile classifiers.py\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nfrom functools import partial\n\nimport numpy as np\nimport torch\nfrom timm.models import skresnext50_32x4d\nfrom timm.models import dpn92, dpn131\nfrom timm.models.dpn import dpn92, dpn131\nfrom timm.models.efficientnet import tf_efficientnet_b4_ns, tf_efficientnet_b3_ns, \\\n    tf_efficientnet_b5_ns, tf_efficientnet_b2_ns, tf_efficientnet_b6_ns, tf_efficientnet_b7_ns, tf_efficientnet_b0_ns\n#from timm.models.senet import seresnext50_32x4d\nfrom timm.models.densenet import densenet201\nfrom torch import nn\nfrom torch.nn.modules.dropout import Dropout\nfrom torch.nn.modules.linear import Linear\nfrom torch.nn.modules.pooling import AdaptiveAvgPool2d, AdaptiveMaxPool2d\n\nencoder_params = {\n    \"densenet201\" : {\n        \"features\": 1920,\n        \"init_op\": partial(densenet201, pretrained=True)\n    },\n    \"dpn92\" : {\n        \"features\": 2688,\n        \"init_op\": partial(dpn92, pretrained=True)\n    },\n    \"dpn131\": {\n        \"features\": 2688,\n        \"init_op\": partial(dpn131, pretrained=True)\n    },\n    \"tf_efficientnet_b0_ns\": {\n        \"features\": 1280,\n        \"init_op\": partial(tf_efficientnet_b0_ns, pretrained=True, drop_path_rate=0.2)\n    },\n    \"tf_efficientnet_b3_ns\": {\n        \"features\": 1536,\n        \"init_op\": partial(tf_efficientnet_b3_ns, pretrained=True, drop_path_rate=0.2)\n    },\n    \"tf_efficientnet_b2_ns\": {\n        \"features\": 1408,\n        \"init_op\": partial(tf_efficientnet_b2_ns, pretrained=True, drop_path_rate=0.2)\n    },\n    \"tf_efficientnet_b4_ns\": {\n        \"features\": 1792,\n        \"init_op\": partial(tf_efficientnet_b4_ns, pretrained=True, drop_path_rate=0.5)\n    },\n    \"tf_efficientnet_b5_ns\": {\n        \"features\": 2048,\n        \"init_op\": partial(tf_efficientnet_b5_ns, pretrained=True, drop_path_rate=0.2)\n    },\n    \"tf_efficientnet_b4_ns_03d\": {\n        \"features\": 1792,\n        \"init_op\": partial(tf_efficientnet_b4_ns, pretrained=True, drop_path_rate=0.3)\n    },\n    \"tf_efficientnet_b5_ns_03d\": {\n        \"features\": 2048,\n        \"init_op\": partial(tf_efficientnet_b5_ns, pretrained=True, drop_path_rate=0.3)\n    },\n    \"tf_efficientnet_b5_ns_04d\": {\n        \"features\": 2048,\n        \"init_op\": partial(tf_efficientnet_b5_ns, pretrained=True, drop_path_rate=0.4)\n    },\n    \"tf_efficientnet_b6_ns\": {\n        \"features\": 2304,\n        \"init_op\": partial(tf_efficientnet_b6_ns, pretrained=True, drop_path_rate=0.2)\n    },\n    \"tf_efficientnet_b7_ns\": {\n        \"features\": 2560,\n        \"init_op\": partial(tf_efficientnet_b7_ns, pretrained=True, drop_path_rate=0.2)\n    },\n    \"tf_efficientnet_b6_ns_04d\": {\n        \"features\": 2304,\n        \"init_op\": partial(tf_efficientnet_b6_ns, pretrained=True, drop_path_rate=0.4)\n    },\n    #\"se50\": {\n    #    \"features\": 2048,\n    #    \"init_op\": partial(seresnext50_32x4d, pretrained=True)\n    #},\n    \"sk50\": {\n        \"features\": 2048,\n        \"init_op\": partial(skresnext50_32x4d, pretrained=True)\n    },\n\n}\n\n\nclass MelanomaClassifier(nn.Module):\n    def __init__(self, encoder, dropout_rate=0.0) -> None:\n        super().__init__()\n        self.encoder = encoder_params[encoder][\"init_op\"]()\n        self.avg_pool = AdaptiveAvgPool2d((1, 1))\n        self.dropout = Dropout(dropout_rate)\n        self.fc = Linear(encoder_params[encoder][\"features\"], 1)\n\n    def forward(self, x):\n        x = self.encoder.forward_features(x)\n        x = self.avg_pool(x).flatten(1)\n        x = self.dropout(x)\n        x = self.fc(x)\n        return x\n\nclass MelanomaClassifierMeta(nn.Module):\n    def __init__(self, encoder, num_meta=30, dropout_rate=0.0) -> None:\n        super().__init__()\n        self.encoder = encoder_params[encoder][\"init_op\"]()\n        self.avg_pool = AdaptiveAvgPool2d((1, 1))\n        self.max_pool = AdaptiveMaxPool2d((1, 1))\n        self.dropout = Dropout(dropout_rate)\n        self.fc = Linear(num_meta + encoder_params[encoder][\"features\"], 1)\n\n    def forward(self, x, m):\n        x = self.encoder.forward_features(x)\n        x = self.avg_pool(x).flatten(1)\n        x = torch.cat((x, m), dim=1)\n        x = self.dropout(x)\n        x = self.fc(x)\n        return x\n\nclass MelanomaClassifier2(nn.Module):\n    def __init__(self, encoder, dropout_rate=0.0) -> None:\n        super().__init__()\n        self.encoder = encoder_params[encoder][\"init_op\"]()\n        self.avg_pool = AdaptiveAvgPool2d((1, 1))\n        self.max_pool = AdaptiveMaxPool2d((1, 1))\n        self.dropout = Dropout(dropout_rate)\n        self.fc = Linear(2 * encoder_params[encoder][\"features\"], 1)\n\n    def forward(self, x):\n        x = self.encoder.forward_features(x)\n        x1 = self.avg_pool(x).flatten(1)\n        x2 = self.max_pool(x).flatten(1)\n        x = torch.cat((x1, x2), dim=1)\n        x = self.dropout(x)\n        x = self.fc(x)\n        return x\n\n","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"%%writefile dataset.py\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport math\nimport os\nimport random\nimport sys\nimport traceback\n\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport skimage.draw\nfrom albumentations import ImageCompression, OneOf, GaussianBlur, Blur\nfrom albumentations.augmentations.functional import image_compression, rot90\nfrom albumentations.pytorch.functional import img_to_tensor\nfrom scipy.ndimage import binary_erosion, binary_dilation\nfrom skimage import measure\nfrom torch.utils.data import Dataset\n\nimport torch\nimport torch.nn.functional as F\n#import dlib\n\n\nclass MelanomaClassifierDataset(Dataset):\n    def __init__(\n        self,\n        df,\n        fold=0,\n        label_smoothing=0.01,\n        normalize={\"mean\": [0.485, 0.456, 0.406],\n                    \"std\": [0.229, 0.224, 0.225]},\n        mode=\"train\",\n        transforms=None,\n        target_transforms=None,\n        data_root=None\n    ):\n        super().__init__()\n        self.df = df\n        self.fold = fold\n        self.mode = mode\n        self.label_smoothing = label_smoothing\n        self.normalize = normalize\n        self.transforms = transforms\n        self.target_transforms = target_transforms\n        self.data_root = data_root\n\n        self.image_name = self.df[\"image_path\"].values\n        self.label = self.df[\"target\"].values\n        self.kmeans = self.df[\"anatom_label\"].values\n    \n    def __getitem__(self, index: int):\n\n        image_name, label = self.image_name[index], self.label[index]\n        if self.mode == \"train\":\n            label = np.clip(label, self.label_smoothing, 1 - self.label_smoothing)\n        #print(image_name)\n        image = cv2.imread(f\"{image_name}\", cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n        if self.transforms:\n            image = self.transforms(image=image)[\"image\"]\n            \n        if label > 0.5:\n            if self.target_transforms:\n                image = self.target_transforms(image=image)[\"image\"]\n\n\n        image = img_to_tensor(image, self.normalize)\n\n        \n\n\n        kmeans = F.one_hot(torch.tensor(self.kmeans[index]), 7)\n\n        return {\n            \"image_name\": image_name,\n            \"image\": image,\n            \"label\": label,\n            \"meta\" : kmeans\n        }\n    \n    def __len__(self):\n        return len(self.image_name)\n    \n    def __get_labels__(self):\n        return list(map(round, self.label.tolist()))\n\n\nclass MelanomaClassifierDatasetTest(Dataset):\n    def __init__(\n        self,\n        df,\n        normalize={\"mean\": [0.485, 0.456, 0.406],\n                    \"std\": [0.229, 0.224, 0.225]},\n        transforms=None,\n        data_root=None\n    ):\n        super().__init__()\n        self.df = df\n        self.normalize = normalize\n        self.transforms = transforms\n        self.data_root = data_root\n\n        self.image_name = self.df[\"image_path\"]\n        self.kmeans = self.df[\"anatom_label\"].values\n    \n    def __getitem__(self, index: int):\n        \n        image = cv2.imread(f\"{self.image_name[index]}\", cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n        if self.transforms:\n            image = self.transforms(image=image)[\"image\"]\n\n        image = img_to_tensor(image, self.normalize)\n        kmeans = F.one_hot(torch.tensor(self.kmeans[index]), 7)\n\n        return {\n            \"image_name\": self.image_name[index],\n            \"image\": image,\n            \"meta\" : kmeans\n        }\n    \n    def __len__(self):\n        return len(self.image_name)\n","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"%%writefile utils.py\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport cv2\nimport numpy as np\nimport sklearn\n\nfrom timm.optim import AdamW\nfrom torch import optim\nfrom torch.optim import lr_scheduler\nfrom torch.optim.rmsprop import RMSprop\nfrom torch.optim.adamw import AdamW\nfrom torch.optim.lr_scheduler import MultiStepLR, CyclicLR\n\nfrom schedulers import ExponentialLRScheduler, PolyLR, LRStepScheduler\n\ncv2.ocl.setUseOpenCL(False)\ncv2.setNumThreads(0)\n\nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\nclass RocAucMeter(object):\n    def __init__(self):\n        self.reset()\n    \n    def reset(self):\n        self.y_true = [0,1]\n        self.y_pred = [0.5, 0.5]\n        self.score = 0\n    \n    def update(self, y_true, y_pred):\n        self.y_true.extend(y_true.cpu().detach().numpy().round().tolist())\n        self.y_pred.extend(y_pred.cpu().detach().numpy().reshape(-1).tolist())\n        y_pred1 = np.array(self.y_pred)\n        y_true1 = np.array(self.y_true)\n        y_pred1[np.isnan(y_pred1)]=0.5\n        self.score = sklearn.metrics.roc_auc_score(y_true1, y_pred1)\n        #self.acc = sklearn.metrics.accuracy_score(y_true1, y_pred1.round())\n        #self.log_loss = sklearn.metrics.log_loss(y_true1, y_pred1)\n    @property\n    def avg(self):\n        return self.score#, self.acc, self.log_loss\n\ndef create_optimizer(optimizer_config, model, master_params=None):\n    \"\"\"Creates optimizer and schedule from configuration\n\n    Parameters\n    ----------\n    optimizer_config : dict\n        Dictionary containing the configuration options for the optimizer.\n    model : Model\n        The network model.\n    \n    Returns\n    -------\n    optimizer : Optimizer\n        The optimizer\n    scheduler : LRScheduler\n        The learning rate scheduler\n    \"\"\"\n    if optimizer_config.get(\"classifier_lr\", -1) != -1:\n        # Separate classifier parameters from all others\n        net_params = []\n        classifier_params = []\n        for k, v in model.named_parameters():\n            if not v.requires_grad:\n                continue\n            if k.find(\"encoder\") != -1:\n                net_params.append(v)\n            else:\n                classifier_params.append(v)\n        params = [\n            {\"params\": net_params},\n            {\"params\": classifier_params, \"lr\": optimizer_config[\"classifier_lr\"]},\n        ]\n    else:\n        if master_params:\n            params = master_params\n        else:\n            params = model.parameters()\n    \n    if optimizer_config[\"type\"] == \"SGD\":\n        optimizer = optim.SGD(params,\n                              lr=optimizer_config[\"learning_rate\"],\n                              momentum=optimizer_config[\"momentum\"],\n                              weight_decay=optimizer_config[\"weight_decay\"],\n                              nesterov=optimizer_config[\"nesterov\"])\n    elif optimizer_config[\"type\"] == \"FusedSGD\":\n        optimizer = FusedSGD(params,\n                             lr=optimizer_config[\"learning_rate\"],\n                             momentum=optimizer_config[\"momentum\"],\n                             weight_decay=optimizer_config[\"weight_decay\"],\n                             nesterov=optimizer_config[\"nesterov\"])\n    elif optimizer_config[\"type\"] == \"Adam\":\n        optimizer = optim.Adam(params,\n                               lr=optimizer_config[\"learning_rate\"],\n                               weight_decay=optimizer_config[\"weight_decay\"])\n    elif optimizer_config[\"type\"] == \"FusedAdam\":\n        optimizer = FusedAdam(params,\n                              lr=optimizer_config[\"learning_rate\"],\n                              weight_decay=optimizer_config[\"weight_decay\"])\n    elif optimizer_config[\"type\"] == \"AdamW\":\n        optimizer = AdamW(params,\n                               lr=optimizer_config[\"learning_rate\"],\n                               weight_decay=optimizer_config[\"weight_decay\"])\n    elif optimizer_config[\"type\"] == \"RmsProp\":\n        optimizer = RMSprop(params,\n                               lr=optimizer_config[\"learning_rate\"],\n                               weight_decay=optimizer_config[\"weight_decay\"])\n    else:\n        raise KeyError(\"unrecognized optimizer {}\".format(optimizer_config[\"type\"]))\n    \n\n    if optimizer_config[\"schedule\"][\"type\"] == \"step\":\n        scheduler = LRStepScheduler(optimizer, **optimizer_config[\"schedule\"][\"params\"])\n    elif optimizer_config[\"schedule\"][\"type\"] == \"clr\":\n        scheduler = CyclicLR(optimizer, **optimizer_config[\"schedule\"][\"params\"])\n    elif optimizer_config[\"schedule\"][\"type\"] == \"multistep\":\n        scheduler = MultiStepLR(optimizer, **optimizer_config[\"schedule\"][\"params\"])\n    elif optimizer_config[\"schedule\"][\"type\"] == \"exponential\":\n        scheduler = ExponentialLRScheduler(optimizer, **optimizer_config[\"schedule\"][\"params\"])\n    elif optimizer_config[\"schedule\"][\"type\"] == \"poly\":\n        scheduler = PolyLR(optimizer, **optimizer_config[\"schedule\"][\"params\"])\n    elif optimizer_config[\"schedule\"][\"type\"] == \"constant\":\n        scheduler = lr_scheduler.LambdaLR(optimizer, lambda epoch: 1.0)\n    elif optimizer_config[\"schedule\"][\"type\"] == \"linear\":\n        def linear_lr(it):\n            return it * optimizer_config[\"schedule\"][\"params\"][\"alpha\"] + optimizer_config[\"schedule\"][\"params\"][\"beta\"]\n\n        scheduler = lr_scheduler.LambdaLR(optimizer, linear_lr)\n    \n    return optimizer, scheduler\n\n","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"%%writefile schedulers.py\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nfrom bisect import bisect_right\n\nfrom torch.optim.lr_scheduler import _LRScheduler\n\nclass LRStepScheduler(_LRScheduler):\n    def __init__(self, optimizer, steps, last_epoch=-1):\n        self.lr_steps = steps\n        super().__init__(optimizer, last_epoch)\n    \n    def get_lr(self):\n        pos = max(bisect_right([x for x, y in self.lr_steps], self.last_epoch) - 1, 0)\n        return [self.lr_steps[pos][1] if self.lr_steps[pos][0] <= self.last_epoch else base_lr for base_lr in self.base_lrs]\n\n\nclass PolyLR(_LRScheduler):\n    \"\"\"Sets the learning rate of each parameter group according to poly learning rate policy\n    \"\"\"\n    def __init__(self, optimizer, max_iter=90000, power=0.9, last_epoch=-1):\n        self.max_iter = max_iter\n        self.power = power\n        super(PolyLR, self).__init__(optimizer, last_epoch)\n\n    def get_lr(self):\n        self.last_epoch = (self.last_epoch + 1) % self.max_iter\n        return [base_lr * ((1 - float(self.last_epoch) / self.max_iter) ** (self.power)) for base_lr in self.base_lrs]\n\n\n\nclass ExponentialLRScheduler(_LRScheduler):\n    \"\"\"Decays the learning rate of each parameter group by gamma every epoch.\n    When last_epoch=-1, sets initial lr as lr.\n    Args:\n        optimizer (Optimizer): Wrapped optimizer.\n        gamma (float): Multiplicative factor of learning rate decay.\n        last_epoch (int): The index of last epoch. Default: -1.\n    \"\"\"\n\n    def __init__(self, optimizer, gamma, last_epoch=-1):\n        self.gamma = gamma\n        super(ExponentialLRScheduler, self).__init__(optimizer, last_epoch)\n\n    def get_lr(self):\n        if self.last_epoch <= 0:\n            return self.base_lrs\n        return [base_lr * self.gamma**self.last_epoch for base_lr in self.base_lrs]\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"%%writefile losses.py\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nfrom typing import Any\n\nfrom pytorch_toolbelt.losses import BinaryFocalLoss\nfrom torch import nn\nfrom torch.nn.modules.loss import BCEWithLogitsLoss\n\nclass WeightedLosses(nn.Module):\n    def __init__(self, losses, weights):\n        super().__init__()\n        self.losses = losses\n        self.weights = weights\n    \n    def forward(self, *input: Any, **kwargs: Any):\n        cum_loss = 0\n        for loss, w in zip(self.losses, self.weights):\n            cum_loss += w * loss.forward(*input, **kwargs)\n        return cum_loss\n\nclass BinaryCrossentropy(BCEWithLogitsLoss):\n    pass\n\n\nclass FocalLoss(BinaryFocalLoss):\n    def __init__(self, alpha=None, gamma=3, ignore_index=None, reduction=\"mean\", normalized=False,\n                 reduced_threshold=None):\n        super().__init__(alpha, gamma, ignore_index, reduction, normalized, reduced_threshold)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"%%writefile albu.py\n\nimport random\n\nimport cv2\nimport numpy as np\nimport albumentations as A\nfrom albumentations import DualTransform, ImageOnlyTransform\nfrom albumentations.augmentations.functional import crop\nfrom albumentations.augmentations import functional as F\nfrom PIL import Image, ImageOps, ImageEnhance\n\ndef train_transforms(size=300):\n    return A.Compose([\n        #HairRemove(p=0.33),\n        #MaskLandmarks(p=0.5),\n        #ShadeGrayCC(p=1),\n        #A.ImageCompression(quality_lower=80, quality_upper=100, p=0.1),\n        #A.GaussianBlur(blur_limit=3, p=0.05),\n        #A.GaussNoise(p=0.05),\n        #RandomEraser(),\n        \n        A.OneOf([\n            A.VerticalFlip(),\n            A.HorizontalFlip(),\n            A.Flip(),\n            A.Rotate()\n        ], p=0.5),\n        A.Transpose(p=0.2),\n        A.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.1, rotate_limit=15),\n        A.RandomRotate90(p=0.1),\n        A.OneOf([\n            A.RandomGridShuffle(p=0.1),\n            A.Cutout(num_holes=8, max_h_size=size//8, max_w_size=size//8, fill_value=0, p=0.2),\n            A.CoarseDropout(max_holes=4, max_height=size//8, max_width=size//8, p=0.2),\n            A.GridDropout(p=0.2),\n            RandomEraser(p=0.2),\n            BitMask(size=size, p=0.1),\n        ], p=0.11),\n        A.RandomBrightness(limit=(-0.2,0.2), p=0.1),\n        A.OneOf([\n            IsotropicResize(max_side=size, interpolation_down=cv2.INTER_AREA, interpolation_up=cv2.INTER_CUBIC),\n            IsotropicResize(max_side=size, interpolation_down=cv2.INTER_AREA, interpolation_up=cv2.INTER_LINEAR),\n            IsotropicResize(max_side=size, interpolation_down=cv2.INTER_LINEAR, interpolation_up=cv2.INTER_LINEAR),\n        ], p=1),\n        A.PadIfNeeded(min_height=size, min_width=size, border_mode=cv2.BORDER_CONSTANT),\n        #RandomAugMix()\n    ])\n\ndef target_transforms(size=300):\n    return A.Compose([\n        A.ImageCompression(quality_lower=80, quality_upper=100, p=0.1),\n        A.GaussianBlur(blur_limit=3, p=0.05),\n        A.GaussNoise(p=0.05),\n        A.OneOf([\n            A.RandomGridShuffle(p=0.1),\n            A.Cutout(num_holes=8, max_h_size=size//8, max_w_size=size//8, fill_value=0, p=0.2),\n            A.CoarseDropout(max_holes=4, max_height=size//8, max_width=size//8, p=0.2),\n            A.GridDropout(p=0.2),\n            RandomEraser(p=0.2),\n            BitMask(size=size, p=0.1),\n        ], p=0.33)\n    ])\n    \ndef valid_transforms(size=300):\n    return A.Compose([\n        #ShadeGrayCC(p=1),\n        IsotropicResize(max_side=size, interpolation_down=cv2.INTER_AREA, interpolation_up=cv2.INTER_CUBIC),\n        A.PadIfNeeded(min_height=size, min_width=size, border_mode=cv2.BORDER_CONSTANT),\n    ])\n\ndef test_transforms(size=300):\n    return A.Compose([\n        #HairRemove(p=0.33),\n        #MaskLandmarks(p=0.2),\n        #ShadeGrayCC(p=0.5),\n        #A.ImageCompression(quality_lower=80, quality_upper=100, p=0.1),\n        #A.GaussianBlur(blur_limit=3, p=0.05),\n        #A.GaussNoise(p=0.05),\n        A.OneOf([\n            A.VerticalFlip(),\n            A.HorizontalFlip(),\n            A.Flip(),\n            A.Rotate()\n        ], p=0.5),\n        A.Transpose(p=0.2),\n        A.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.1, rotate_limit=15),\n        A.RandomRotate90(p=0.1),\n        A.RandomBrightness(limit=(-0.2,0.2), p=0.1),\n        A.OneOf([\n            IsotropicResize(max_side=size, interpolation_down=cv2.INTER_AREA, interpolation_up=cv2.INTER_CUBIC),\n            IsotropicResize(max_side=size, interpolation_down=cv2.INTER_AREA, interpolation_up=cv2.INTER_LINEAR),\n            IsotropicResize(max_side=size, interpolation_down=cv2.INTER_LINEAR, interpolation_up=cv2.INTER_LINEAR),\n        ], p=1),\n        A.PadIfNeeded(min_height=size, min_width=size, border_mode=cv2.BORDER_CONSTANT),\n    ])\n    \n\ndef isotropically_resize_image(img, size, interpolation_down=cv2.INTER_AREA, interpolation_up=cv2.INTER_CUBIC):\n    h, w = img.shape[:2]\n    if max(w, h) == size:\n        return img\n    if w > h:\n        scale = size / w\n        h = h * scale\n        w = size\n    else:\n        scale = size / h\n        w = w * scale\n        h = size\n    interpolation = interpolation_up if scale > 1 else interpolation_down\n    resized = cv2.resize(img, (int(w), int(h)), interpolation=interpolation)\n    return resized\n\nclass IsotropicResize(DualTransform):\n    def __init__(self, max_side, interpolation_down=cv2.INTER_AREA, interpolation_up=cv2.INTER_CUBIC,\n                 always_apply=False, p=1):\n        super(IsotropicResize, self).__init__(always_apply, p)\n        self.max_side = max_side\n        self.interpolation_down = interpolation_down\n        self.interpolation_up = interpolation_up\n\n    def apply(self, img, interpolation_down=cv2.INTER_AREA, interpolation_up=cv2.INTER_CUBIC, **params):\n        return isotropically_resize_image(img, size=self.max_side, interpolation_down=interpolation_down,\n                                          interpolation_up=interpolation_up)\n\n    def apply_to_mask(self, img, **params):\n        return self.apply(img, interpolation_down=cv2.INTER_NEAREST, interpolation_up=cv2.INTER_NEAREST, **params)\n\n    def get_transform_init_args_names(self):\n        return (\"max_side\", \"interpolation_down\", \"interpolation_up\")\n\n\n\nclass Resize4xAndBack(ImageOnlyTransform):\n    def __init__(self, always_apply=False, p=0.5):\n        super(Resize4xAndBack, self).__init__(always_apply, p)\n\n    def apply(self, img, **params):\n        h, w = img.shape[:2]\n        scale = random.choice([2, 4])\n        img = cv2.resize(img, (w // scale, h // scale), interpolation=cv2.INTER_AREA)\n        img = cv2.resize(img, (w, h),\n                         interpolation=random.choice([cv2.INTER_CUBIC, cv2.INTER_LINEAR, cv2.INTER_NEAREST]))\n        return img\n\n\nclass RandomSizedCropNonEmptyMaskIfExists(DualTransform):\n\n    def __init__(self, min_max_height, w2h_ratio=[0.7, 1.3], always_apply=False, p=0.5):\n        super(RandomSizedCropNonEmptyMaskIfExists, self).__init__(always_apply, p)\n\n        self.min_max_height = min_max_height\n        self.w2h_ratio = w2h_ratio\n\n    def apply(self, img, x_min=0, x_max=0, y_min=0, y_max=0, **params):\n        cropped = crop(img, x_min, y_min, x_max, y_max)\n        return cropped\n\n    @property\n    def targets_as_params(self):\n        return [\"mask\"]\n\n    def get_params_dependent_on_targets(self, params):\n        mask = params[\"mask\"]\n        mask_height, mask_width = mask.shape[:2]\n        crop_height = int(mask_height * random.uniform(self.min_max_height[0], self.min_max_height[1]))\n        w2h_ratio = random.uniform(*self.w2h_ratio)\n        crop_width = min(int(crop_height * w2h_ratio), mask_width - 1)\n        if mask.sum() == 0:\n            x_min = random.randint(0, mask_width - crop_width + 1)\n            y_min = random.randint(0, mask_height - crop_height + 1)\n        else:\n            mask = mask.sum(axis=-1) if mask.ndim == 3 else mask\n            non_zero_yx = np.argwhere(mask)\n            y, x = random.choice(non_zero_yx)\n            x_min = x - random.randint(0, crop_width - 1)\n            y_min = y - random.randint(0, crop_height - 1)\n            x_min = np.clip(x_min, 0, mask_width - crop_width)\n            y_min = np.clip(y_min, 0, mask_height - crop_height)\n\n        x_max = x_min + crop_height\n        y_max = y_min + crop_width\n        y_max = min(mask_height, y_max)\n        x_max = min(mask_width, x_max)\n        return {\"x_min\": x_min, \"x_max\": x_max, \"y_min\": y_min, \"y_max\": y_max}\n\n    def get_transform_init_args_names(self):\n        return \"min_max_height\", \"height\", \"width\", \"w2h_ratio\"\n\n\ndef prepare_bit_masks(mask):\n    h, w = mask.shape\n    mid_w = w // 2\n    mid_h = w // 2\n    masks = []\n    ones = np.ones_like(mask)\n    ones[:mid_h] = 0\n    masks.append(ones)\n    ones = np.ones_like(mask)\n    ones[mid_h:] = 0\n    masks.append(ones)\n    ones = np.ones_like(mask)\n    ones[:, :mid_w] = 0\n    masks.append(ones)\n    ones = np.ones_like(mask)\n    ones[:, mid_w:] = 0\n    masks.append(ones)\n    ones = np.ones_like(mask)\n    ones[:mid_h, :mid_w] = 0\n    ones[mid_h:, mid_w:] = 0\n    masks.append(ones)\n    ones = np.ones_like(mask)\n    ones[:mid_h, mid_w:] = 0\n    ones[mid_h:, :mid_w] = 0\n    masks.append(ones)\n    return masks\n\nclass BitMask(ImageOnlyTransform):\n    def __init__(self, size=300 ,always_apply=False, p=0.5):\n        super(BitMask, self).__init__(always_apply, p)\n        mask = np.zeros((size,size), dtype=np.uint8)\n        self.masks = prepare_bit_masks(mask)\n\n    def apply(self, img, **params):\n        \n        bitmap_msk = random.choice(self.masks)\n        #if np.count_nonzero(mask * bitmap_msk) > 20:\n        #mask *= bitmap_msk\n        try:\n            img *= np.expand_dims(bitmap_msk, axis=-1)\n        except:\n            img = img\n        \n        return img\n\nclass HairRemove(ImageOnlyTransform):\n    def __init__(self, always_apply=False, p=0.5):\n        super(HairRemove, self).__init__(always_apply, p)\n\n    def apply(self, img, **params):\n\n        try:\n        \n            gry = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n            \n            th = np.random.randint(5,10)\n            # Kernel for the morphological filtering\n            kernel = cv2.getStructuringElement(1,(17,17))\n            \n            # Perform the blackHat filtering on the grayscale image to find the \n            # hair countours\n            blackhat = cv2.morphologyEx(gry, cv2.MORPH_BLACKHAT, kernel)\n\n            # intensify the hair countours in preparation for the inpainting \n            # algorithm\n            ret,thresh2 = cv2.threshold(blackhat,th,255,cv2.THRESH_BINARY)\n            img = cv2.inpaint(img, thresh2,1, cv2.INPAINT_TELEA)\n        except:\n            img = img\n        \n        return img\n\nclass ShadeGrayCC(ImageOnlyTransform):\n    def __init__(self, power=6, gamma=None, always_apply=False, p=0.5):\n        super(ShadeGrayCC, self).__init__(always_apply, p)\n        \n        self.power = power\n        self.gamma = gamma\n\n    def apply(self, img, **params):\n\n        try:\n        \n            img_dtype = img.dtype\n            \n            if self.gamma is not None:\n                look_up_table = np.ones((256,1), dtype='uint8') * 0\n                for i in range(256):\n                    look_up_table[i][0] = 255 * pow(i/255, 1/self.gamma)\n                img = cv2.LUT(img, look_up_table)\n            \n            img = img.astype(\"float32\")\n            img_power = np.power(img, self.power)\n            rgb_vec = np.power(np.mean(img_power, (0,1)), 1/self.power)\n            rgb_norm = np.sqrt(np.sum(np.power(rgb_vec, 2.0)))\n            rgb_vec = rgb_vec/rgb_norm\n            rgb_vec = 1/(rgb_vec*np.sqrt(3))\n            img = np.multiply(img, rgb_vec)\n\n            img.astype(img_dtype)\n        except:\n            img = img\n        \n        return img\n\nclass RandomEraser(ImageOnlyTransform):\n    def __init__(self, always_apply=False, p=0.5):\n        super(RandomEraser, self).__init__(always_apply, p)\n        \n\n    def apply(self, img, **params):\n        \n        try:\n\n            img_h, img_w, img_c = img.shape\n            \n            s_l=0.02; s_h=0.4; r_1=0.3; r_2=1/0.3; v_l=0; v_h=255\n            \n            while True:\n                s = np.random.uniform(s_l, s_h) * img_h * img_w\n                r = np.random.uniform(r_1, r_2)\n                w = int(np.sqrt(s / r))\n                h = int(np.sqrt(s * r))\n                left = np.random.randint(0, img_w)\n                top = np.random.randint(0, img_h)\n\n                if left + w <= img_w and top + h <= img_h:\n                    break\n\n            if np.random.rand() > 0.5:\n                c = np.random.uniform(v_l, v_h, (h, w, img_c))\n            else:\n                c = np.random.uniform(v_l, v_h)\n\n            img[top:top + h, left:left + w, :] = c\n        except:\n            img = img\n        \n        \n        return img\n\nclass MaskLandmarks(ImageOnlyTransform):\n    def __init__(self, always_apply=False, p=0.5):\n        super(MaskLandmarks, self).__init__(always_apply, p)\n        \n\n    def apply(self, img, **params):\n\n        try:\n        \n            img_gry = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY) \n        \n            # Otsu's thresholding\n            ret2,th2 = cv2.threshold(img_gry,0,255,cv2.THRESH_BINARY+cv2.THRESH_OTSU)\n\n            retval, thresh_gray = cv2.threshold(th2, thresh=100, maxval=255, \\\n                                        type=cv2.THRESH_BINARY_INV)\n\n            contours, hierarchy = cv2.findContours(thresh_gray,cv2.RETR_LIST, \\\n                                            cv2.CHAIN_APPROX_SIMPLE)\n\n            # Find object with the biggest bounding box\n            mx = (0,0,0,0)      # biggest bounding box so far\n            mx_area = 0\n            for cont in contours:\n                x,y,w,h = cv2.boundingRect(cont)\n                area = w*h\n                if area > mx_area:\n                    mx = x,y,w,h\n                    mx_area = area\n                    \n            x,y,w,h = mx\n            \n            img = img[y:y+h, x:x+w]\n        except:\n            img = img\n        \n        return img\n\n\ndef int_parameter(level, maxval):\n    \"\"\"Helper function to scale `val` between 0 and maxval .\n    Args:\n    level: Level of the operation that will be between [0, `PARAMETER_MAX`].\n    maxval: Maximum value that the operation can have. This will be scaled to\n      level/PARAMETER_MAX.\n    Returns:\n    An int that results from scaling `maxval` according to `level`.\n    \"\"\"\n    return int(level * maxval / 10)\n\n\ndef float_parameter(level, maxval):\n    \"\"\"Helper function to scale `val` between 0 and maxval.\n    Args:\n    level: Level of the operation that will be between [0, `PARAMETER_MAX`].\n    maxval: Maximum value that the operation can have. This will be scaled to\n      level/PARAMETER_MAX.\n    Returns:\n    A float that results from scaling `maxval` according to `level`.\n    \"\"\"\n    return float(level) * maxval / 10.\n\n\ndef sample_level(n):\n    return np.random.uniform(low=0.1, high=n)\n\n\ndef autocontrast(pil_img, _):\n    return ImageOps.autocontrast(pil_img)\n\n\ndef equalize(pil_img, _):\n    return ImageOps.equalize(pil_img)\n\n\ndef posterize(pil_img, level):\n    level = int_parameter(sample_level(level), 4)\n    return ImageOps.posterize(pil_img, 4 - level)\n\n\ndef rotate(pil_img, level):\n    degrees = int_parameter(sample_level(level), 30)\n    if np.random.uniform() > 0.5:\n        degrees = -degrees\n    return pil_img.rotate(degrees, resample=Image.BILINEAR)\n\n\ndef solarize(pil_img, level):\n    level = int_parameter(sample_level(level), 256)\n    return ImageOps.solarize(pil_img, 256 - level)\n\n\ndef shear_x(pil_img, level):\n    level = float_parameter(sample_level(level), 0.3)\n    if np.random.uniform() > 0.5:\n        level = -level\n    return pil_img.transform(pil_img.size,\n                           Image.AFFINE, (1, level, 0, 0, 1, 0),\n                           resample=Image.BILINEAR)\n\n\ndef shear_y(pil_img, level):\n    level = float_parameter(sample_level(level), 0.3)\n    if np.random.uniform() > 0.5:\n        level = -level\n    return pil_img.transform(pil_img.size,\n                           Image.AFFINE, (1, 0, 0, level, 1, 0),\n                           resample=Image.BILINEAR)\n\n\ndef translate_x(pil_img, level):\n    level = int_parameter(sample_level(level), pil_img.size[0] / 3)\n    if np.random.random() > 0.5:\n        level = -level\n    return pil_img.transform(pil_img.size,\n                           Image.AFFINE, (1, 0, level, 0, 1, 0),\n                           resample=Image.BILINEAR)\n\n\ndef translate_y(pil_img, level):\n    level = int_parameter(sample_level(level), pil_img.size[0] / 3)\n    if np.random.random() > 0.5:\n        level = -level\n    return pil_img.transform(pil_img.size,\n                           Image.AFFINE, (1, 0, 0, 0, 1, level),\n                           resample=Image.BILINEAR)\n\n\n# operation that overlaps with ImageNet-C's test set\ndef color(pil_img, level):\n    level = float_parameter(sample_level(level), 1.8) + 0.1\n    return ImageEnhance.Color(pil_img).enhance(level)\n\n\n# operation that overlaps with ImageNet-C's test set\ndef contrast(pil_img, level):\n    level = float_parameter(sample_level(level), 1.8) + 0.1\n    return ImageEnhance.Contrast(pil_img).enhance(level)\n\n\n# operation that overlaps with ImageNet-C's test set\ndef brightness(pil_img, level):\n    level = float_parameter(sample_level(level), 1.8) + 0.1\n    return ImageEnhance.Brightness(pil_img).enhance(level)\n\n\n# operation that overlaps with ImageNet-C's test set\ndef sharpness(pil_img, level):\n    level = float_parameter(sample_level(level), 1.8) + 0.1\n    return ImageEnhance.Sharpness(pil_img).enhance(level)\n\n\naugmentations = [\n    autocontrast, equalize, posterize, rotate, solarize, shear_x, shear_y,\n    translate_x, translate_y\n]\n\naugmentations_all = [\n    autocontrast, equalize, posterize, rotate, solarize, shear_x, shear_y,\n    translate_x, translate_y, color, contrast, brightness, sharpness\n]\n\ndef normalize(image):\n    \"\"\"Normalize input image channel-wise to zero mean and unit variance.\"\"\"\n    return image - 127\n\ndef apply_op(image, op, severity):\n    #   image = np.clip(image, 0, 255)\n    pil_img = Image.fromarray(image)  # Convert to PIL.Image\n    pil_img = op(pil_img, severity)\n    return np.asarray(pil_img)\n\ndef augment_and_mix(image, severity=3, width=3, depth=-1, alpha=1.):\n    \"\"\"Perform AugMix augmentations and compute mixture.\n    Args:\n    image: Raw input image as float32 np.ndarray of shape (h, w, c)\n    severity: Severity of underlying augmentation operators (between 1 to 10).\n    width: Width of augmentation chain\n    depth: Depth of augmentation chain. -1 enables stochastic depth uniformly\n      from [1, 3]\n    alpha: Probability coefficient for Beta and Dirichlet distributions.\n    Returns:\n    mixed: Augmented and mixed image.\n    \"\"\"\n    ws = np.float32(\n      np.random.dirichlet([alpha] * width))\n    m = np.float32(np.random.beta(alpha, alpha))\n\n    mix = np.zeros_like(image).astype(np.float32)\n    for i in range(width):\n        image_aug = image.copy()\n        depth = depth if depth > 0 else np.random.randint(1, 4)\n        for _ in range(depth):\n            op = np.random.choice(augmentations)\n            image_aug = apply_op(image_aug, op, severity)\n        # Preprocessing commutes since all coefficients are convex\n        mix += ws[i] * image_aug\n#         mix += ws[i] * normalize(image_aug)\n\n    mixed = (1 - m) * image + m * mix\n#     mixed = (1 - m) * normalize(image) + m * mix\n    return mixed\n\n\nclass RandomAugMix(ImageOnlyTransform):\n\n    def __init__(self, severity=3, width=3, depth=-1, alpha=1., always_apply=False, p=0.5):\n        super().__init__(always_apply, p)\n        self.severity = severity\n        self.width = width\n        self.depth = depth\n        self.alpha = alpha\n\n    def apply(self, image, **params):\n        image = augment_and_mix(\n            image,\n            self.severity,\n            self.width,\n            self.depth,\n            self.alpha\n        )\n        return image","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"%%writefile train.py\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport argparse\nimport json\nimport os\nfrom collections import defaultdict\nfrom sklearn.model_selection import KFold\nfrom catalyst.data.sampler import DistributedSampler, BalanceClassSampler\nfrom torch import topk\n\nimport numpy as np\nimport pandas as pd\n\nimport classifiers\nimport losses\nfrom losses import WeightedLosses\nfrom dataset import *\nfrom config import args\nfrom utils import create_optimizer, AverageMeter, RocAucMeter\nfrom albu import *\n\n\nimport torch\nimport torch.nn as nn\nfrom torch.backends import cudnn\nfrom torch.nn import DataParallel\nfrom torch.utils.data import DataLoader\nfrom tqdm import tqdm\nimport torch.distributed as dist\n\ntorch.backends.cudnn.benchmark = True\n\nos.environ[\"MKL_NUM_THREADS\"] = \"1\"\nos.environ[\"NUMEXPR_NUM_THREADS\"] = \"1\"\nos.environ[\"OMP_NUM_THREADS\"] = \"1\"\nos.environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID'\nos.environ[\"CUDA_VISIBLE_DEVICES\"] = '0'\n\n#from apex import amp\n#from apex.parallel import DistributedDataParallel, convert_syncbn_model\n\n\ndef train_epoch(args, model, train_loader, optimizer, scheduler, loss_functions, device, epoch):\n    losses = AverageMeter()\n    scores = RocAucMeter()\n\n    model.train()\n    scaler = torch.cuda.amp.GradScaler()\n\n    t = tqdm(train_loader)\n    for i, sample in enumerate(t):\n        imgs = sample[\"image\"].to(device)\n        labels = sample[\"label\"].to(device)\n        meta = sample[\"meta\"].to(device)\n\n        if args.mixup and args.cutmix:\n            if np.random.rand(1) > 0.5:\n                imgs, targets_a, targets_b, lam = mixup_data(imgs, labels,\n                                                        args.alpha, True)\n            else:\n                imgs, targets_a, targets_b, lam = cutmix_data(imgs, labels,\n                                                        args.alpha, True)\n            \n            imgs, targets_a, targets_b = map(torch.autograd.Variable, (imgs,\n                                                      targets_a, targets_b))\n        elif args.mixup:\n            imgs, targets_a, targets_b, lam = mixup_data(imgs, labels,\n                                                        args.alpha, True)\n            imgs, targets_a, targets_b = map(torch.autograd.Variable, (imgs,\n                                                      targets_a, targets_b))\n\n        elif args.cutmix:\n            imgs, targets_a, targets_b, lam = cutmix_data(imgs, labels,\n                                                       args.beta, True)\n            \n            imgs, targets_a, targets_b = map(torch.autograd.Variable, (imgs,\n                                                      targets_a, targets_b))\n\n\n        optimizer.zero_grad()\n\n        # Casts operations to mixed precision\n        with torch.cuda.amp.autocast():\n            outputs = model(imgs)\n            if args.mixup or args.cutmix:\n                loss = mixup_criterion(loss_functions[\"classifier_loss\"], outputs, targets_a, targets_b, lam)\n            else:\n                loss = loss_functions[\"classifier_loss\"](outputs, labels.view(-1, 1))\n        #loss = loss_fn(outputs, labels)\n\n        bs = imgs.size(0)\n        scores.update(labels, torch.sigmoid(outputs))\n        losses.update(loss.item(), bs)\n\n        if args.fp16:\n            with amp.scale_loss(loss, optimizer) as scaled_loss:\n                scaled_loss.backward()\n        else:\n            # Scales the loss, and calls backward()\n            # to create scaled gradients\n            scaler.scale(loss).backward()\n            \n            # Uncales gradients and calls\n            # or skips optimizer.step()\n            scaler.step(optimizer)\n\n            # Updates the scale for next iteration\n            scaler.update()\n\n        #torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), 1)\n        #optimizer.step()\n        #scheduler.step()\n\n        t.set_description(f\"Train E:{epoch} - Loss:{losses.avg:0.4f} - AUC:{scores.avg:0.4f} \")\n\n    t.close()\n    return scores.avg, losses.avg\n\ndef valid_epoch(args, model, valid_loader,loss_functions, device, epoch):\n    losses = AverageMeter()\n    scores = RocAucMeter()\n\n    model.eval()\n    with torch.no_grad():\n        t = tqdm(valid_loader)\n        for i, sample in enumerate(t):\n            imgs = sample[\"image\"].to(device)\n            labels = sample[\"label\"].to(device)\n            meta = sample[\"meta\"].to(device)\n\n            outputs = model(imgs)\n            #loss = loss_functions[\"classifier_loss\"](outputs, labels.view(-1, 1))\n            #loss = loss_fn(outputs, labels)\n\n            bs = imgs.size(0)\n            scores.update(labels, torch.sigmoid(outputs))\n            #losses.update(loss.item(), bs)\n\n            t.set_description(f\"Valid E:{epoch} - AUC:{scores.avg:0.4f} \")\n\n    t.close()\n    return scores.avg\n\ndef test_epoch(args, model, test_loader, device):\n\n    probs = []\n    image_names = []\n\n    model.eval()\n    t = tqdm(test_loader)\n    with torch.no_grad():\n        for i, sample in enumerate(t):\n            imgs = sample[\"image\"].to(device)\n            meta = sample[\"meta\"].to(device)\n            img_names = sample[\"image_name\"]\n\n\n            out = model(imgs)\n            preds = torch.sigmoid(out).cpu().numpy().tolist()\n            \n\n            probs.extend(preds)\n            image_names.extend(img_names)\n    \n    t.close()\n    return probs, image_names\n\n\nlr_start   = 0.000005\nlr_max     = 0.00000125 * args.batch_size\nlr_min     = 0.000001\nlr_ramp_ep = 5\nlr_sus_ep  = 0\nlr_decay   = 0.8\n\ndef lrfn(epoch):\n    if epoch < args.LR_RAMPUP_EPOCHS:\n        lr = (args.LR_MAX - args.LR_START) / args.LR_RAMPUP_EPOCHS * epoch + args.LR_START\n    elif epoch < args.LR_RAMPUP_EPOCHS + args.LR_SUSTAIN_EPOCHS:\n        lr = args.LR_MAX\n    else:\n        lr = (args.LR_MAX - args.LR_MIN) * args.LR_EXP_DECAY**(epoch - args.LR_RAMPUP_EPOCHS - args.LR_SUSTAIN_EPOCHS) + args.LR_MIN\n    return lr\n\ndef adjust_learning_rate(optimizer, epoch):\n    \"\"\"Sets the learning rate to the initial LR decayed by 10 every 30 epochs\"\"\"\n    lr = lrfn(epoch)\n    for param_group in optimizer.param_groups:\n        param_group['lr'] = lr \n\n\ndef loss_fn(output, target):\n    return nn.BCEWithLogitsLoss()(output, target.view(-1, 1))\n\ndef rand_bbox(size, lam):\n    W = size[2]\n    H = size[3]\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = np.int(W * cut_rat)\n    cut_h = np.int(H * cut_rat)\n\n    # uniform\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n\n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n\n    return bbx1, bby1, bbx2, bby2\n\ndef mixup_data(x, y, alpha=1.0, use_cuda=True):\n    '''Returns mixed inputs, pairs of targets, and lambda'''\n    if alpha > 0:\n        lam = np.random.beta(alpha, alpha)\n    else:\n        lam = 1\n\n    batch_size = x.size()[0]\n    if use_cuda:\n        index = torch.randperm(batch_size).cuda()\n    else:\n        index = torch.randperm(batch_size)\n\n    mixed_x = lam * x + (1 - lam) * x[index, :]\n    y_a, y_b = y, y[index]\n    return mixed_x, y_a, y_b, lam\n\ndef mixup_criterion(criterion, pred, y_a, y_b, lam):\n    return lam * criterion(pred, y_a.view(-1, 1)) + (1 - lam) * criterion(pred, y_b.view(-1, 1))\n\n\ndef cutmix_data(x, y, beta=1.0, use_cuda=True):\n    '''Returns mixed inputs, pairs of targets, and lambda'''\n    \n    lam = np.random.beta(beta, beta)\n    \n\n    batch_size = x.size()[0]\n    if use_cuda:\n        index = torch.randperm(batch_size).cuda()\n    else:\n        index = torch.randperm(batch_size)\n    \n    target_a = y\n    target_b = y[index]\n\n    bbx1, bby1, bbx2, bby2 = rand_bbox(x.size(), lam)\n    x[:, :, bbx1:bbx2, bby1:bby2] = x[index, :, bbx1:bbx2, bby1:bby2]\n\n    # adjust lambda to exactly match pixel ratio\n    lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (x.size()[-1] * x.size()[-2]))\n\n    return x, target_a, target_b, lam\n\ndef cutmix_criterion(criterion, pred, y_a, y_b, lam):\n    return lam * criterion(pred, y_a.view(-1, 1)) + (1 - lam) * criterion(pred, y_b.view(-1, 1))\n\n\ndef main(fold, idxT, idxV):\n\n    # Setting seed\n    seed = args.seed\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n\n    args.fold = fold\n    args.save_path = os.path.join(args.output_dir, args.exp_name)\n    os.makedirs(args.save_path, exist_ok=True)\n\n    model = classifiers.__dict__[args.network](encoder=args.encoder)\n\n    train_df = pd.read_csv(args.train_csv)\n    train_2019_df = pd.read_csv(args.train_2019_csv)\n    test_df = pd.read_csv(args.test_csv)\n    \n\n    train_folds = train_df[train_df.kfold.isin(idxT)]\n    valid_folds = train_df[train_df.kfold.isin(idxV)]\n    \n    \n    if args.train_fn:\n        train_folds = train_folds[train_folds.difference > args.train_fn]\n        print(\"FN 2020 :\", train_folds.shape)\n    \n    if args.train_fp:\n        train_folds = train_folds[train_folds.difference < args.train_fp]\n        print(\"FP 2020 :\", train_folds.shape)\n\n    if args.malignant:\n        malignant_df2 = pd.read_csv(args.malignant_csv)\n\n        train_folds = train_folds.append(malignant_df2)\n        print(\"Melignant :\", train_folds.shape)\n\n    if args.psudo_label:\n        psudo_df = pd.read_csv(args.psudo_csv)\n        train_folds = train_folds.append(psudo_df)\n        print(\"Sudo :\",train_folds.shape)\n\n    if args.train_2019:\n        train_2019_folds = train_2019_df[train_2019_df.tfrecord != -1]\n        #train_2019_folds = train_2019_df[train_2019_df.tfrecord.isin([i*2 for i in range(15)])]\n        #train_2019_folds = train_2019_df[train_2019_df.diagnosis.isin(['NV', 'MEL'])]\n        if args.train_2019_fn:\n            train_2019_folds = train_2019_folds[train_2019_folds.difference > args.train_2019_fn]\n            print(\"FN 2019 :\",train_folds.shape)\n        \n        if args.train_2019_fp:\n            train_2019_folds = train_2019_folds[train_2019_folds.difference < args.train_2019_fp]\n            print(\"FP 2019 :\", train_2019_folds.shape)\n\n        train_folds = train_folds.append(train_2019_folds)\n        \n\n    train_dataset = MelanomaClassifierDataset(\n        df=train_folds,\n        mode=\"train\",\n        label_smoothing=args.label_smoothing,\n        normalize=args.normalize,\n        transforms=train_transforms(size=args.size),\n        #target_transforms=target_transforms(size=args.size),\n        data_root=args.train_image_path\n    )\n\n    valid_dataset = MelanomaClassifierDataset(\n        df=valid_folds,\n        mode=\"valid\",\n        label_smoothing=args.label_smoothing,\n        normalize=args.normalize,\n        transforms=valid_transforms(size=args.size),\n        data_root=args.train_image_path\n    )\n\n    test_dataset = MelanomaClassifierDatasetTest(\n        df=test_df,\n        normalize=args.normalize,\n        transforms=valid_transforms(size=args.size),\n        data_root=args.test_image_path   \n    )\n\n    tta_dataset = MelanomaClassifierDatasetTest(\n        df=test_df,\n        normalize=args.normalize,\n        transforms=test_transforms(size=args.size),\n        data_root=args.test_image_path   \n    )\n\n    loss_fn = []\n    weights = []\n    for loss_name, weight in args.losses.items():\n        loss_fn.append(losses.__dict__[loss_name](reduction=\"mean\"))\n        weights.append(weight)\n    \n    loss = WeightedLosses(loss_fn, weights)\n    loss_functions = {\"classifier_loss\": loss}\n    optimizer, scheduler = create_optimizer(args.optimizer, model)\n\n    device = \"cuda\"\n    model = model.cuda()\n\n    if args.fp16:\n        model, optimizer = amp.initialize(model, optimizer,\n                                          opt_level=args.opt_level,\n                                          loss_scale='dynamic')\n\n    #loss_functions = None\n\n    \n\n    \n    \"\"\"\n    optimizer = torch.optim.Adam(\n        model.parameters(),\n        lr=args.learning_rate\n    )\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n        optimizer,\n        patience=3,\n        threshold=0.001,\n        mode=\"max\"\n    )\n    \"\"\"\n    \"\"\"\n    train_sampler = torch.utils.data.DistributedSampler(\n        train_dataset,\n        num_replicas=xm.xrt_world_size(),\n        rank=xm.get_ordinal(),\n        shuffle=True\n    )\n    \"\"\"\n    train_loader = torch.utils.data.DataLoader(\n        train_dataset,\n        batch_size=args.batch_size,\n        #sampler=BalanceClassSampler(labels=train_dataset.__get_labels__(), mode=\"upsampling\"),\n        shuffle=True,\n        drop_last=True,\n        num_workers=4\n    )\n\n    \"\"\"\n    valid_sampler = torch.utils.data.DistributedSampler(\n        valid_dataset,\n        num_replicas=xm.xrt_world_size(),\n        rank=xm.get_ordinal(),\n        shuffle=False\n    )\n    \"\"\"\n    valid_loader = torch.utils.data.DataLoader(\n        valid_dataset,\n        batch_size=args.batch_size * 2,\n        #sampler=valid_sampler,\n        drop_last=False,\n        num_workers=4\n    )\n    \"\"\"\n    test_sampler = torch.utils.data.DistributedSampler(\n        test_dataset,\n        num_replicas=xm.xrt_world_size(),\n        rank=xm.get_ordinal(),\n        shuffle=False\n    )\n    \"\"\"\n    test_loader = torch.utils.data.DataLoader(\n        test_dataset,\n        batch_size=args.batch_size * 2,\n        #sampler=test_sampler,\n        drop_last=False,\n        num_workers=4\n    )\n\n    tta_loader = torch.utils.data.DataLoader(\n        tta_dataset,\n        batch_size=args.batch_size * 2,\n        #sampler=test_sampler,\n        drop_last=False,\n        num_workers=4\n    )\n    \n    best_auc = 0\n    \n    print(\"Training started ..... \")\n\n    test_preds = []\n\n    #if args.fold == 2:\n    #    model.load_state_dict(torch.load(os.path.join(args.save_path, f\"fold-{args.fold}.bin\")))\n    #    model.to(device)\n    #    start_epoch = 4\n    #    best_auc = 0.9262593624711052\n    #else:\n    start_epoch = 0\n    \n    for epoch in range(start_epoch, args.epochs):\n\n        \n        if epoch >= args.cutmix_mixup_epoch:\n            args.cutmix = True\n            args.mixup = True\n        else:\n            args.cutmix = False\n            args.mixup = False\n        \n\n        \n        adjust_learning_rate(optimizer, epoch)\n        \n        train_auc, train_loss = train_epoch(\n            args,\n            model,\n            train_loader,\n            optimizer,\n            scheduler,\n            loss_functions,\n            device,\n            epoch\n        )\n\n        if epoch >= 0:\n\n            #para_loader = pl.ParallelLoader(valid_loader, [device])\n            valid_auc = valid_epoch(\n                args,\n                model,\n                valid_loader,\n                loss_functions,\n                device,\n                epoch\n            )\n            print(f\"Epoch : {epoch} - AUC : {valid_auc}\")\n\n            if valid_auc > best_auc:\n                print(f\"###***### Model Improved from {best_auc} to {valid_auc}\")\n                torch.save(model.state_dict(), os.path.join(args.save_path, f\"fold-{args.fold}.bin\"))\n                best_auc = valid_auc\n            \n            if epoch >= 0:\n\n                preds, img_names = test_epoch(\n                    args,\n                    model,\n                    tta_loader,\n                    device\n                )\n                #test_preds.append(preds)\n\n                #final_test_preds = np.mean(test_preds, axis=0)\n                np.save(os.path.join(args.save_path, f\"test-pred-fold-{args.fold}-epoch-{epoch}.npy\"), preds)        \n\n    \n    model.load_state_dict(torch.load(os.path.join(args.save_path, f\"fold-{args.fold}.bin\")))\n    model.to(device)    \n    preds_list = []\n    for epoch in range(args.TTA):\n\n        preds, img_names = test_epoch(\n            args,\n            model,\n            tta_loader,\n            device\n        )\n        preds_list.append(preds)\n    final_preds = np.mean(preds_list, axis=0)\n    np.save(os.path.join(args.save_path, f\"tta-pred-fold-{args.fold}.npy\"), final_preds)\n    \n    model.load_state_dict(torch.load(os.path.join(args.save_path, f\"fold-{args.fold}.bin\")))\n    model.to(device)\n\n    preds, img_names = test_epoch(\n        args,\n        model,\n        valid_loader,\n        device\n    )\n\n    off_df = pd.DataFrame({\n        \"image_name\": img_names,\n        \"prediction\": preds\n    })\n\n    off_df.to_csv(os.path.join(args.save_path, f\"off-pred-fold-{args.fold}.csv\"), index=False)\n\n\nif __name__ == \"__main__\":\n\n    skf = KFold(n_splits=args.folds, shuffle=True, random_state=args.seed)\n    for fold, (idxT, idxV) in enumerate(skf.split(np.arange(15))):\n        if fold >= 0:\n            print(\"#\"*20); print(f\"#### FOLD {fold}\");\n            main(fold, idxT, idxV)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%writefile config.py\n\nclass args:\n\n    exp_name = \"Final_NS_D201_384_224_5\"\n    output_dir = \"output\"\n    train_image_path = \"train\"\n    test_image_path = \"test\"\n\n    network = \"MelanomaClassifier\"\n    encoder = \"densenet201\"\n\n    train_csv = \"train_csv.csv\"\n    train_2019_csv = \"train_2019_csv.csv\"\n    test_csv = \"test_csv.csv\"\n\n    label_smoothing = 0.0\n    size = 224\n    normalize = {\n        \"mean\": [0.485, 0.456, 0.406],\n        \"std\": [0.229, 0.224, 0.225]\n    }\n\n\n    epochs = 5\n\n    seed = 2020\n\n    folds = 5\n\n    losses = {\n        \"BinaryCrossentropy\": 1\n    }\n    optimizer = {\n        \"type\": \"Adam\",\n        \"momentum\": 0.9,\n        \"weight_decay\": 1e-5,\n        \"learning_rate\": 0.256,\n        \"nesterov\": True,\n\n        \"schedule\": {\n            \"type\": \"poly\",\n            \"mode\": \"step\",\n            \"epoch\": 5,\n            \"params\": {\"max_iter\": 1500}\n        }\n    }\n    batch_size = 32\n\n    TTA = 10\n\n    # CUSTOM LEARNING SCHEUDLE\n    LR_START = 0.00001\n    LR_MAX = 0.00005\n    LR_MIN = 0.00001\n    LR_RAMPUP_EPOCHS = 5\n    LR_SUSTAIN_EPOCHS = 0\n    LR_EXP_DECAY = .8\n\n    learning_rate = 0.00002\n\n    fp16 = False\n    opt_level = 'O3'\n\n    cutmix_mixup_epoch = 4\n\n    mixup = False\n    alpha = 1\n\n    cutmix = False\n    beta = 0.1\n\n    psudo_label = False\n    psudo_csv = \"drive/My Drive/SIIM-ISIC Melanoma Classification/input/psudo_label.csv\"\n\n\n    malignant_csv = \"malignat_csv.csv\"\n    malignant = True\n    train_2019 = True\n\n    train_2019_fn = -0.7\n    train_fn = -0.8\n\n    train_2019_fp = 0.7\n    train_fp = 0.8","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!python3 train.py","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}