{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"!pip install wtfml\n!pip install pretrainedmodels","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\nimport torch\nimport albumentations\nimport numpy as np\nimport pandas as pd\n\nimport torch.nn as nn\nfrom sklearn import metrics\nfrom sklearn import model_selection\nfrom torch.nn import functional as F\n\nfrom wtfml.utils import EarlyStopping\nfrom wtfml.engine import Engine\nfrom wtfml.data_loaders.image import ClassificationLoader\n\nimport pretrainedmodels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras import backend as K\n\n\ndef focal_loss_label_smothing(gamma=2.0, pos_weight=1, label_smoothing=0.05):\n    \"\"\" binary focal loss with label_smoothing \"\"\"\n\n    def binary_focal_loss(labels, p):\n        \"\"\" bfl clojure \"\"\"\n        labels = tf.dtypes.cast(labels, dtype=p.dtype)\n        if label_smoothing is not None:\n            labels = (1 - label_smoothing) * labels + label_smoothing * 0.5\n\n        # Predicted probabilities for the negative class\n        q = 1 - p\n\n        # For numerical stability (so we don't inadvertently take the log of 0)\n        p = tf.math.maximum(p, K.epsilon())\n        q = tf.math.maximum(q, K.epsilon())\n\n        # Loss for the positive examples\n        pos_loss = -(q ** gamma) * tf.math.log(p) * pos_weight\n\n        # Loss for the negative examples\n        neg_loss = -(p ** gamma) * tf.math.log(q)\n\n        # Combine loss terms\n        loss = labels * pos_loss + (1 - labels) * neg_loss\n\n        return loss\n\n    return binary_focal_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pip install pretrainedmodels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport pretrainedmodels\nimport torch\nimport torch.nn as nn\nimport pretrainedmodels\n\n\ndef l2_norm(input, axis=1):\n    norm = torch.norm(input, 2, axis, True)\n    output = torch.div(input, norm)\n    return output\n\n\nclass BinaryHead(nn.Module):\n    def __init__(self, num_class=1, emb_size=2048, s=16.0):\n        super(BinaryHead, self).__init__()\n        self.s = s\n        self.fc = nn.Sequential(nn.Linear(emb_size, num_class))\n\n    def forward(self, fea):\n        fea = l2_norm(fea)\n        logit = self.fc(fea) * self.s\n        return logit\n\n\nclass SEResnext50_32x4d(nn.Module):\n    def __init__(self):\n        super(SEResnext50_32x4d, self).__init__()\n\n        self.model_ft = nn.Sequential(\n            *list(pretrainedmodels.__dict__[\"se_resnext50_32x4d\"](num_classes=1000, pretrained=\"imagenet\").children())[\n                :-2\n            ]\n        )\n        self.avg_pool = nn.AdaptiveAvgPool2d((1, 1))\n        self.model_ft.last_linear = None\n        self.fea_bn = nn.BatchNorm1d(2048)\n        self.fea_bn.bias.requires_grad_(False)\n        self.binary_head = BinaryHead(1, emb_size=2048, s=1)\n        self.dropout = nn.Dropout(p=0.2)\n        \n    def forward(self, image, targets):\n        batch_size, _, _, _ = image.shape\n        \n        img_feature = self.model_ft(image)\n        img_feature = self.avg_pool(img_feature)\n        img_feature = img_feature.view(img_feature.size(0), -1)\n        fea = self.fea_bn(img_feature)\n        # fea = self.dropout(fea)\n        output = self.binary_head(fea)\n        loss = nn.BCEWithLogitsLoss()(output, targets.view(-1, 1).type_as(output))\n\n        return out, loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create folds\ndf = pd.read_csv(\"../input/siim-isic-melanoma-classification/train.csv\")\ndf[\"kfold\"] = -1    \ndf = df.sample(frac=1).reset_index(drop=True)\ny = df.target.values\nkf = model_selection.StratifiedKFold(n_splits=5)\n\nfor f, (t_, v_) in enumerate(kf.split(X=df, y=y)):\n    df.loc[v_, 'kfold'] = f\n\ndf.to_csv(\"train_folds.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\n\ndef train(fold):\n    print(f\"FOLD NUMBER: {fold}\")\n    training_data_path = \"../input/siic-isic-224x224-images/train/\"\n    df = pd.read_csv(\"/kaggle/working/train_folds.csv\")\n    device = \"cuda\"\n    epochs = 7\n    train_bs = 32\n    valid_bs = 16\n\n    df_train = df[df.kfold != fold].reset_index(drop=True)\n    df_valid = df[df.kfold == fold].reset_index(drop=True)\n\n    model = SEResnext50_32x4d()\n    model.to(device)\n\n    mean = (0.485, 0.456, 0.406)\n    std = (0.229, 0.224, 0.225)\n    train_aug = albumentations.Compose(\n        [\n            albumentations.OneOf([albumentations.RandomBrightness(limit=0.1, p=1), albumentations.RandomContrast(limit=0.1, p=1)]),\n            albumentations.OneOf([albumentations.MotionBlur(blur_limit=3), albumentations.MedianBlur(blur_limit=3), albumentations.GaussianBlur(blur_limit=3)], p=0.5),\n            albumentations.VerticalFlip(p=0.5),\n            albumentations.HorizontalFlip(p=0.5),\n            albumentations.ShiftScaleRotate(\n                shift_limit=0.2,\n                scale_limit=0.2,\n                rotate_limit=20,\n                interpolation=cv2.INTER_LINEAR,\n                border_mode=cv2.BORDER_REFLECT_101,\n                p=1,\n            ),\n            albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True)\n        ]\n    )\n\n    valid_aug = albumentations.Compose(\n        [\n            albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True)\n        ]\n    )\n\n    train_images = df_train.image_name.values.tolist()\n    train_images = [os.path.join(training_data_path, i + \".png\") for i in train_images]\n    train_targets = df_train.target.values\n\n    valid_images = df_valid.image_name.values.tolist()\n    valid_images = [os.path.join(training_data_path, i + \".png\") for i in valid_images]\n    valid_targets = df_valid.target.values\n\n    train_dataset = ClassificationLoader(\n        image_paths=train_images,\n        targets=train_targets,\n        resize=None,\n        augmentations=train_aug,\n    )\n\n    train_loader = torch.utils.data.DataLoader(\n        train_dataset, batch_size=train_bs, shuffle=True, num_workers=4\n    )\n\n    valid_dataset = ClassificationLoader(\n        image_paths=valid_images,\n        targets=valid_targets,\n        resize=None,\n        augmentations=valid_aug,\n    )\n\n    valid_loader = torch.utils.data.DataLoader(\n        valid_dataset, batch_size=valid_bs, shuffle=False, num_workers=4\n    )\n\n    optimizer = torch.optim.Adam(model.parameters(), lr=1e-4)\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n        optimizer,\n        patience=3,\n        threshold=0.001,\n        mode=\"max\"\n    )\n\n    es = EarlyStopping(patience=5, mode=\"max\")\n    # model to apex\n    #model, optimizer = amp.initialize(model, optimizer, opt_level=\"O1\")\n\n    for epoch in range(epochs):\n        train_loss = Engine.train(train_loader, model, optimizer, device=device,fp16=False)\n        predictions, valid_loss = Engine.evaluate(\n            valid_loader, model, device=device\n        )\n        predictions = np.vstack((predictions)).ravel()\n        auc = metrics.roc_auc_score(valid_targets, predictions)\n        print(f\"Epoch = {epoch}, AUC = {auc}\")\n        scheduler.step(auc)\n\n        es(auc, model, model_path=f\"model_fold_{fold}.bin\")\n        if es.early_stop:\n            print(\"Early stopping\")\n            break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict(fold):\n    print(f'FOLD: {fold}\\n')\n    test_data_path = \"../input/siic-isic-224x224-images/test/\"\n    df = pd.read_csv(\"../input/siim-isic-melanoma-classification/test.csv\")\n    device = \"cuda\"\n    model_path=f\"model_fold_{fold}.bin\"\n\n    mean = (0.485, 0.456, 0.406)\n    std = (0.229, 0.224, 0.225)\n    aug = albumentations.Compose(\n        [\n            albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True)\n        ]\n    )\n\n    images = df.image_name.values.tolist()\n    images = [os.path.join(test_data_path, i + \".png\") for i in images]\n    targets = np.zeros(len(images))\n\n    test_dataset = ClassificationLoader(\n        image_paths=images,\n        targets=targets,\n        resize=None,\n        augmentations=aug,\n    )\n\n    test_loader = torch.utils.data.DataLoader(\n        test_dataset, batch_size=16, shuffle=False, num_workers=4\n    )\n\n    model = SEResnext50_32x4d(pretrained=None)\n    model.load_state_dict(torch.load(model_path))\n    model.to(device)\n\n    predictions = Engine.predict(test_loader, model, device=device)\n    predictions = np.vstack((predictions)).ravel()\n\n    return predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train(0)\ntrain(1)\ntrain(2)\ntrain(3)\ntrain(4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"p1 = predict(0)\np2 = predict(1)\np3 = predict(2)\np4 = predict(3)\np5 = predict(4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = (p1 + p2 + p3 + p4 + p5) / 5\nsample = pd.read_csv(\"../input/siim-isic-melanoma-classification/sample_submission.csv\")\nsample.loc[:, \"target\"] = predictions\nsample.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}