{"cells":[{"metadata":{},"cell_type":"markdown","source":"# 1. Imports and Configuration\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -U efficientnet_pytorch","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport cv2\nimport random\nimport matplotlib.pyplot as plt\nimport torch\nimport torch.nn as nn\nimport torchvision\nimport torchvision.models as models\nfrom torchvision import transforms\nfrom torch.utils.data import Dataset, random_split\nfrom torch.autograd import Variable\nfrom PIL import Image\nfrom tqdm.notebook import tqdm\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.preprocessing import LabelEncoder, OneHotEncoder\nfrom sklearn.model_selection import train_test_split\nfrom xgboost import XGBRegressor\nfrom sklearn.model_selection import RandomizedSearchCV\n\nfrom efficientnet_pytorch import EfficientNet\n\n\nfrom collections import OrderedDict\n\nimport torch.nn.functional as F\n\n\n# We are using this image dataset that has already been treated so that every image has the same size\nbase_dir = '/kaggle/input/jpeg-melanoma-384x384/'\n# Running everything on the GPU\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n# Model saving/loading path\nPATH = 'efficient_net.pth'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# This is the largest base2 we can fit in GPU memory\nBATCH_SIZE = 16\nNUM_EPOCHS = 30\n# Early stopping patience\nPATIENCE = 5\nlearning_rate = 0.001\nloss_func = nn.BCELoss()\nrandom_seed = 42","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2. Load Data","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"# Here we are defining how we will load our data\nclass CustomImageDataset(Dataset):\n    def __init__(self, path, image_names, labels=None, transforms=None):\n        super().__init__()\n        self.path = path\n        self.image_names = image_names\n        self.labels = labels\n        self.transforms = transforms\n\n    def __getitem__(self, idx: int):\n        # We simply return the image and label for the given index\n        im_path = os.path.join(self.path, self.image_names[idx] + '.jpg')\n        image = cv2.imread(im_path)\n\n        if self.transforms:\n            image = self.transforms(image)\n            \n        if self.labels is None: \n            # Or just the image if we don't have labels (test set)\n            return image\n        return image, float(self.labels[idx])\n\n    def __len__(self) -> int:\n        return self.image_names.shape[0]\n\n    def get_labels(self):\n        return list(self.labels)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Set up Transformations","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class Microscope:\n    \"\"\"\n    Cutting out the edges around the center circle of the image\n    Imitating a picture, taken through the microscope\n\n    Args:\n        p (float): probability of applying an augmentation\n    \"\"\"\n\n    def __init__(self, p: float = 0.5):\n        self.p = p\n\n    def __call__(self, img):\n        \"\"\"\n        Args:\n            img (PIL Image): Image to apply transformation to.\n\n        Returns:\n            PIL Image: Image with transformation.\n        \"\"\"\n        if random.random() < self.p:\n            circle = cv2.circle((np.ones(img.shape) * 255).astype(np.uint8), # image placeholder\n                        (img.shape[0]//2, img.shape[1]//2), # center point of circle\n                        random.randint(img.shape[0]//2 - 3, img.shape[0]//2 + 15), # radius\n                        (0, 0, 0), # color\n                        -1)\n\n            mask = circle - 255\n            img = np.multiply(img, mask)\n        \n        return img\n\n    def __repr__(self):\n        return f'{self.__class__.__name__}(p={self.p})'\n    \nclass DrawHair:\n    \"\"\"\n    Draw a random number of pseudo hairs\n\n    Args:\n        hairs (int): maximum number of hairs to draw\n        width (tuple): possible width of the hair in pixels\n    \"\"\"\n\n    def __init__(self, hairs:int = 4, width:tuple = (1, 2)):\n        self.hairs = hairs\n        self.width = width\n\n    def __call__(self, img):\n        \"\"\"\n        Args:\n            img (PIL Image): Image to draw hairs on.\n\n        Returns:\n            PIL Image: Image with drawn hairs.\n        \"\"\"\n        if not self.hairs:\n            return img\n        \n        width, height, _ = img.shape\n        \n        for _ in range(random.randint(0, self.hairs)):\n            # The origin point of the line will always be at the top half of the image\n            origin = (random.randint(0, width), random.randint(0, height // 2))\n            # The end of the line \n            end = (random.randint(0, width), random.randint(0, height))\n            color = (0, 0, 0)  # color of the hair. Black.\n            cv2.line(img, origin, end, color, random.randint(self.width[0], self.width[1]))\n        \n        return img\n\n    def __repr__(self):\n        return f'{self.__class__.__name__}(hairs={self.hairs}, width={self.width})'\n    \nclass AdvancedHairAugmentation:\n    \"\"\"\n    Impose an image of a hair to the target image\n\n    Args:\n        hairs (int): maximum number of hairs to impose\n        hairs_folder (str): path to the folder with hairs images\n    \"\"\"\n\n    def __init__(self, hairs: int = 5, hairs_folder: str = \"\"):\n        self.hairs = hairs\n        self.hairs_folder = hairs_folder\n\n    def __call__(self, img):\n        \"\"\"\n        Args:\n            img (PIL Image): Image to draw hairs on.\n\n        Returns:\n            PIL Image: Image with drawn hairs.\n        \"\"\"\n        n_hairs = random.randint(0, self.hairs)\n        \n        if not n_hairs:\n            return img\n        \n        height, width, _ = img.shape  # target image width and height\n        hair_images = [im for im in os.listdir(self.hairs_folder) if 'png' in im]\n        \n        for _ in range(n_hairs):\n            hair = cv2.imread(os.path.join(self.hairs_folder, random.choice(hair_images)))\n            hair = cv2.flip(hair, random.choice([-1, 0, 1]))\n            hair = cv2.rotate(hair, random.choice([0, 1, 2]))\n\n            h_height, h_width, _ = hair.shape  # hair image width and height\n            roi_ho = random.randint(0, img.shape[0] - hair.shape[0])\n            roi_wo = random.randint(0, img.shape[1] - hair.shape[1])\n            roi = img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width]\n\n            # Creating a mask and inverse mask\n            img2gray = cv2.cvtColor(hair, cv2.COLOR_BGR2GRAY)\n            ret, mask = cv2.threshold(img2gray, 10, 255, cv2.THRESH_BINARY)\n            mask_inv = cv2.bitwise_not(mask)\n\n            # Now black-out the area of hair in ROI\n            img_bg = cv2.bitwise_and(roi, roi, mask=mask_inv)\n\n            # Take only region of hair from hair image.\n            hair_fg = cv2.bitwise_and(hair, hair, mask=mask)\n\n            # Put hair in ROI and modify the target image\n            dst = cv2.add(img_bg, hair_fg)\n\n            img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width] = dst\n                \n        return img\n\n    def __repr__(self):\n        return f'{self.__class__.__name__}(hairs={self.hairs}, hairs_folder=\"{self.hairs_folder}\")'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Reading metadata\ntrain_df_csv = pd.read_csv(base_dir + 'train.csv') \ntest_df_csv = pd.read_csv(base_dir + 'test.csv') \n\n# Preprocessor for images\ntrain_transform = transforms.Compose([\n    AdvancedHairAugmentation(hairs_folder='/kaggle/input/melanoma-hairs'),\n    Microscope(p=0.5),\n    transforms.ToPILImage(),\n    transforms.RandomResizedCrop(size=384, scale=(0.8, 1.0)),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomVerticalFlip(),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])\n])\ntest_transform = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])\n])\n\n# Define datasets\nds_train = CustomImageDataset(\n    path= base_dir + 'train/',\n    image_names=train_df_csv.image_name.values,\n    labels=train_df_csv.target.values,\n    transforms=train_transform\n)\nds_test = CustomImageDataset(\n    path= base_dir + 'test/',\n    image_names=test_df_csv.image_name.values,\n    transforms=test_transform\n)\n\n\n# Split into train/validation\nlen_ds=len(ds_train)\nlen_train=int(0.85*len_ds)\nlen_val=len_ds-len_train\n\ntorch.manual_seed(random_seed)\nds_train,ds_val=random_split(ds_train,[len_train,len_val])\n\n# Create the data loaders that will provide us with data in batches to reduce memory usage\n# Note that we only shuffle the training set, the rest is unecessary \ntrain_loader = torch.utils.data.DataLoader(ds_train, batch_size = BATCH_SIZE, shuffle = True)\nval_loader = torch.utils.data.DataLoader(ds_val, batch_size = BATCH_SIZE*2, shuffle = False)\ntest_loader = torch.utils.data.DataLoader(ds_test, batch_size = BATCH_SIZE*2, shuffle = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Treat metadata for the XGB further down the line\n\n# Fill nulls\ntrain_df_csv['sex'].fillna('male', inplace=True)\ntest_df_csv['sex'].fillna('male', inplace=True)\n\ntrain_df_csv['age_approx'].fillna(train_df_csv['age_approx'].mode().values[0], inplace=True)\ntest_df_csv['age_approx'].fillna(test_df_csv['age_approx'].mode().values[0], inplace=True)\n\ntrain_df_csv['anatom_site_general_challenge'].fillna('torso', inplace=True)\ntest_df_csv['anatom_site_general_challenge'].fillna('torso', inplace=True)\n\n# Normalize age\ntrain_df_csv['age_enc'] = train_df_csv['age_approx'] / np.mean(train_df_csv['age_approx'])\ntest_df_csv['age_enc'] = test_df_csv['age_approx'] / np.mean(test_df_csv['age_approx'])\n\n# Encode categorical to one hot\n\ntrain_df_csv = pd.get_dummies(train_df_csv, columns=['sex', 'anatom_site_general_challenge'])\ntest_df_csv = pd.get_dummies(test_df_csv, columns=['sex', 'anatom_site_general_challenge'])\n\n# Create the sets\nX_train_full = train_df_csv[list(train_df_csv.columns[9:])]\ny_train_full = train_df_csv.target\n\nX_test = test_df_csv[list(test_df_csv.columns[5:])]\n\n# Train/ validation split\nX_train, X_val, y_train, y_val = train_test_split(X_train_full, y_train_full, test_size=0.15, random_state=random_seed)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3. Setup Efficient Net","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class CustomEfficientNet(nn.Module):\n    def __init__(self):\n        super().__init__()\n        # We use the backbone from the pretrained efficient net\n        self.arch = EfficientNet.from_pretrained('efficientnet-b1')\n        \n        # And add to it a final linear layer\n        self.arch._fc = nn.Linear(in_features=1280, out_features=500, bias=True)\n        \n        # And this will be our output\n        self.ouput = nn.Linear(500, 1)\n        \n    def forward(self, x):\n        # We simply go through the efficient net and through our linear layer\n        x = self.arch(x)\n        x = self.ouput(x)\n        # We return a sigmoid so that we can have a result from 0-1\n        return torch.sigmoid(x)\n    \n\n# Actually define the model and send it to the GPU\nmodel = CustomEfficientNet()\nmodel.to(device)\n\n# Use Adam optimizer with a decreasing learning rate\noptimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\nscheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer=optimizer, mode='max', patience=1, verbose=True, factor=0.2)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 4. Train Efficient Net","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def accuracy_batch(output, target):\n    # Create an accuracy score for the batch\n    \n    # get output class\n    pred = torch.round(output)\n    \n    # compare output class with target class\n    corrects=pred.eq(target.view_as(pred)).sum().item()\n    return corrects\n\n\ndef auc_epoch(output, target):\n    # Create AUC for the given epoch\n    #     Note that we can't do this for a batch since it may not have \n    #     all classes defined\n    return roc_auc_score(target, output)\n\ndef train_val(epochs, model, loss_func, opt, train_dl, val_dl, sanity_check=False):    \n    # history of loss values in each epoch\n    loss_history={\n        \"train\": [],\n        \"val\": [],\n    }\n    \n    # history of metric values in each epoch\n    metric_history={\n        \"train\": [],\n        \"val\": [],\n    }\n    \n    # initialize best loss to a small value\n    best_metric=float('-inf')\n    \n    # initialize patience\n    patience = PATIENCE\n    \n    # The actual training loop\n    for epoch in range(epochs):\n        # train model on training dataset\n        model.train()\n        train_loss,train_metric=loss_epoch(model,loss_func,train_dl,opt)\n        \n        # collect loss and metric for training dataset\n        loss_history[\"train\"].append(train_loss)\n        metric_history[\"train\"].append(train_metric)\n        \n        # evaluate model on validation dataset    \n        model.eval()\n        with torch.no_grad():\n            val_loss, val_metric=loss_epoch(model, loss_func, val_dl)\n            \n        # Update our learning rate\n        scheduler.step(val_metric)\n        \n        # collect loss and metric for validation dataset\n        loss_history[\"val\"].append(val_loss)\n        metric_history[\"val\"].append(val_metric)\n        \n        # Print epoch metrics\n        print(\"epoch: %d, train loss: %.6f, val loss: %.6f, roc_auc: %.2f\" %(epoch, train_loss, val_loss, val_metric))\n        \n        # store if it is our best model\n        if val_metric > best_metric:\n            patience = PATIENCE\n            best_metric = val_metric\n            # store weights into a local file\n            torch.save(model.state_dict(), PATH)\n            print(\"Copied best model weights!\")\n        else: \n            # If our score didn't improve we will update our patience\n            patience -= 1\n            if patience == 0:\n                # If we reach our patience limit then do early stopping\n                print(\"Early stopping, best Val roc: {:-3f}\".format(best_metric))\n                break\n       \n    return model, loss_history, metric_history, epoch\n\n\ndef loss_epoch(model,loss_func,dataset_dl,opt=None, sanity_check=False):\n    running_loss=0.0\n    running_metric=0.0\n    len_data=len(dataset_dl.dataset)\n    \n    # If we are validating the epoch we want to keep the predictions and targets\n    #     to send them to our AUC calculation\n    if opt is None: \n        val_preds = []\n        targets = []\n\n    # Run through one epoch\n    for xb, yb in tqdm(dataset_dl):\n        # move batch to device\n        xb=xb.to(device)\n        yb=Variable(yb).float()\n        yb=yb.to(device)\n        \n        # get model output\n        output=model(xb)\n        \n        # Save the preds and targets if validating\n        if opt is None:\n            val_preds += list(output.cpu().detach())\n            targets += list(yb.cpu().detach())\n        \n        # get loss per batch\n        loss_b,metric_b=loss_batch(loss_func, output, yb, opt)\n        \n        # update running loss\n        running_loss+=loss_b\n        \n        # update running metric\n        if metric_b is not None and opt is not None:\n            running_metric+=metric_b\n\n        # break the loop in case of sanity check\n        if sanity_check is True:\n            break\n    \n    # average loss value\n    loss=running_loss/float(len_data)\n    \n    # average metric value\n    if opt is not None: \n        metric=running_metric/float(len_data)\n    # If validating we want to calculate the AUC\n    else: \n        metric= auc_epoch(val_preds, targets)\n    \n    return loss, metric\n\n\ndef loss_batch(loss_func, output, target, opt=None):\n    # get loss \n    loss = loss_func(output, target.unsqueeze(1))\n    \n    # get performance metric\n    metric_b = accuracy_batch(output,target)\n    \n    # Do the back propagation\n    if opt is not None:\n        opt.zero_grad()\n        loss.backward()\n        opt.step()\n\n    return loss.item(), metric_b","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model, loss_history, metric_history, epochs_ran = train_val(NUM_EPOCHS, model, loss_func, optimizer, train_loader, val_loader)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 5. Evaluate Efficient Net","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train-Validation Progress\n\n# plot loss progress\nplt.title(\"Train-Val Loss\")\nplt.plot(range(0,epochs_ran+1),loss_history[\"train\"],label=\"train\")\nplt.plot(range(0,epochs_ran+1),loss_history[\"val\"],label=\"val\")\nplt.ylabel(\"Loss\")\nplt.xlabel(\"Training Epochs\")\nplt.legend()\nplt.show()\n\n# plot auc progress\n\nplt.title(\"Val AUC\")\nplt.plot(range(0,epochs_ran+1),metric_history[\"val\"],label=\"val\")\nplt.ylabel(\"AUC\")\nplt.xlabel(\"Training Epochs\")\nplt.legend()\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Run our model through our validation set and check our AUC\ntotal_outputs = []\ntotal_labels = []\nwith torch.no_grad():\n    for i, data in enumerate(tqdm(val_loader), 0):\n        # Get the prediction\n        inputs, labels = data\n        inputs = inputs.to(device)\n        output = model(inputs)\n        \n        #Move the prediction back to the cpu and append it to the list\n        output = output.cpu().detach()\n        total_outputs += list(output)\n        total_labels += labels\n\nprint('roc score:',roc_auc_score(total_labels, total_outputs))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 6. Setup XGB and train XGB","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# These parameters were tuned by using GridSearch (Thanks to @Md Awsafur Rahman)\nxgb = XGBRegressor(base_score=0.5, booster=None, colsample_bylevel=1,\n             colsample_bynode=1, colsample_bytree=0.8, gamma=1, gpu_id=-1,\n             importance_type='gain', interaction_constraints=None,\n             learning_rate=0.002, max_delta_step=0, max_depth=10,\n             min_child_weight=1, missing=None, monotone_constraints=None,\n             n_estimators=700, n_jobs=-1, nthread=-1, num_parallel_tree=1,\n             objective='binary:logistic', random_state=0, reg_alpha=0,\n             reg_lambda=1, scale_pos_weight=1, subsample=0.8,\n             tree_method=None, validate_parameters=False, verbosity=None)\n\nxgb.fit(X_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 7. Create Submission with ensemble","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Run our Neural Net model throught the Test set and get the predictions\nnet_outputs = []\nwith torch.no_grad():\n    for i, data in enumerate(tqdm(test_loader), 0):\n        inputs = data\n        inputs = inputs.to(device)\n\n        output = model(inputs)\n        net_outputs += output\n\n# Get predictions back from GPU\npreds_net = [output.data.cpu().numpy().item() for output in net_outputs]\n\n# Get XGB predictions\npreds_xgb = xgb.predict(X_test)\n\n# Create Dataframe with predictions\npred_df = pd.DataFrame({'image_name': test_df_csv.image_name, 'preds_net': preds_net, 'preds_xgb': preds_xgb})\n# Set the final target as a result from both our models\npred_df['target'] = (pred_df['preds_net'] * 0.9) + (pred_df['preds_xgb'] * 0.1)\n\n# Create the csv file to submit\npred_df[['image_name', 'target']].to_csv('submission_ensemble.csv', index=False)\n\n# Create csv with only nn to check if its better\npred_df['target'] = pred_df['preds_net']\npred_df[['image_name', 'target']].to_csv('submission_efficientNet.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 8. Extras and Experiments\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Augment data \n# Wheights for classes\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# LOADING MODEL\n#model = CustomEfficientNet()\n#model.load_state_dict(torch.load('/kaggle/input/very-simple-pytorch-efficientnet//efficient_net.pth'))\n#model.to(device)\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}