{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip -q install jcopdl efficientnet_pytorch","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport os\nfrom PIL import Image\nfrom tqdm import tqdm\nfrom time import sleep\n\nfrom sklearn.preprocessing import OneHotEncoder, StandardScaler\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torchvision.transforms import transforms\nfrom torch.utils.data import Dataset, DataLoader\n\nimport jcopdl\nfrom jcopdl.callback import Callback, set_config\n\nfrom efficientnet_pytorch import EfficientNet\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_IMAGES_PATH = \"/kaggle/input/siim-isic-train-resized/\"\nTEST_IMAGES_PATH = \"/kaggle/input/siim-isic-test-resized/\"\nTEST_DF_PATH = \"/kaggle/input/siim-isic-melanoma-classification/test.csv\"\nTRAIN_DF_PATH = \"/kaggle/input/siim-isic-melanoma-classification/train.csv\"\n\nconfig = set_config(dict(image_size=(256, 256), hidden_layer=600, dropout=0.4, batch_size=32, lr=1e-4))\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data preprocessing and loading\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class MelanomaDataset(Dataset):\n    \n    def __init__(self, df, SS, OHE, transform=None, test=False):\n        \n    \n        self.df = df\n        self._ages = SS.transform(self.df[[\"age_approx\"]])\n        self._oh_categories = OHE.transform(self.df[[\"sex\", \"anatom_site_general_challenge\"]])\n        self.transform = transform\n        self._test = test\n        \n    def __getitem__(self, idx):\n        if torch.is_tensor(idx):\n            idx = idx.tolist()\n            \n        image = Image.open(self.df[\"image_name\"].iloc[idx])\n        age = self._ages[idx]\n        oh_categories = self._oh_categories[idx]\n        meta = np.r_[age, oh_categories].astype(np.float32)\n        \n        if self.transform:\n            image = self.transform(image)\n        \n        if not self._test: \n            target = self.df[\"target\"].iloc[idx]\n            return (image, meta), target\n        else:\n            return image, meta\n        \n    \n    def __len__(self):\n        \n        return self.df.shape[0]\n        \n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MelanomaDatasetBuilder():\n    \n    def __init__(self, train_csv, test_csv):\n        \n        self.df_train = pd.read_csv(train_csv)\n        self.df_test = pd.read_csv(test_csv)\n        \n        self.df_train = self._preprocess(self.df_train)\n        self.df_test = self._preprocess(self.df_test, test=True)\n        \n        # Splitting the validation and training \n        self.df_train, self.df_val = train_test_split(self.df_train, stratify=self.df_train[\"target\"], test_size=0.1)\n        \n        self.SS, self.OHE = StandardScaler(), OneHotEncoder(sparse=False)\n        self.SS.fit(self.df_train[[\"age_approx\"]])\n        self.OHE.fit(self.df_train[[\"sex\", \"anatom_site_general_challenge\"]])\n        \n        \n    \n    def __call__(self, train_transform=None, test_transform=None):\n        \n        train_dataset = MelanomaDataset(self.df_train, self.SS, self.OHE, train_transform)\n        val_dataset = MelanomaDataset(self.df_val, self.SS, self.OHE, test_transform)\n        test_dataset = MelanomaDataset(self.df_test, self.SS, self.OHE, test_transform, test=True)\n\n        return train_dataset, val_dataset, test_dataset\n    \n    def _preprocess(self, df, test=False):\n        \n        df = df.fillna({\"sex\": \"male\",\n                                  \"age_approx\": self.df_train[\"age_approx\"].mode().iloc[0],\n                                  \"anatom_site_general_challenge\": \"torso\"})\n        \n        DIR = TEST_IMAGES_PATH if test else TRAIN_IMAGES_PATH\n        \n        df[\"image_name\"] = df[\"image_name\"].apply(lambda x : DIR + x + \".jpg\")\n        \n        return df\n        \n        \n    \n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset_builder = MelanomaDatasetBuilder(TRAIN_DF_PATH, TEST_DF_PATH)\n\ntrain_transforms = transforms.Compose([\n    transforms.RandomResizedCrop(size=config.image_size, scale=(0.7, 1.0)),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomVerticalFlip(),\n    transforms.ColorJitter(brightness=32. / 255.,saturation=0.5),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])\n])\n\ntest_transforms = transforms.Compose([\n     transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])\n])\n\ntrain_dataset, val_dataset, test_dataset = dataset_builder(train_transforms, test_transforms)\ntrain_dataloader = DataLoader(train_dataset, batch_size=config.batch_size, shuffle=True)\nval_dataloader, test_dataloader = DataLoader(val_dataset, batch_size=1), DataLoader(test_dataset, batch_size=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class LinLeakBNDrop(nn.Module):\n    \n    def __init__(self, in_features, out_features, dropout):\n        \n        super().__init__()\n        \n        self.layer = nn.Sequential(\n            nn.Linear(in_features, out_features), \n            nn.LeakyReLU(), \n            nn.BatchNorm1d(out_features), \n            nn.Dropout(dropout)\n        )\n        \n    def forward(self, x):\n        \n        return self.layer(x)\n\nclass LinELUBNDrop(nn.Module):\n    \n    def __init__(self, in_features, out_features, dropout):\n        \n        super().__init__()\n        \n        self.layer = nn.Sequential(\n            nn.Linear(in_features, out_features), \n            nn.ELU(), \n            nn.BatchNorm1d(out_features), \n            nn.Dropout(dropout)\n        )\n        \n    def forward(self, x):\n        \n        return self.layer(x)\n    \n\nclass MelanomaDense(nn.Module):\n    \n    def __init__(self, in_features, hidden_layer=400, dropout=0.3):\n        \n        super().__init__()\n        \n        self.layer = nn.Sequential(\n            LinELUBNDrop(in_features, hidden_layer, dropout), \n            LinELUBNDrop(hidden_layer, hidden_layer, dropout),\n            LinELUBNDrop(hidden_layer, hidden_layer, dropout)\n        )\n        \n        self.out_features = hidden_layer\n        \n    def forward(self, x):\n        \n        return self.layer(x)\n        \n\nclass MelanomaNet(nn.Module):\n    \n    def __init__(self, base_model):\n        \"\"\"\n        Base Model is expected to be an efficient net. \n        \"\"\"\n        \n        super().__init__()\n        \n        self.base_model = base_model\n        self.base_model._fc = LinELUBNDrop(self.base_model._fc.in_features, config.hidden_layer, config.dropout)\n#         self.base_model._fc = nn.Linear(self.base_model._fc.in_features, config.hidden_layer)\n        \n        self.dense = MelanomaDense(9, config.hidden_layer, config.dropout)\n        \n        self.output = nn.Linear(config.hidden_layer + config.hidden_layer, 1)\n        \n        \n        \n    def forward(self, x):\n        \n        image, meta = x\n        outCNN = self.base_model(image)\n        outDNN = self.dense(meta)\n        \n        features = torch.cat((outCNN, outDNN), dim=1)\n        return self.output(features)\n    \n    def freeze(self):\n        \n        for name, child in self.base_model.named_children():\n            if name != \"_fc\":\n                for param in child.parameters():\n                    param.requires_grad = False\n        \n    \n    def unfreeze(self):\n        \n        for param in self.base_model.parameters():\n            param.requires_grad = True\n        \n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base_model = EfficientNet.from_pretrained(\"efficientnet-b4\")\nmodel = MelanomaNet(base_model)\nmodel = model.to(device)\nmodel.freeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"criterion = nn.BCEWithLogitsLoss()\noptimizer= torch.optim.AdamW(model.parameters(), lr=config.lr)\nscheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode=\"max\", factor=0.2, patience=2)\ncallback = Callback(model, config, early_stop_patience=4, outdir=\"model\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ModelTrainer():\n    \n    def __init__(self, model, criterion, optimizer, scheduler,  device, callback, tqdm=True):\n        \n        self.model = model\n        self.criterion = criterion\n        self.optimizer = optimizer\n        self.scheduler = scheduler\n        self.callback = callback \n        self.device = device\n        self.tqdm = tqdm\n        \n        self.epoch = 1\n        \n        \n    def train_loop(self, train_dataloader):\n        \n        cost = 0\n        self.model.train()\n        \n        train_pred = torch.zeros(len(train_dataloader.dataset), 1, dtype=torch.float32, device = self.device, requires_grad=False)\n        train_true = torch.zeros(len(train_dataloader.dataset), 1, dtype=torch.float32, device = self.device, requires_grad=False)\n        \n        t = tqdm(train_dataloader, desc=f\"Epoch {self.epoch}\") if self.tqdm == True else train_dataloader\n        \n        for i, (features, target) in enumerate(t):\n            \n            features[0] = features[0].to(device)\n            features[1] = features[1].to(device)\n            target = target.to(device).float()\n            \n            pred = self.model(features)\n            loss = self.criterion(pred, target.unsqueeze(1))\n            loss.backward()\n            \n            self.optimizer.step()\n            self.optimizer.zero_grad()\n            \n            cost += loss.item() * features[1].shape[0]\n            \n            train_true[i*train_dataloader.batch_size:min((i+1)*train_dataloader.batch_size, len(train_dataloader.dataset))]=\\\n            target.unsqueeze(1)\n            \n            train_pred[i*train_dataloader.batch_size:min((i+1)*train_dataloader.batch_size, len(train_dataloader.dataset))]=\\\n            pred\n            \n        roc = roc_auc_score(train_true.cpu().detach().numpy(), train_pred.cpu().detach\n                            ().numpy())\n        \n        return cost / len(train_dataloader.dataset), roc\n    \n    \n    def validate_loop(self, val_dataloader):\n        \n        cost = 0\n        roc = 0\n        self.model.eval()\n        \n        val_preds = torch.zeros(len(val_dataloader.dataset), 1, dtype=torch.float32, device=self.device)\n        val_true = torch.zeros(len(val_dataloader.dataset), 1, dtype=torch.float32, device=self.device)\n        \n        t = tqdm(val_dataloader, desc=f\"Epoch {self.epoch}\") if self.tqdm == True else val_dataloader\n        \n        with torch.no_grad():\n            for i, (features, target) in enumerate(t):\n                features[0] = features[0].to(device)\n                features[1] = features[1].to(device)\n                target = target.to(device).float()\n            \n                pred = self.model(features)\n                loss = self.criterion(pred, target.unsqueeze(1)) \n                cost += loss.item() * features[1].shape[0]\n                val_preds[i] = pred\n                val_true[i] = target\n                \n        \n        roc = roc_auc_score(val_true.cpu().numpy(), val_preds.cpu().numpy())\n        \n        return (cost / len(val_dataloader.dataset)), roc\n    \n    def train(self, train_dataloader, val_dataloader, max_epochs=np.inf):\n        \n        while self.epoch < max_epochs:\n            train_cost, train_score = self.train_loop(train_dataloader)\n            val_cost, val_score = self.validate_loop(val_dataloader)\n            \n            self.scheduler.step(val_score)\n            self.epoch += 1\n            \n            _ = self.callback.log(train_cost, val_cost, train_score, val_score)\n            _ = self.callback.save_checkpoint()\n        \n            if self.callback.early_stopping(self.model, monitor=\"test_score\"):\n                self.callback.plot_cost()\n                self.callback.plot_score()\n                break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modeltrainer = ModelTrainer(model, criterion, optimizer, scheduler, device, callback, tqdm=True)\nmodeltrainer.train(train_dataloader, val_dataloader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.unfreeze()\nmodeltrainer.optimizer= torch.optim.AdamW(model.parameters(), lr=1e-5)\nmodeltrainer.scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(modeltrainer.optimizer,\n                                                                    mode=\"max\", factor=0.2, \n                                                                    patience=2)\n\nmodeltrainer.train(train_dataloader, val_dataloader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.eval()\npreds = np.zeros(len(test_dataloader.dataset))\nwith torch.no_grad():\n    for i, features in enumerate(test_dataloader):\n        features[0], features[1] = features[0].to(device), features[1].to(device)\n        pred = torch.sigmoid(model(features))\n        preds[i] = pred.cpu().item()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_sub = pd.read_csv(\"../input/siim-isic-melanoma-classification/sample_submission.csv\")\ndf_sub[\"target\"] = preds\ndf_sub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}