{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\nimport torch\nimport torchvision\nimport pandas as pd\nimport torch.nn as nn\nfrom tqdm.notebook import tqdm\nimport torch.nn.functional as F\nimport matplotlib.pyplot as plt\nimport torchvision.transforms as T\nimport torchvision.models as models\nfrom torch.utils.data import Dataset\nfrom torch.utils.data import DataLoader\nfrom torchvision.utils import make_grid\nfrom torch.utils.data import random_split\nfrom torchvision.datasets.folder import default_loader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_DIR = \"../input/siim-isic-melanoma-classification/jpeg/train\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/siim-isic-melanoma-classification/train.csv\")\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MyDataset(Dataset):\n    def __init__(self, root_dir, csv_file, transform=None):\n        self.label = pd.read_csv(csv_file)\n        self.root_dir = root_dir\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.label)\n    \n    def __getitem__(self, idx):\n        if torch.is_tensor(idx):\n            idx = idx.to_list()\n            \n        image_path = os.path.join(self.root_dir, f\"{self.label.iloc[idx, 0]}.jpg\")\n        image_label = self.label.iloc[idx, 7]\n        \n        image = default_loader(image_path)\n        \n        if self.transform:\n            image = self.transform(image)\n            \n        return (image, image_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transform_train = T.Compose([\n    T.Resize((128,128)),\n    T.RandomHorizontalFlip(),\n    T.RandomVerticalFlip(),\n    T.ToTensor(),\n    #T.RandomErasing()\n])\n\ntrain_ds = MyDataset(\n    root_dir=TRAIN_DIR,\n    csv_file=\"../input/siim-isic-melanoma-classification/train.csv\",\n    transform=transform_train\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"images, labels = train_ds[10]\nprint(\"Label:\", labels)\nplt.imshow(images.permute(1,2,0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size=128","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_ds_size= int(len(train_ds) * 0.1)\ntrain_ds_size= len(train_ds) - val_ds_size\n\ntrain_ds, val_ds = random_split(train_ds, [train_ds_size, val_ds_size])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"train_ds lenght:\", len(train_ds))\nprint(\"val_ds lenght:\", len(val_ds))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dl = DataLoader(train_ds, batch_size, shuffle=True, num_workers=4, pin_memory=True)\nval_dl = DataLoader(val_ds, batch_size, num_workers=4, pin_memory=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Moving the data to the GPU","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_device():\n    if torch.cuda.is_available():\n        return torch.device(\"cuda\")\n    else:\n        return torch.device(\"cpu\")\n    \ndef to_device(data, device):\n    if isinstance(data, (list,tuple)):\n        return [to_device(x, device) for x in data]\n    return data.to(device, non_blocking=True)\n\nclass DeviceDataLoader():\n    def __init__(self, dl, device):\n        self.dl = dl\n        self.device = device\n        \n    def __iter__(self):\n        for x in self.dl:\n            yield to_device(x, self.device)\n            \n    def __len__(self):\n        return len(self.dl)\n    \ndevice = get_device()\ndevice","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dl = DeviceDataLoader(train_dl, device)\nval_dl = DeviceDataLoader(val_dl, device)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## The model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def accuracy(out, labels):\n    _, preds = torch.max(out, dim=1)\n    return torch.tensor(torch.sum(preds == labels).item() / len(preds))\n\nclass ImageClassificationBase(nn.Module):\n    def training_step(self, batch):\n        images, labels = batch\n        out = self(images)\n        loss = F.cross_entropy(out, labels)\n        return loss\n    \n    def validation_step(self, batch):\n        images, labels = batch\n        out = self(images)\n        loss = F.cross_entropy(out, labels)\n        acc = accuracy(out, labels)\n        return {\"val_loss\": loss.detach(), \"val_acc\": acc}\n    \n    def validation_epoch_end(self, outputs):\n        batch_loss = [x[\"val_loss\"] for x in outputs]\n        epoch_loss = torch.stack(batch_loss).mean()\n        batch_acc = [x[\"val_acc\"] for x in outputs]\n        epoch_acc = torch.stack(batch_acc).mean()\n        return {\"val_loss\": epoch_loss.item(), \"val_acc\": epoch_acc.item()}\n    \n    def epoch_end(self, epoch, epochs, result):\n        print(\"Epoch: [{}/{}], last_lr: {:.6f}, train_loss: {:.4f}, val_loss: {:.4f}, val_acc: {:.4f}\".format(\n            epoch+1, epochs, result[\"lrs\"][-1], result[\"train_loss\"], result[\"val_loss\"], result[\"val_acc\"]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class resnet(ImageClassificationBase):\n    def __init__(self):\n        super().__init__()\n        self.network = models.resnet34(pretrained=True)\n        number_of_features = self.network.fc.in_features\n        self.network.fc = nn.Linear(number_of_features, 2)\n        \n    def forward(self, xb):\n        return self.network(xb)\n    \n    def freeze(self):\n        for param in self.network.parameters():\n            param.requires_grad=False\n        for param in self.network.fc.parameters():\n            param.requires_grad=True\n            \n    def unfreeze(self):\n        for param in self.network.parameters():\n            param.requires_grad=True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = to_device(resnet(), device)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"@torch.no_grad()\ndef evaluate(model, val_dl):\n    outputs = [model.validation_step(batch) for batch in val_dl]\n    return model.validation_epoch_end(outputs)\n\ndef get_lr(optimizer):\n    for param_group in optimizer.param_groups:\n        return param_group[\"lr\"]\n        \ndef fit_one_cycle(epochs, max_lr, model, train_dl, val_dl, weight_decay=0,\n                 grad_clip=None, opt_func=torch.optim.Adam):\n    \n    history = []\n    opt = opt_func(model.parameters(), max_lr, weight_decay=weight_decay)\n    sched = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr, epochs=epochs,\n                                               steps_per_epoch=len(train_dl))\n    \n    for epoch in range(epochs):\n        model.train()\n        train_loss = []\n        lrs = []\n        for batch in tqdm(train_dl):\n            loss = model.training_step(batch)\n            train_loss.append(loss)\n            loss.backward()\n            \n            if grad_clip:\n                nn.utils.clip_grad_value_(model.parameters(), grad_clip)\n                \n            opt.step()\n            opt.zero_grad()\n            \n            lrs.append(get_lr(opt))\n            sched.step()\n            \n        result = evaluate(model, val_dl)\n        result[\"train_loss\"] = torch.stack(train_loss).mean().item()\n        result[\"lrs\"] = lrs\n        model.epoch_end(epoch, epochs, result)\n        history.append(result)\n    return history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = [evaluate(model, val_dl)]\nhistory","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.freeze() #freeze all the layers (which are already trained) and train only the last one.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs= 5\nmax_lr = 10e-5\ngrad_clip = 0.1\nweight_decay = 10e-4\nopt_func = torch.optim.Adam","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nhistory += fit_one_cycle(epochs, max_lr, model, train_dl, val_dl,\n                        weight_decay=weight_decay, grad_clip=grad_clip,\n                        opt_func=opt_func)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.unfreeze() #unfreeze all the layers and train for some more","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nhistory += fit_one_cycle(epochs, max_lr, model, train_dl, val_dl,\n                        weight_decay=weight_decay, grad_clip=grad_clip,\n                        opt_func=opt_func)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}