{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pydicom\nimport os\nimport torch\nimport pandas as pd\nimport numpy as np\nfrom torch.utils.data import Dataset, random_split, DataLoader\nfrom PIL import Image\nimport torchvision.models as models\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm\nimport torchvision.transforms as T\nfrom sklearn.metrics import f1_score\nimport torch.nn.functional as F\nimport torch.nn as nn\nfrom torchvision.utils import make_grid\nfrom sklearn.preprocessing import OneHotEncoder\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%ls '../input/siim-isic-melanoma-classification/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_DIR = '../input/siim-isic-melanoma-classification/'\nTRAIN_DIR = DATA_DIR + 'jpeg/train/'\nTEST_DIR = DATA_DIR + 'jpeg/test/'\nTRAIN_CSV = DATA_DIR + 'train.csv'\nTEST_CSV = DATA_DIR + 'test.csv'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_df = pd.read_csv(TRAIN_CSV)\ndata_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(data_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from os import listdir\nfrom os.path import isfile, join\nIMAGE_DIR = DATA_DIR+'jpeg/train/'\nall_train_images = [f for f in listdir(IMAGE_DIR) if isfile(join(IMAGE_DIR, f))]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"###validate that the all images are available of the train data\nfor index,row in data_df.iterrows():\n    if row['image_name']+\".jpg\" not in all_train_images:\n        print(\"Image {} not present for train data\".format(row['image_name']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ClassifierDataset(Dataset):\n    def __init__(self, df, root_dir, transform=None):\n        self.df = df\n        self.transform = transform\n        self.root_dir = root_dir\n        \n    def __len__(self):\n        return len(self.df)    \n    \n    def __getitem__(self, idx):\n        row = self.df.loc[idx]\n        img_id, img_label = row['image_name'], row['target']\n        img_fname = self.root_dir + \"/\" + str(img_id) + \".jpg\"\n        img = Image.open(img_fname)\n        if self.transform:\n            img = self.transform(img)\n        return img, encode_label(img_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nnp.random.seed(42)\nmsk = np.random.rand(len(data_df)) <0.9\ntrain_df = data_df[msk].reset_index()\nval_df = data_df[~msk].reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"imagenet_stats = ([0.5,0.5,0.5],[0.5,0.5,0.5])\n\ntrain_tfms = T.Compose([\n#     T.RandomCrop(256,padding=8,padding_mode='reflect'),\n    T.RandomHorizontalFlip(),\n    T.RandomRotation(10),\n    T.ToTensor(),\n])\n\nvalid_tfms = T.Compose([\n    T.ToTensor(),\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ds = ClassifierDataset(train_df,TRAIN_DIR,transform=train_tfms)\nval_ds = ClassifierDataset(val_df,TRAIN_DIR,transform=valid_tfms)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = {\n    0: '0',\n    1: '1'\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def encode_label(label):\n    target = torch.zeros(2)\n    for l in str(label).split(' '):\n        target[int(l)] = 1.\n    return target\n\ndef decode_target(target, text_labels=False, threshold=0.5):\n    result = []\n    for i, x in enumerate(target):\n        if (x >= threshold):\n            if text_labels:\n                result.append(labels[i] + \"(\" + str(i) + \")\")\n            else:\n                result.append(str(i))\n    return ' '.join(result)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_sample(img, target, invert=True):\n    if invert:\n        plt.imshow(1 - img.permute((1, 2, 0)))\n    else:\n        plt.imshow(img.permute(1, 2, 0))\n    print('Labels:', decode_target(target, text_labels=True))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_sample(*train_ds[1032])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_sample(*train_ds[3312])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 12","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dl = DataLoader(train_ds, batch_size, shuffle=True, \n                      num_workers=3, pin_memory=True)\nval_dl = DataLoader(val_ds, batch_size*2, \n                    num_workers=2, pin_memory=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_batch(dl, invert=True):\n    for images, labels in dl:\n        fig, ax = plt.subplots(figsize=(16, 8))\n        ax.set_xticks([]); ax.set_yticks([])\n        data = 1-images if invert else images\n        ax.imshow(make_grid(data, nrow=16).permute(1, 2, 0))\n        break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_batch(train_dl, invert=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Model Transfer Learning","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def F_score(output, label, threshold=0.5, beta=1):\n    prob = output > threshold\n    label = label > threshold\n\n    TP = (prob & label).sum(1).float()\n    TN = ((~prob) & (~label)).sum(1).float()\n    FP = (prob & (~label)).sum(1).float()\n    FN = ((~prob) & label).sum(1).float()\n\n    precision = torch.mean(TP / (TP + FP + 1e-12))\n    recall = torch.mean(TP / (TP + FN + 1e-12))\n    F2 = (1 + beta**2) * precision * recall / (beta**2 * precision + recall + 1e-12)\n    return F2.mean(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ImageClassificationBase(nn.Module):\n    def training_step(self, batch):\n        images, targets = batch \n        out = self(images)                      \n        loss = F.binary_cross_entropy(out, targets)      \n        return loss\n    \n    def validation_step(self, batch):\n        images, targets = batch \n        out = self(images)                           # Generate predictions\n        loss = F.binary_cross_entropy(out, targets)  # Calculate loss\n        score = F_score(out, targets)\n        return {'val_loss': loss.detach(), 'val_score': score.detach() }\n        \n    def validation_epoch_end(self, outputs):\n        batch_losses = [x['val_loss'] for x in outputs]\n        epoch_loss = torch.stack(batch_losses).mean()   # Combine losses\n        batch_scores = [x['val_score'] for x in outputs]\n        epoch_score = torch.stack(batch_scores).mean()      # Combine accuracies\n        return {'val_loss': epoch_loss.item(), 'val_score': epoch_score.item()}\n    \n    def epoch_end(self, epoch, result):\n        print(\"Epoch [{}], last_lr: {:.4f}, train_loss: {:.4f}, val_loss: {:.4f}, val_score: {:.4f}\".format(\n            epoch, result['lrs'][-1], result['train_loss'], result['val_loss'], result['val_score']))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TrainResnet(ImageClassificationBase):\n    def __init__(self):\n        super().__init__()\n        # Use a pretrained model\n        self.network = models.resnet34(pretrained=True)\n        # Replace last layer\n        num_ftrs = self.network.fc.in_features\n        self.network.fc = nn.Linear(num_ftrs, 10)\n    \n    def forward(self, xb):\n        return torch.sigmoid(self.network(xb))\n    \n    def freeze(self):\n        # To freeze the residual layers\n        for param in self.network.parameters():\n            param.require_grad = False\n        for param in self.network.fc.parameters():\n            param.require_grad = True\n    \n    def unfreeze(self):\n        # Unfreeze all layers\n        for param in self.network.parameters():\n            param.require_grad = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_default_device():\n    \"\"\"Pick GPU if available, else CPU\"\"\"\n    if torch.cuda.is_available():\n        return torch.device('cuda')\n    else:\n        return torch.device('cpu')\n    \ndef to_device(data, device):\n    \"\"\"Move tensor(s) to chosen device\"\"\"\n    if isinstance(data, (list,tuple)):\n        return [to_device(x, device) for x in data]\n    return data.to(device, non_blocking=True)\n\nclass DeviceDataLoader():\n    \"\"\"Wrap a dataloader to move data to a device\"\"\"\n    def __init__(self, dl, device):\n        self.dl = dl\n        self.device = device\n        \n    def __iter__(self):\n        \"\"\"Yield a batch of data after moving it to device\"\"\"\n        for b in self.dl: \n            yield to_device(b, self.device)\n\n    def __len__(self):\n        \"\"\"Number of batches\"\"\"\n        return len(self.dl)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = get_default_device()\ndevice","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dl = DeviceDataLoader(train_dl, device)\nval_dl = DeviceDataLoader(val_dl, device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@torch.no_grad()\ndef evaluate(model, val_loader):\n    model.eval()\n    outputs = [model.validation_step(batch) for batch in val_loader]\n    return model.validation_epoch_end(outputs)\n\ndef get_lr(optimizer):\n    for param_group in optimizer.param_groups:\n        return param_group['lr']\n\ndef fit_one_cycle(epochs, max_lr, model, train_loader, val_loader, \n                  weight_decay=0, grad_clip=None, opt_func=torch.optim.SGD):\n    torch.cuda.empty_cache()\n    history = []\n    \n    # Set up cutom optimizer with weight decay\n    optimizer = opt_func(model.parameters(), max_lr, weight_decay=weight_decay)\n    # Set up one-cycle learning rate scheduler\n    sched = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr, epochs=epochs, \n                                                steps_per_epoch=len(train_loader))\n    \n    for epoch in range(epochs):\n        # Training Phase \n        model.train()\n        train_losses = []\n        lrs = []\n        for batch in tqdm(train_loader):\n            loss = model.training_step(batch)\n            train_losses.append(loss)\n            loss.backward()\n            \n            # Gradient clipping\n            if grad_clip: \n                nn.utils.clip_grad_value_(model.parameters(), grad_clip)\n            \n            optimizer.step()\n            optimizer.zero_grad()\n            \n            # Record & update learning rate\n            lrs.append(get_lr(optimizer))\n            sched.step()\n        \n        # Validation phase\n        result = evaluate(model, val_loader)\n        result['train_loss'] = torch.stack(train_losses).mean().item()\n        result['lrs'] = lrs\n        model.epoch_end(epoch, result)\n        history.append(result)\n    return history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = to_device(TrainResnet(), device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = [evaluate(model, val_dl)]\nhistory","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.freeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = 60\nmax_lr = 0.01\ngrad_clip = 0.1\nweight_decay = 1e-4\nopt_func = torch.optim.Adam","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nhistory += fit_one_cycle(epochs, max_lr, model, train_dl, val_dl, \n                         grad_clip=grad_clip, \n                         weight_decay=weight_decay, \n                         opt_func=opt_func)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.unfreeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nhistory += fit_one_cycle(epochs, 0.001, model, train_dl, val_dl, \n                         grad_clip=grad_clip, \n                         weight_decay=weight_decay, \n                         opt_func=opt_func)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# data_df['anatom_site_general_challenge'] = data_df['anatom_site_general_challenge'].apply(lambda x : str(x).split(\"/\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"anatom_site_general_challenge = (data_df.anatom_site_general_challenge.str.split('\\s*/\\s*', expand=True)\n   .stack()\n   .str.get_dummies()\n   .sum(level=0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_df.join(anatom_site_general_challenge)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_df.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# creating instance of one-hot-encoder\nenc = OneHotEncoder(handle_unknown='ignore')# passing bridge-types-cat column (label encoded values of bridge_types)\nenc_df = pd.DataFrame(enc.fit_transform(data_df[['diagnosis','sex']]).toarray())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}