{"cells":[{"metadata":{"_uuid":"7cb794ce2b4542b28902b5bef1e1c9bb56f78d7e"},"cell_type":"markdown","source":"### General information\n\nIn this kernel I'll work with data from Histopathologic Cancer Detection Challenge. Our task is to identify metastatic cancer in small image patches taken from larger digital pathology scans. Or to be more precise - to classify 32x32 center crops as having cancer or not."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# libraries\nimport numpy as np\nimport pandas as pd\nimport os\nimport cv2\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\nimport torch\nfrom torch.utils.data import TensorDataset, DataLoader,Dataset\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\nimport torchvision.transforms as transforms\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nimport time \nimport tqdm\nfrom PIL import Image\ntrain_on_gpu = True\nfrom torch.utils.data.sampler import SubsetRandomSampler\nfrom torch.optim.lr_scheduler import StepLR, ReduceLROnPlateau, CosineAnnealingLR\n\nimport cv2","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"9e2c32398c8467332c224d6400f54b2c0d16c013"},"cell_type":"code","source":"!pip install albumentations > /dev/null 2>&1","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"0a9f8bb797876c96206e00899b0a132a4c1b9980"},"cell_type":"code","source":"!pip install pretrainedmodels > /dev/null 2>&1","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"1a767991ddc3d4ad62737a2020f4d58958de93e2"},"cell_type":"code","source":"import albumentations\nfrom albumentations import torch as AT\nimport pretrainedmodels","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"56a074d815c5f776a858a75b1b64d6a6b5245970"},"cell_type":"markdown","source":"### Preparing data"},{"metadata":{"trusted":true,"_uuid":"a1d6ad51becc33c9026787b23608ea2e162411df"},"cell_type":"code","source":"labels = pd.read_csv('../input/train_labels.csv')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"print(f'{len(os.listdir(\"../input/train\"))} pictures in train.')\nprint(f'{len(os.listdir(\"../input/test\"))} pictures in test.')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"536444719dc66e498b4fba55200879453c6b9b29"},"cell_type":"markdown","source":"Let's have a look at some images."},{"metadata":{"trusted":true,"_uuid":"425563d5a3fb92f906aba54e247760a114fbf98a"},"cell_type":"code","source":"fig = plt.figure(figsize=(25, 4))\n# display 20 images\ntrain_imgs = os.listdir(\"../input/train\")\nfor idx, img in enumerate(np.random.choice(train_imgs, 20)):\n    ax = fig.add_subplot(2, 20//2, idx+1, xticks=[], yticks=[])\n    im = Image.open(\"../input/train/\" + img)\n    plt.imshow(im)\n    lab = labels.loc[labels['id'] == img.split('.')[0], 'label'].values[0]\n    ax.set_title(f'Label: {lab}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4ae6ad29e819b39b74820db88899369b19519e00"},"cell_type":"markdown","source":"Well, I can't find where the cancer is, though it seems cancer means some big cells. Let's hope that the model will be able to do it."},{"metadata":{"trusted":true,"_uuid":"677355bdb97c59c21059c1460495d48afde028c5"},"cell_type":"code","source":"labels.label.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"29987136ea6005aa58bb7db84de0860ec60b523e"},"cell_type":"markdown","source":"No disbalance here. Let's start building a model."},{"metadata":{"_uuid":"a9d858c899c7c9ec3251e11234521436ecb3809b"},"cell_type":"markdown","source":"### Model from scratch\n\nAt first I'll try to build a naive model from scratch. And I'll use 32x32 images, so that we cam directly classify the patches, which are interesting to us."},{"metadata":{"trusted":true,"_uuid":"9f90de5b26d9620a1fdead94bdb75d7c9e977973"},"cell_type":"code","source":"data_transforms = transforms.Compose([\n    transforms.CenterCrop(32),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomVerticalFlip(),\n    transforms.ToTensor(),\n    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n    ])\ndata_transforms_test = transforms.Compose([\n    transforms.CenterCrop(32),\n    transforms.ToTensor(),\n    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n    ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4f6ba7c971d21d450746c821b8b696a07c6471d4"},"cell_type":"code","source":"# indices for validation\ntr, val = train_test_split(labels.label, stratify=labels.label, test_size=0.1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e33f4d90d8e663637cf8227315e63fe3132b7cdd"},"cell_type":"markdown","source":"I haven't found a way to load data in Pytorch, when all train images are in one folder without subfolders. I have found an alternative - writing my own class for dataset creation."},{"metadata":{"trusted":true,"_uuid":"3f22915b81f3a929e391c8ec3a372555fdc5d92d"},"cell_type":"code","source":"# dictionary with labels and ids of train data\nimg_class_dict = {k:v for k, v in zip(labels.id, labels.label)}","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"a28ebb8b9c79e1b0440d5bae7369bd3bbbfb13f3"},"cell_type":"code","source":"class CancerDataset(Dataset):\n    def __init__(self, datafolder, datatype='train', transform = transforms.Compose([transforms.CenterCrop(32),transforms.ToTensor()]), labels_dict={}):\n        self.datafolder = datafolder\n        self.datatype = datatype\n        self.image_files_list = [s for s in os.listdir(datafolder)]\n        self.transform = transform\n        self.labels_dict = labels_dict\n        if self.datatype == 'train':\n            self.labels = [labels_dict[i.split('.')[0]] for i in self.image_files_list]\n        else:\n            self.labels = [0 for _ in range(len(self.image_files_list))]\n\n    def __len__(self):\n        return len(self.image_files_list)\n\n    def __getitem__(self, idx):\n        img_name = os.path.join(self.datafolder, self.image_files_list[idx])\n        image = Image.open(img_name)\n        image = self.transform(image)\n        img_name_short = self.image_files_list[idx].split('.')[0]\n\n        if self.datatype == 'train':\n            label = self.labels_dict[img_name_short]\n        else:\n            label = 0\n        return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9a941b55d5ecf7bdcea3203186faced865493d0d"},"cell_type":"code","source":"%%time\n# Load train data \ndataset = CancerDataset(datafolder='../input/train/', datatype='train', transform=data_transforms, labels_dict=img_class_dict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7166da04cbcd13f47041655f8b9186f5af6fb5e8"},"cell_type":"code","source":"# # get labels in correct order.\n# ims = pd.DataFrame(dataset.image_files_list, columns=['img'])\n# ims['img'] = ims['img'].apply(lambda x: x.split('.')[0])\n# joined = ims.join(labels)\n# lbs = joined.label.values\n# dataset.labels = lbs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9245b5dc20700b99bc6b42ca6f0b0f7276ace664"},"cell_type":"code","source":"test_set = CancerDataset(datafolder='../input/test/', datatype='test', transform=data_transforms_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"08818c98d09bbc6d1c303bb6938ffc82f8cd2b24"},"cell_type":"code","source":"train_sampler = SubsetRandomSampler(list(tr.index))\nvalid_sampler = SubsetRandomSampler(list(val.index))\nbatch_size = 512\nnum_workers = 0\n# prepare data loaders (combine dataset and sampler)\ntrain_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=train_sampler, num_workers=num_workers)\nvalid_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=valid_sampler, num_workers=num_workers)\ntest_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, num_workers=num_workers)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5706fa3a7ddef7b9b84caa19d0460ca5d88d2b55"},"cell_type":"markdown","source":"#### Model\n\nThis is a simple CNN structure."},{"metadata":{"trusted":true,"_uuid":"9061c7c82c7f5b10b3d82b76f794955f9865ce8e"},"cell_type":"code","source":"class Net(nn.Module):\n    def __init__(self):\n        super(Net, self).__init__()\n        self.conv1 = nn.Conv2d(3, 16, 3, padding=1)\n        self.conv2 = nn.Conv2d(16, 32, 3, padding=1)\n        self.conv3 = nn.Conv2d(32, 64, 3, padding=1)\n        self.pool = nn.MaxPool2d(2, 2)\n        self.fc1 = nn.Linear(64 * 4 * 4, 512)\n        self.fc2 = nn.Linear(512, 256)\n        self.fc3 = nn.Linear(256, 2)\n        self.dropout = nn.Dropout(0.2)\n        self.sig = nn.Sigmoid()\n\n    def forward(self, x):\n        x = self.pool(F.relu(self.conv1(x)))\n        x = self.pool(F.relu(self.conv2(x)))\n        x = self.pool(F.relu(self.conv3(x)))\n        x = x.view(-1, 64 * 4 * 4)\n        #x = self.dropout(x)\n        x = F.relu(self.fc1(x))\n        x = self.dropout(x)\n        x = F.relu(self.fc2(x))\n        x = self.sig(self.fc3(x))\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8e87480f786a01a1d5d112ec319e04ddf2091402"},"cell_type":"code","source":"# model_conv = Net()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9d62e5fd5e0f4e971926cbf5fd67df02397a1721"},"cell_type":"code","source":"# model_conv.cuda()\ncriterion = nn.BCELoss()\n\n# optimizer = optim.SGD(model_conv.parameters(), lr=0.001, momentum=0.9)\n# exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"21c7aad6143cf1346318beb400056a56fccce517"},"cell_type":"markdown","source":"Training."},{"metadata":{"trusted":true,"_uuid":"7490345f34f7a59f38844c953562d8978f114c24","scrolled":true},"cell_type":"code","source":"# valid_loss_min = np.Inf\n# patience = 7\n# # current number of epochs, where validation loss didn't increase\n# p = 0\n# # whether training should be stopped\n# stop = False\n\n# # number of epochs to train the model\n# n_epochs = 10\n# for epoch in range(1, n_epochs+1):\n#     print(time.ctime(), 'Epoch:', epoch)\n\n#     train_loss = []\n#     exp_lr_scheduler.step()\n#     train_auc = []\n\n#     for batch_i, (data, target) in enumerate(train_loader):\n\n#         data, target = data.cuda(), target.cuda()\n\n#         optimizer.zero_grad()\n#         output = model_conv(data)\n#         loss = criterion(output[:,1], target.float())\n#         train_loss.append(loss.item())\n        \n#         a = target.data.cpu().numpy()\n#         b = output[:,-1].detach().cpu().numpy()\n#         train_auc.append(roc_auc_score(a, b))\n\n#         loss.backward()\n#         optimizer.step()\n    \n#     model_conv.eval()\n#     val_loss = []\n#     val_auc = []\n#     for batch_i, (data, target) in enumerate(valid_loader):\n#         data, target = data.cuda(), target.cuda()\n#         output = model_conv(data)\n\n#         loss = criterion(output[:,1], target.float())\n\n#         val_loss.append(loss.item()) \n#         a = target.data.cpu().numpy()\n#         b = output[:,-1].detach().cpu().numpy()\n#         val_auc.append(roc_auc_score(a, b))\n\n#     print(f'Epoch {epoch}, train loss: {np.mean(train_loss):.4f}, valid loss: {np.mean(val_loss):.4f}, train auc: {np.mean(train_auc):.4f}, valid acc: {np.mean(val_auc):.4f}')\n    \n#     valid_loss = np.mean(val_loss)\n#     if valid_loss <= valid_loss_min:\n#         print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(\n#         valid_loss_min,\n#         valid_loss))\n#         torch.save(model_conv.state_dict(), 'model.pt')\n#         valid_loss_min = valid_loss\n#         p = 0\n\n#     # check if validation loss didn't improve\n#     if valid_loss > valid_loss_min:\n#         p += 1\n#         print(f'{p} epochs of increasing val loss')\n#         if p > patience:\n#             print('Stopping training')\n#             stop = True\n#             break        \n            \n#     if stop:\n#         break","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bcb8e6ec808c020a2fbe870327b827d8962919d4"},"cell_type":"markdown","source":"### Using pretrained model.\n\nAnd now let's try something better - pre-trained net. I'll be using Resnet. \n\nIt is very important to remember, that Resnet requires images with 224x224 height and width. So I'll pad the imaged and create new loaders."},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"4eb9607f9ff91ccb32dba088adce2c6dae9e9d2f"},"cell_type":"code","source":"class CancerDataset(Dataset):\n    def __init__(self, datafolder, datatype='train', transform = transforms.Compose([transforms.CenterCrop(32),transforms.ToTensor()]), labels_dict={}):\n        self.datafolder = datafolder\n        self.datatype = datatype\n        self.image_files_list = [s for s in os.listdir(datafolder)]\n        self.transform = transform\n        self.labels_dict = labels_dict\n        if self.datatype == 'train':\n            self.labels = [labels_dict[i.split('.')[0]] for i in self.image_files_list]\n        else:\n            self.labels = [0 for _ in range(len(self.image_files_list))]\n\n    def __len__(self):\n        return len(self.image_files_list)\n\n    def __getitem__(self, idx):\n        img_name = os.path.join(self.datafolder, self.image_files_list[idx])\n        img = cv2.imread(img_name)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        image = self.transform(image=img)\n        image = image['image']\n\n        img_name_short = self.image_files_list[idx].split('.')[0]\n\n        if self.datatype == 'train':\n            label = self.labels_dict[img_name_short]\n        else:\n            label = 0\n        return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bfb235294fa68e9aee0d875d84455ad8bbf4f011"},"cell_type":"code","source":"data_transforms = albumentations.Compose([\n    albumentations.Resize(224, 224),\n    albumentations.HorizontalFlip(),\n    albumentations.RandomBrightness(),\n    albumentations.ShiftScaleRotate(rotate_limit=15, scale_limit=0.10),\n    albumentations.JpegCompression(80),\n    albumentations.HueSaturationValue(),\n    albumentations.Normalize(),\n    AT.ToTensor()\n    ])\ndata_transforms_test = albumentations.Compose([\n    albumentations.Resize(224, 224),\n    albumentations.Normalize(),\n    AT.ToTensor()\n    ])\n\ndataset = CancerDataset(datafolder='../input/train/', datatype='train', transform=data_transforms, labels_dict=img_class_dict)\ntest_set = CancerDataset(datafolder='../input/test/', datatype='test', transform=data_transforms_test)\ntrain_sampler = SubsetRandomSampler(list(tr.index))\nvalid_sampler = SubsetRandomSampler(list(val.index))\nbatch_size = 32\nnum_workers = 0\n# prepare data loaders (combine dataset and sampler)\ntrain_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=train_sampler, num_workers=num_workers)\nvalid_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=valid_sampler, num_workers=num_workers)\ntest_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, num_workers=num_workers)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce72638bc193910919f72b4c7727f72d0f6b27ab","scrolled":true},"cell_type":"code","source":"model_conv = torchvision.models.resnet50(pretrained=True)\nfor i, param in model_conv.named_parameters():\n    param.requires_grad = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e5947b87b524f6d88476ae45eb47ca7f879b2517"},"cell_type":"code","source":"num_ftrs = model_conv.fc.in_features\nmodel_conv.fc = nn.Linear(2048, 2)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6f8355c31a97adc453a1a0c6f10268d42048b26a"},"cell_type":"markdown","source":"#### Training"},{"metadata":{"trusted":true,"_uuid":"f313cbc64a652ac9bd7d4acd479ac642ce7262cb"},"cell_type":"code","source":"model_conv.cuda()\ncriterion = nn.BCEWithLogitsLoss()\n\noptimizer = optim.SGD(model_conv.fc.parameters(), lr=0.005, momentum=0.99)\n#scheduler = CyclicLR(optimizer, base_lr=lr, max_lr=0.01, step_size=5, mode='triangular2')\nscheduler = lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.1)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"scrolled":true,"_uuid":"82bbb418089188e2c7cc32ecd32aa799147ab18c"},"cell_type":"code","source":"valid_loss_min = np.Inf\npatience = 5\n# current number of epochs, where validation loss didn't increase\np = 0\n# whether training should be stopped\nstop = False\n\n# number of epochs to train the model\nn_epochs = 7\nfor epoch in range(1, n_epochs+1):\n    print(time.ctime(), 'Epoch:', epoch)\n\n    train_loss = []\n    # scheduler.step(epoch)\n    train_auc = []\n\n    for batch_i, (data, target) in enumerate(train_loader):\n\n        data, target = data.cuda(), target.cuda()\n\n        optimizer.zero_grad()\n        output = model_conv(data)\n        loss = criterion(output[:,1], target.float())\n        train_loss.append(loss.item())\n        \n        a = target.data.cpu().numpy()\n        b = output[:,-1].detach().cpu().numpy()\n        # train_auc.append(roc_auc_score(a, b))\n\n        loss.backward()\n        optimizer.step()\n    \n    model_conv.eval()\n    val_loss = []\n    val_auc = []\n    for batch_i, (data, target) in enumerate(valid_loader):\n        data, target = data.cuda(), target.cuda()\n        output = model_conv(data)\n\n        loss = criterion(output[:,1], target.float())\n\n        val_loss.append(loss.item()) \n        a = target.data.cpu().numpy()\n        b = output[:,-1].detach().cpu().numpy()\n        # val_auc.append(roc_auc_score(a, b))\n\n    # print(f'Epoch {epoch}, train loss: {np.mean(train_loss):.4f}, valid loss: {np.mean(val_loss):.4f}, train auc: {np.mean(train_auc):.4f}, valid auc: {np.mean(val_auc):.4f}')\n    print(f'Epoch {epoch}, train loss: {np.mean(train_loss):.4f}, valid loss: {np.mean(val_loss):.4f}.')\n    \n    valid_loss = np.mean(val_loss)\n    scheduler.step(valid_loss)\n    if valid_loss <= valid_loss_min:\n        print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(\n        valid_loss_min,\n        valid_loss))\n        torch.save(model_conv.state_dict(), 'model.pt')\n        valid_loss_min = valid_loss\n        p = 0\n\n    # check if validation loss didn't improve\n    if valid_loss > valid_loss_min:\n        p += 1\n        print(f'{p} epochs of increasing val loss')\n        if p > patience:\n            print('Stopping training')\n            stop = True\n            break        \n            \n    if stop:\n        break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"f34333b58c66bde6d3d7bb5c7d64dfc079c130eb"},"cell_type":"code","source":"model_conv.eval()\npreds = []\nfor batch_i, (data, target) in enumerate(test_loader):\n    data, target = data.cuda(), target.cuda()\n    output = model_conv(data)\n\n    pr = output[:,1].detach().cpu().numpy()\n    for i in pr:\n        preds.append(i)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"99bbfe6dd0539381444c1aea81243d9f26c6127f"},"cell_type":"code","source":"test_preds = pd.DataFrame({'imgs': test_set.image_files_list, 'preds': preds})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab18f6e469f80c91d4a0eca5fc73fdbfe5aeae65"},"cell_type":"code","source":"test_preds['imgs'] = test_preds['imgs'].apply(lambda x: x.split('.')[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9c8112b3732c78574a6f1b2927f9e569efd8277f"},"cell_type":"code","source":"sub = pd.read_csv('../input/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1c1390f71bc95b4464b79edf6f21eb8ae07c2d47"},"cell_type":"code","source":"sub = pd.merge(sub, test_preds, left_on='id', right_on='imgs')\nsub = sub[['id', 'preds']]\nsub.columns = ['id', 'label']\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"99dd127dcabca938db34e9374ea8640abc8ea65c"},"cell_type":"code","source":"sub.to_csv('sub.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}