{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\n\nimport torch\nimport torch.nn as nn\nfrom torch.optim import lr_scheduler\nimport torch.utils.model_zoo as model_zoo\nfrom sklearn.metrics import roc_curve, auc\nimport torchvision\nfrom sklearn.model_selection import StratifiedShuffleSplit\n\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nfrom IPython.display import clear_output\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport random\nimport copy\nimport os\n\n# Check if gpu support is available\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\n%matplotlib inline\n\ncsv_submission_ex_file = '../input/sample_submission.csv'\ncsv_file = '../input/train_labels.csv'\ntrain_dir = '../input/train/'\ntest_dir = '../input/test/'\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5ad9da6e06a628120ac7f36bfaea9b9173565e4c","trusted":true},"cell_type":"code","source":"csv_pd = pd.read_csv(csv_file)   \ncsv_pd.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class NormFinderDataset(torch.utils.data.dataset.Dataset):\n    def __init__(self, csv_file, img_dir, transform=None):\n        self.csv_file = pd.read_csv(csv_file)\n        self.img_dir = img_dir\n\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.csv_file)\n\n    def __getitem__(self, idx):\n        img_name = os.path.join(self.img_dir, self.csv_file.iloc[idx, 0])\n        img_name = img_name + '.tif'\n\n        sample = Image.open(img_name)\n\n        if self.transform is not None:\n            sample = self.transform(sample)\n\n        return {'sample': sample}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"96498519b3df13a54d1169fe1b602fe5b97dae9e"},"cell_type":"code","source":"normfinder_transformations = torchvision.transforms.Compose([\n    torchvision.transforms.Resize(224),\n    torchvision.transforms.ToTensor(),\n])\n\nnormfinder_test_dataset = NormFinderDataset(csv_submission_ex_file, test_dir, normfinder_transformations)\nnormfinder_test_dataloader = torch.utils.data.DataLoader(normfinder_test_dataset, batch_size=1024, shuffle=False)\n\nnormfinder_train_dataset = NormFinderDataset(csv_file, train_dir, normfinder_transformations)\nnormfinder_train_dataloader = torch.utils.data.DataLoader(normfinder_train_dataset, batch_size=1024, shuffle=False)\n\npop_mean = []\npop_std0 = []\nfor data in tqdm(normfinder_test_dataloader, 0):\n    # shape (batch_size, 3, height, width)\n    numpy_image = data['sample'].numpy()\n\n    # shape (3,)\n    batch_mean = np.mean(numpy_image, axis=(0, 2, 3))\n    batch_std0 = np.std(numpy_image, axis=(0, 2, 3))\n\n    pop_mean.append(batch_mean)\n    pop_std0.append(batch_std0)\n\nfor data in tqdm(normfinder_train_dataloader, 0):\n    # shape (batch_size, 3, height, width)\n    numpy_image = data['sample'].numpy()\n\n    # shape (3,)\n    batch_mean = np.mean(numpy_image, axis=(0, 2, 3))\n    batch_std0 = np.std(numpy_image, axis=(0, 2, 3))\n\n    pop_mean.append(batch_mean)\n    pop_std0.append(batch_std0)\n\n# shape (num_iterations, 3) -> (mean across 0th axis) -> shape (3,)\npop_mean = np.array(pop_mean).mean(axis=0)\npop_std0 = np.array(pop_std0).mean(axis=0)\n\nprint('mean: {}'.format(pop_mean))\nprint('std0: {}'.format(pop_std0))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a6572a1dbad2ccc04ad3021a9e813c6767c89a98","trusted":true},"cell_type":"code","source":"# Let's define some transformations for the input data, crop to 64px and then resize to 224 to fit resnet input size\ndata_transformations = torchvision.transforms.Compose([\n    torchvision.transforms.RandomHorizontalFlip(),\n    torchvision.transforms.RandomVerticalFlip(),\n    torchvision.transforms.Resize(224),\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize(mean=[0.70017236, 0.5436771, 0.6961061], std=[0.22246036, 0.26757348, 0.19798167]),\n])\n\n# Let's define some transformations for the test data, crop to 32px and then resize to 224 to fit resnet input size\ntest_transformations = torchvision.transforms.Compose([\n    torchvision.transforms.Resize(224),\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize(mean=[0.70017236, 0.5436771, 0.6961061], std=[0.22246036, 0.26757348, 0.19798167]),\n])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e53482d1475882c441cb4339fc4f0cb3a0b326d6","trusted":true},"cell_type":"code","source":"# let's see some negative images\nfig=plt.figure(figsize=(16, 16))\n\nsamples_per_type = 5\nnegative_found = 0\n\nwhile negative_found < samples_per_type:\n    idx = random.randint(0, len(csv_pd))\n    # Negatives\n    if csv_pd.iloc[idx, 1] == 0:\n        negative_found = negative_found + 1\n        image = Image.open(train_dir + csv_pd.iloc[idx, 0] + '.tif')\n        fig.add_subplot(2, 5, negative_found)\n        plt.title('Negative Label')\n        plt.imshow(image)\n        \n        image = data_transformations(image)\n        back_transform = torchvision.transforms.ToPILImage()\n        image = back_transform(image)\n        fig.add_subplot(2, 5, negative_found + samples_per_type)\n        plt.title('Transformed')\n        plt.imshow(image)\n\nplt.subplots_adjust(bottom=0.3, top=0.7, hspace=0)   \nplt.show()    \n\n\n# let's see some positive images\nfig2=plt.figure(figsize=(16, 16))\n\npositive_found = 0\n        \nwhile positive_found < samples_per_type:\n    idx = random.randint(0, len(csv_pd))\n    # Positives\n    if csv_pd.iloc[idx, 1] == 1:\n        positive_found = positive_found + 1\n        image = Image.open(train_dir + csv_pd.iloc[idx, 0] + '.tif')\n        fig2.add_subplot(2, 5, positive_found)\n        plt.title('Positive Label')\n        plt.imshow(image)\n                \n        image = data_transformations(image)\n        back_transform = torchvision.transforms.ToPILImage()\n        image = back_transform(image)\n        fig2.add_subplot(2, 5, positive_found + samples_per_type)\n        plt.title('Transformed')\n        plt.imshow(image)\n\nplt.subplots_adjust(bottom=0.3, top=0.7, hspace=0)   \nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"047467383dca63ff5b0e3bc5c01060b78c90571a","trusted":true},"cell_type":"code","source":"# let's see some images after transform.\nfig=plt.figure(figsize=(16, 16))\n\nsamples = 5\n\nfor i in range(samples):\n    random_file = random.choice(os.listdir(test_dir))\n    image = Image.open(test_dir + random_file)\n    fig.add_subplot(2, 5, i + 1)\n    plt.title('Original')\n    plt.imshow(image)\n    \n    image = test_transformations(image)\n    back_transform = torchvision.transforms.ToPILImage()\n    image = back_transform(image)\n    fig.add_subplot(2, 5, i + 1 + samples)\n    plt.title('Transformed')\n    plt.imshow(image)\n\nplt.subplots_adjust(bottom=0.3, top=0.7, hspace=0)   \nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"# Dataset Class for reading the data ids and labels from the CSV file.\n# From the CSV file only the specified indexes are saved in this dataset(specified indexes after the train/val split)\n\nclass HPLCDDataset(torch.utils.data.dataset.Dataset):\n    \"\"\"HPLCDDataset dataset.\"\"\"\n\n    def __init__(self, csv_file, img_dir, idxs, transform=None):\n        \"\"\"\n        Args:\n            csv_file (string): Path to the csv file with annotations.\n            img_dir (string): Directory with all the images.\n            idxs (list): List with indexes.\n            transform (callable, optional): Optional transform to be applied\n                on a sample\n        \"\"\"\n        self.csv_file = pd.read_csv(csv_file)\n        self.img_dir = img_dir\n        self.idxs = idxs\n\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.idxs)\n\n    def __getitem__(self, idx):\n        img_name = os.path.join(self.img_dir, self.csv_file.iloc[self.idxs[idx], 0])\n        img_name = img_name + '.tif'\n\n        sample = Image.open(img_name)\n        target = self.csv_file.iloc[self.idxs[idx], 1]\n\n        if self.transform is not None:\n            sample = self.transform(sample)\n\n        return sample, target\n    \n    def __getlabel__(self, idx):\n        return self.csv_file.iloc[self.idxs[idx], 1]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"19537aa5af8d1082f1a763172a32a66b7631e573","trusted":true},"cell_type":"code","source":"# Use a batch size of 256 images\nbatch_size = 256\n\n# Split data 10% validation 90% train with balanced data between the two datasets\nsss = StratifiedShuffleSplit(n_splits=1, test_size=0.1, random_state=7)\n\n# Indexes in the CSV file for both train/val datasets\ntrain_index, test_index = next(sss.split(csv_pd[\"id\"], csv_pd[\"label\"]))\n\ntrain_dataset = HPLCDDataset(csv_file, train_dir, train_index, data_transformations)\ntest_dataset = HPLCDDataset(csv_file, train_dir, test_index, test_transformations)\n\n# Create loders for training/validation sets\ntrain_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, drop_last=True)\ntest_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, drop_last=True)\n\ndataloaders = {'train': train_loader, 'test': test_loader}","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"936f43425851a917361dc01db4a79ddb9971701c","trusted":true},"cell_type":"code","source":"# Lets see how the data is balanced in the two Datasets\ntrain_positive_labels = 0\ntrain_negative_labels = 0\nfor index in range(train_dataset.__len__()):\n    label = train_dataset.__getlabel__(index)\n    if label == 0:\n        train_negative_labels = train_negative_labels + 1\n    else:\n        train_positive_labels = train_positive_labels + 1\n\n# Lets see how the data is balanced in the two Datasets\ntest_positive_labels = 0\ntest_negative_labels = 0\nfor index in range(test_dataset.__len__()):\n    label = test_dataset.__getlabel__(index)\n    if label == 0:\n        test_negative_labels = test_negative_labels + 1\n    else:\n        test_positive_labels = test_positive_labels + 1\n\n# Pie chart, where the slices will be ordered and plotted counter-clockwise:\nlabels = 'Negative', 'Positive'\n\nfig, axs = plt.subplots(1, 2)\nplt.subplots_adjust(right=1.5)\n\ndef make_autopct(values):\n    def my_autopct(pct):\n        total = sum(values)\n        val = int(round(pct*total/100.0))\n        return '{p:.3f}%\\n({v:d})'.format(p=pct,v=val)\n    return my_autopct\n\naxs[0].pie([train_negative_labels, train_positive_labels], labels=labels, autopct=make_autopct([train_negative_labels, train_positive_labels]),\n        shadow=True, startangle=90)\naxs[0].axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle.\naxs[0].set_title('Train Dataset\\n Samples:{}'.format(train_negative_labels + train_positive_labels))\n\naxs[1].pie([test_negative_labels, test_positive_labels], labels=labels, autopct=make_autopct([test_negative_labels, test_positive_labels]),\n        shadow=True, startangle=90)\naxs[1].axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle.\naxs[1].set_title('Test Dataset\\n Samples:{}'.format(test_negative_labels + test_positive_labels))\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"10cb116699fe6534c4722e376ba4d6c02afadd7a","trusted":true},"cell_type":"code","source":"def train(model, criterion, optimizer, scheduler, dataloaders, logger):\n    \n    dataset_sizes = {'train': len(dataloaders['train']),\n                     'test': len(dataloaders['test'])}\n    \n    train_loss = 0.0\n    train_acc = 0.0\n    test_loss = 0.0\n    test_acc = 0.0\n\n    # Each epoch has a training and validation phase\n    for phase in ['train', 'test']:\n        if phase == 'train':\n            scheduler.step()\n            model.train()  # Set model to training mode\n        else:\n            model.eval()   # Set model to evaluate mode\n\n        running_loss = 0.0\n        running_corrects = 0\n            \n        test_output_results = []\n        test_output_expected = []\n\n        pbar = tqdm(enumerate(dataloaders[phase]))\n        # Iterate over data.\n        for i, (inputs, labels) in pbar:\n            inputs = inputs.to(device)\n            labels = labels.to(device)\n\n            # zero the parameter gradients\n            optimizer.zero_grad()\n\n            # forward\n            # track history if only in train\n            with torch.set_grad_enabled(phase == 'train'):\n                outputs = model(inputs)\n                loss = criterion(outputs, labels.float().view(dataloaders[phase].batch_size, 1))\n                                \n                # backward + optimize only if in training phase\n                if phase == 'train':\n                    loss.backward()\n                    optimizer.step()\n                \n            # statistics\n            running_loss += loss.item()\n            \n            preds = torch.where(outputs > 0.5, torch.Tensor([1]).to(device), torch.Tensor([0]).to(device))\n            running_corrects += torch.sum(preds == labels.float().view(dataloaders[phase].batch_size, 1)).item()\n                \n            if phase == 'test':\n                test_output_results = np.concatenate([test_output_results, outputs.view(-1).cpu().numpy()])\n                test_output_expected = np.concatenate([test_output_expected, labels.view(-1).cpu().numpy()])\n\n            pbar.set_description('[{} {}/{}] Loss: {:.4f}, Acc: {:.4f}'.format(phase, i, dataset_sizes[phase],\n                running_loss / (i+1), running_corrects / ((i+1) * dataloaders[phase].batch_size)))\n\n        epoch_loss = running_loss / dataset_sizes[phase]\n        epoch_acc = running_corrects / dataset_sizes[phase] / dataloaders[phase].batch_size\n\n        if phase == 'train':\n            train_loss = epoch_loss\n            train_acc = epoch_acc\n\n        if phase == 'test':\n            test_loss = epoch_loss\n            test_acc = epoch_acc\n            logger.append([train_loss, train_acc, test_loss, test_acc])\n            \n        test_output = {'expected': test_output_expected, 'results': test_output_results}\n\n    return model, optimizer, scheduler, test_output, logger","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"070c619cd01d772c8f7dfb9174f1fe7ca17dc9bc","trusted":true},"cell_type":"code","source":"def plot_results(logger):\n    plt.plot(logger)\n    plt.xlabel('Epoch')\n    plt.grid(True)\n    plt.legend(['train loss', 'train accuracy', 'validation loss', 'validation accuracy'])\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"36803f423dcd6953338ea10bd37bee9437cd952b"},"cell_type":"code","source":"def plot_roc_auc(test_output):        \n    fpr = dict()\n    tpr = dict()\n    roc_auc = dict()   \n    fpr, tpr, _ = roc_curve(test_output['expected'], test_output['results'])\n    roc_auc = auc(fpr, tpr)\n   \n    plt.subplot(121, aspect='equal')\n    plt.plot(fpr, tpr, color='darkorange', lw=2, label='ROC curve (area = %0.5f)' % roc_auc)\n    plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')\n    plt.xlim([0.0, 1.0])\n    plt.ylim([0.0, 1.0])\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.title('Receiver operating characteristic')\n    plt.legend(loc=\"lower right\")\n    \n    negative_results = []\n    positive_results = []\n    for idx, (res) in enumerate(test_output['expected']):\n        if res == 1:\n            positive_results.append(test_output['results'][idx])\n        elif res == 0:\n            negative_results.append(test_output['results'][idx])\n        else:\n            print('ERROR HIST!!!')\n    \n    bins = np.linspace(min(test_output['results']), max(test_output['results']), 100)\n    \n    plt.subplot(122)\n    plt.hist(positive_results, bins, alpha=0.5, label='Positive', histtype='step')\n    plt.hist(negative_results, bins, alpha=0.5, label='Negative', histtype='step')\n    plt.yscale('log')\n    plt.legend(loc='upper center')\n    plt.grid(True)\n    \n    plt.subplots_adjust(bottom=0.0, right=2.2, top=1)    \n    plt.show() ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce5a17a42a282fc58e28e2bc20ddee7722eb24a6"},"cell_type":"code","source":"model = nn.Sequential(torchvision.models.resnet18(pretrained=False, num_classes=1), nn.Sigmoid())\nmodel.to(device)\n\noptimizer = torch.optim.Adam(model.parameters(), lr=0.001, weight_decay=1e-3)\ncriterion = nn.BCELoss()\n\n# Decay LR by a factor of 0.1 every x epochs\nexp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.1)\n\n# define the global logger\nlogger = [[0.45, 0.55, 0.45, 0.55]]\n\nprevious_epochs = 0\n\nload_checkpoint = False\nif load_checkpoint == True:\n    checkpoint = torch.load(\"checkpoint_ep1.plt\")\n    model.load_state_dict(checkpoint['model_state_dict'])\n    optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n    previous_epochs = checkpoint['epoch']\n    dataloaders = checkpoint['dataloaders']\n    logger = checkpoint['logger']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e09db5ec7ae113e78c00b6496264e6dd9f36d00c"},"cell_type":"code","source":"epochs = 8\n\nfor i in range(epochs):\n    print('Epoch: {}/{}'.format(i + previous_epochs + 1, epochs + previous_epochs))\n    model, optimizer, exp_lr_scheduler, test_output, logger = train(model, criterion, optimizer, exp_lr_scheduler, dataloaders, logger)\n\n    #SAVE\n    #torch.save({\n    #        'epoch': i + 1 + previous_epochs,\n    #        'model_state_dict': model.state_dict(),\n    #        'optimizer_state_dict': optimizer.state_dict(),\n    #        'logger': logger,\n    #        'dataloaders': dataloaders\n    #        }, \"checkpoint_ep{}.plt\".format(i + 1 + previous_epochs))\n    \n    clear_output()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"777a81518827446b8cf677e5f8e71bd4db7d336d"},"cell_type":"code","source":"plot_roc_auc(test_output) \nplot_results(logger)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"37c8421494ccc2d32a58da54af1b483e2f7ba112"},"cell_type":"markdown","source":"def test_alone(model, criterion, dataloader):\n    running_loss = 0.0\n    running_corrects = 0\n            \n    test_output_results = []\n    test_output_expected = []\n    \n    model.eval()   # Set model to evaluate mode\n    \n    pbar = tqdm(enumerate(dataloader))\n        # Iterate over data.\n    for i, (inputs, labels) in pbar:\n        inputs = inputs.to(device)\n        labels = labels.to(device)\n        \n        with torch.set_grad_enabled(False):\n            outputs = model(inputs)\n            loss = criterion(outputs, labels.float().view(dataloader.batch_size, 1))\n        \n        # statistics\n        running_loss += loss.item()\n            \n        preds = torch.where(outputs > 0.5, torch.Tensor([1]).to(device), torch.Tensor([0]).to(device))\n        running_corrects += torch.sum(preds == labels.float().view(dataloader.batch_size, 1)).item()\n                \n        test_output_results = np.concatenate([test_output_results, outputs.view(-1).cpu().numpy()])\n        test_output_expected = np.concatenate([test_output_expected, labels.view(-1).cpu().numpy()])\n\n        pbar.set_description('[{} {}/{}] Loss: {:.4f}, Acc: {:.4f}'.format('test', i, len(dataloader),\n            running_loss / (i+1), running_corrects / ((i+1) * dataloader.batch_size)))\n            \n    test_output = {'expected': test_output_expected, 'results': test_output_results}\n\n    return test_output\n\ntest_alone_needed = False;\nif test_alone_needed == True:\n    test_output = test_alone(model, criterion, dataloaders['test'])\n    plot_roc_auc(test_output)"},{"metadata":{"_uuid":"80f37ac466c502a7d4fb9b0f345c721853449bb8","trusted":true},"cell_type":"markdown","source":"def predict(model, img_folder_path, transform=None):\n    targets = []\n    predictions = []\n    \n    model.eval()   # Set model to evaluate mode\n\n    for filename in tqdm(os.listdir(img_folder_path)):\n        image = Image.open(img_folder_path + filename)\n\n        # Preprocess the image\n        image_tensor = transform(image)\n\n        # Add an extra batch dimension since pytorch treats all images as batches\n        image_tensor = image_tensor.unsqueeze_(0)\n\n        input = torch.autograd.Variable(image_tensor.to(device))\n\n        # Predict the class of the image\n        output = model(input)\n        \n        targets.append(filename.replace('.tif', ''))\n        predictions.append(int(torch.where(output > 0.5, torch.Tensor([1]).to(device), torch.Tensor([0]).to(device)).item()))\n        \n    my_submission = pd.DataFrame({'id': targets, 'label': predictions})\n    my_submission.to_csv('hplcd_submission.csv', index=False)"},{"metadata":{"trusted":true,"_uuid":"464b926ba61bad17a54f44d9db1febc0549b9218"},"cell_type":"markdown","source":"predict(model, test_dir, test_transformations)"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}