{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"scrolled":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d18642c1bcaec05236c28de039e7ce2f6512f6d3"},"cell_type":"markdown","source":"so we have 4 files to work with.\nand we will load in some librarys, there will probably mix of pytourch, sklearn."},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true,"_uuid":"834a681c1d797530f254b850145abb4378111a99"},"cell_type":"code","source":"import csv\nfrom PIL import Image\nimport numpy as np\nimport pandas as pd\nimport os\nimport cv2\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nfrom sklearn.model_selection import train_test_split\nimport torchvision.transforms as transforms\n\nimport torch\nfrom torch.utils.data import TensorDataset, DataLoader,Dataset\nfrom torch.utils.data.sampler import SubsetRandomSampler\n\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.autograd import Variable\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data import sampler\nimport torchvision\nimport torchvision.datasets as dset\nimport torchvision.transforms as T\nimport timeit\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom torch.autograd import Variable\nfrom torch.autograd import Function\nfrom torch.nn.modules.module import Module\nfrom torch.nn.parameter import Parameter\nfrom torch.nn.functional import conv2d\nimport torch.nn.functional as F\nimport numpy as np\n\n\nclass Kerv2d(nn.Conv2d):\n    '''\n    kervolution with following options:\n    kernel_type: [linear, polynomial, gaussian, etc.]\n    default is convolution:\n             kernel_type --> linear,\n    balance, power, gamma is valid only when the kernel_type is specified\n    if learnable_kernel = True,  they just be the initial value of learable parameters\n    if learnable_kernel = False, they are the value of kernel_type's parameter\n    the parameter [power] cannot be learned due to integer limitation\n    '''\n    def __init__(self, in_channels, out_channels, kernel_size, \n            stride=1, padding=0, dilation=1, groups=1, bias=True,\n            kernel_type='linear', learnable_kernel=False, kernel_regularizer=False,\n            balance=1, power=3, gamma=1):\n\n        super(Kerv2d, self).__init__(in_channels, out_channels, kernel_size, stride, padding, dilation, groups, bias)\n        self.kernel_type = kernel_type\n        self.learnable_kernel, self.kernel_regularizer = learnable_kernel, kernel_regularizer\n        self.balance, self.power, self.gamma = balance, power, gamma\n\n        # parameter for kernel type\n        if learnable_kernel == True:\n            self.balance = nn.Parameter(torch.cuda.FloatTensor([balance] * out_channels), requires_grad=True).view(-1, 1)\n            self.gamma   = nn.Parameter(torch.cuda.FloatTensor([gamma]   * out_channels), requires_grad=True).view(-1, 1)\n\n    def forward(self, input):\n\n        minibatch, in_channels, input_width, input_hight = input.size()\n        assert(in_channels == self.in_channels)\n        input_unfold = F.unfold(input, kernel_size=self.kernel_size, dilation=self.dilation, padding=self.padding, stride=self.stride)\n        input_unfold = input_unfold.view(minibatch, 1, self.kernel_size[0]*self.kernel_size[1]*self.in_channels, -1)\n        weight_flat  = self.weight.view(self.out_channels, -1, 1)\n        output_width = (input_width - self.kernel_size[0] + 2 * self.padding[0]) // self.stride[0] + 1\n        output_hight = (input_hight - self.kernel_size[1] + 2 * self.padding[1]) // self.stride[1] + 1\n\n        if self.kernel_type == 'linear':\n            output = (input_unfold * weight_flat).sum(dim=2)\n\n        elif self.kernel_type == 'manhattan':\n            output = -((input_unfold - weight_flat).abs().sum(dim=2))\n\n        elif self.kernel_type == 'euclidean':\n            output = -(((input_unfold - weight_flat)**2).sum(dim=2))\n\n        elif self.kernel_type == 'polynomial':\n            output = ((input_unfold * weight_flat).sum(dim=2) + self.balance)**self.power\n\n        elif self.kernel_type == 'gaussian':\n            output = (-self.gamma*((input_unfold - weight_flat)**2).sum(dim=2)).exp() + 0\n\n        else:\n            raise NotImplementedError(self.kernel_type+' kervolution not implemented')\n\n        if self.bias is not None:\n            output += self.bias.view(self.out_channels, -1)\n\n        return output.view(minibatch, self.out_channels, output_width, output_hight)\n\n\nclass Kerv1d(nn.Conv1d):\n    r\"\"\"Applies a 1D kervolution over an input signal composed of several input\n        planes.\n        Args:\n            in_channels (int): Number of channels in the input image\n            out_channels (int): Number of channels produced by the convolution\n            kernel_size (int or tuple): Size of the convolving kernel\n            stride (int or tuple, optional): Stride of the convolution. Default: 1\n            padding (int or tuple, optional): Zero-padding added to both sides of\n                the input. Default: 0\n            padding_mode (string, optional). Accepted values `zeros` and `circular` Default: `zeros`\n            dilation (int or tuple, optional): Spacing between kernel\n                elements. Default: 1\n            groups (int, optional): Number of blocked connections from input\n                channels to output channels. Default: 1\n            bias (bool, optional): If ``True``, adds a learnable bias to the output. Default: ``True``\n            kernel_type (str), Default: 'linear'\n            learnable_kernel (bool): Learnable kernel parameters.  Default: False \n            balance=1, power=3, gamma=1\n        Shape:\n            - Input: :math:`(N, C_{in}, L_{in})`\n            - Output: :math:`(N, C_{out}, L_{out})` where\n            .. math::\n                L_{out} = \\left\\lfloor\\frac{L_{in} + 2 \\times \\text{padding} - \\text{dilation}\n                            \\times (\\text{kernel\\_size} - 1) - 1}{\\text{stride}} + 1\\right\\rfloor\n        Examples::\n            >>> m = Kerv1d(16, 33, 3, kernel_type='polynomial', learnable_kernel=True)\n            >>> input = torch.randn(20, 16, 50)\n            >>> output = m(input)\n        .. _kervolution:\n            https://arxiv.org/pdf/1904.03955.pdf\n        \"\"\"\n\n    def __init__(self, in_channels, out_channels, kernel_size, \n            stride=1, padding=0, dilation=1, groups=1, bias=True, padding_mode='zeros',\n            kernel_type='linear', learnable_kernel=False, balance=1, power=3, gamma=1):\n\n        super(Kerv1d, self).__init__(in_channels, out_channels, kernel_size, stride, padding, dilation, groups, bias, padding_mode)\n        self.kernel_type, self.learnable_kernel = kernel_type, learnable_kernel\n        self.balance, self.power, self.gamma = balance, power, gamma\n        self.unfold = nn.Unfold((kernel_size,1), (dilation,1), (padding, 0), (stride,1))\n\n        # parameter for kernels\n        # if learnable_kernel == True:\n        # self.balance = nn.Parameter(torch.FloatTensor([balance] * out_channels)).view(-1, 1)\n        # self.gamma   = nn.Parameter(torch.FloatTensor([gamma]   * out_channels)).view(-1, 1)\n\n\n    def forward(self, input):\n        input = self.unfold(input.unsqueeze(-1)).unsqueeze(1)\n        weight  = self.weight.view(self.out_channels, -1, 1)\n\n        if self.kernel_type == 'linear':\n            output = (input * weight).sum(dim=2)\n\n        elif self.kernel_type == 'manhattan':\n            output = -((input - weight).abs().sum(dim=2))\n\n        elif self.kernel_type == 'euclidean':\n            output = -(((input - weight)**2).sum(dim=2))\n\n        elif self.kernel_type == 'polynomial':\n            output = ((input * weight).sum(dim=2) + self.balance)**self.power\n\n        elif self.kernel_type == 'gaussian':\n            output = (-self.gamma*((input - weight)**2).sum(dim=2)).exp() + 0\n\n        else:\n            raise NotImplementedError(self.kernel_type+' Kerv1d not implemented')\n\n        if self.bias is not None:\n            output += self.bias.view(self.out_channels, -1)\n\n        return output\n\n\n    def cuda(self, device=None):\n        if self.learnable_kernel == True:\n            self.balance = self.balance.cuda(device)\n            self.gamma = self.gamma.cuda(device)\n        return self._apply(lambda t: t.cuda(device))\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class LinearKernel(torch.nn.Module):\n    def __init__(self):\n        super(LinearKernel, self).__init__()\n    \n    def forward(self, x_unf, w, b):\n        t = x_unf.transpose(1, 2).matmul(w.view(w.size(0), -1).t()).transpose(1, 2)\n        if b is not None:\n            return t + b\n        return t\n        \n        \nclass PolynomialKernel(LinearKernel):\n    def __init__(self, cp=2.0, dp=3, train_cp=True):\n        super(PolynomialKernel, self).__init__()\n        self.cp = torch.nn.parameter.Parameter(torch.tensor(cp, requires_grad=train_cp))\n        self.dp = dp\n\n    def forward(self, x_unf, w, b):\n        return (self.cp + super(PolynomialKernel, self).forward(x_unf, w, b))**self.dp\n\n\nclass GaussianKernel(torch.nn.Module):\n    def __init__(self, gamma):\n        super(GaussianKernel, self).__init__()\n        self.gamma = torch.nn.parameter.Parameter(\n                            torch.tensor(gamma, requires_grad=True))\n    \n    def forward(self, x_unf, w, b):\n        l = x_unf.transpose(1, 2)[:, :, :, None] - w.view(1, 1, -1, w.size(0))\n        l = torch.sum(l**2, 2)\n        t = torch.exp(-self.gamma * l)\n        if b:\n            return t + b\n        return t\n        \n       \nclass KernelConv2d(torch.nn.Conv2d):\n    def __init__(self, in_channels, out_channels, kernel_size, kernel_fn=PolynomialKernel,\n                 stride=1, padding=0, dilation=1, groups=1, bias=None,\n                 padding_mode='zeros'):\n        '''\n        Follows the same API as torch Conv2d except kernel_fn.\n        kernel_fn should be an instance of the above kernels.\n        '''\n        super(KernelConv2d, self).__init__(in_channels, out_channels, \n                                           kernel_size, stride, padding,\n                                           dilation, groups, bias, padding_mode)\n        self.kernel_fn = kernel_fn()\n   \n    def compute_shape(self, x):\n        h = (x.shape[2] + 2 * self.padding[0] - 1 * (self.kernel_size[0] - 1) - 1) // self.stride[0] + 1\n        w = (x.shape[3] + 2 * self.padding[1] - 1 * (self.kernel_size[1] - 1) - 1) // self.stride[1] + 1\n        return h, w\n    \n    def forward(self, x):\n        x_unf = torch.nn.functional.unfold(x, self.kernel_size, self.dilation,self.padding, self.stride)\n        h, w = self.compute_shape(x)\n        return self.kernel_fn(x_unf, self.weight, self.bias).view(x.shape[0], -1, h, w)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(os.listdir(\"../input/kernvolution-pytorch-library/kervolution-pytorch-master\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path.append(\"../input/kernvolution-pytorch-library/kervolution-pytorch-master/\")\nfrom layer import KernelConv2d, GaussianKernel, PolynomialKernel\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"af2dc7552bd0c835fdb741396c052757d189c8b8"},"cell_type":"code","source":"class CancerDataset(Dataset):\n    def __init__(self, datafolder, datatype='train', transform = transforms.Compose([transforms.ToTensor()]), labels_dict={}):\n        self.datafolder = datafolder\n        self.datatype = datatype\n        self.image_files_list = [s for s in os.listdir(datafolder)]\n        self.transform = transform\n        self.labels_dict = labels_dict\n        if self.datatype == 'train':\n            lab = [labels_dict[i.split('.')[0]] for i in self.image_files_list]\n            print(lab)\n            self.labels = lab \n        else:\n            self.labels = [0 for _ in range(len(self.image_files_list))]\n\n    def __len__(self):\n        return len(self.image_files_list)\n\n    def __getitem__(self, idx):\n        img_name = os.path.join(self.datafolder, self.image_files_list[idx])\n        image = Image.open(img_name)\n        image = self.transform(image)\n        img_name_short = self.image_files_list[idx].split('.')[0]\n\n        if self.datatype == 'train':\n            label = self.labels_dict[img_name_short]\n        else:\n            label = 0\n        return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nimport pandas as pd\nimport os\nos.listdir(\"../input/histopathologic-cancer-detection\")\n#pd.read_csv('../input/histopathologic-cancer-detection/train_labels.csv')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"IMAGE_NOT_FOUND_COUNTER = 0\n\nlabels = pd.read_csv('../input/histopathologic-cancer-detection/train_labels.csv')\n\ndata_transforms = transforms.Compose([\n    #transforms.CenterCrop(32),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomVerticalFlip(),\n    transforms.ToTensor(),\n    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n    ])\ndata_transforms_test = transforms.Compose([\n    #transforms.CenterCrop(32),\n    transforms.ToTensor(),\n    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n    ])\n\n\n\ntr, val = train_test_split(labels.label, stratify=labels.label, test_size=0.2)\nprint(\"number of training data: \",len(tr))\nprint(\"number of testing  data: \",len(val))\n# dictionary with labels and ids of train data\nimg_class_dict = {k:v for k, v in zip(labels.id, labels.label)}\n\ntrain_sampler = SubsetRandomSampler(list(tr.index))\nvalid_sampler = SubsetRandomSampler(list(val.index))\nbatch_size = 128\nnum_workers = 0\n\ndataset = CancerDataset(datafolder='../input/histopathologic-cancer-detection/train/', datatype='train', transform=data_transforms, labels_dict=img_class_dict)\ntest_set = CancerDataset(datafolder='../input/histopathologic-cancer-detection/test/', datatype='test', transform=data_transforms_test)\n# prepare data loaders (combine dataset and sampler)\ntrain_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=train_sampler, num_workers=num_workers)\nvalid_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=valid_sampler, num_workers=num_workers)\ntest_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, num_workers=num_workers)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90bae2c4a31beaf2945d2f54c9f092155bfed1a9"},"cell_type":"code","source":"class Flatten(nn.Module):\n    def forward(self, x):\n        N, C, H, W = x.size() # read in N, C, H, W\n        return x.view(N, -1)  # \"flatten\" the C * H * W values into a single vector per image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9112b0d404cc38ead7693b7ee6c9cb5cf792d4b4"},"cell_type":"code","source":"avg_loss_list = []\nacc_list = []\n\ndef train(model, train_loader ,loss_fn, optimizer, num_epochs = 1):\n    total_loss =0\n\n    for epoch in range(num_epochs):\n        print('Starting epoch %d / %d' % (epoch + 1, num_epochs))\n        model.train()\n        for t, (x, y) in enumerate(train_loader):\n            x_var = Variable(x.type(gpu_dtype))\n            y_var = Variable(y.type(gpu_dtype).long())\n\n            scores = model(x_var)\n            loss = loss_fn(scores, y_var)\n            total_loss += loss.data\n            \n            if (t + 1) % print_every == 0:\n                avg_loss = total_loss/print_every\n                print('t = %d, avg_loss = %.4f' % (t + 1, avg_loss) )\n                avg_loss_list.append(avg_loss)\n                total_loss = 0\n                \n\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n        acc = check_accuracy(fixed_model_gpu, valid_loader)\n        print('acc = %f' %(acc))\n            \ndef check_accuracy(model, loader):\n    print('Checking accuracy on test set')   \n    num_correct = 0\n    num_samples = 0\n    model.eval() # Put the model in test mode (the opposite of model.train(), essentially)\n    for x, y in loader:\n        x_var = Variable(x.type(gpu_dtype))\n\n        scores = model(x_var)\n        _, preds = scores.data.cpu().max(1)\n        num_correct += (preds == y).sum()\n        num_samples += preds.size(0)\n    acc = float(num_correct) / num_samples\n    acc_list.append(acc)\n    return acc\n    print('Got %d / %d correct (%.2f)' % (num_correct, num_samples, 100 * acc))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# class KernelConv2d(torch.nn.Conv2d):\n#     def __init__(self, in_channels, out_channels, kernel_size, kernel_fn=PolynomialKernel,\n#                  stride=1, padding=0, dilation=1, groups=1, bias=None,\n#                  padding_mode='zeros'):\n#         '''\n#         Follows the same API as torch Conv2d except kernel_fn.\n#         kernel_fn should be an instance of the above kernels.\n#         '''\n        \n#         super(KernelConv2d, self).__init__(in_channels, out_channels, \n#                                            kernel_size=kernel_size, stride=stride, padding=1, bias=False)\n#         self.kernel_fn = kernel_fn()\n   \n#     def compute_shape(self, x):\n#         h = (x.shape[2] + 2 * self.padding[0] - 1 * (self.kernel_size[0] - 1) - 1) // self.stride[0] + 1\n#         w = (x.shape[3] + 2 * self.padding[1] - 1 * (self.kernel_size[1] - 1) - 1) // self.stride[1] + 1\n#         return h, w\n    \n#     def forward(self, x):\n#         x_unf = torch.nn.functional.unfold(x, self.kernel_size, self.dilation,self.padding, self.stride)\n#         h, w = self.compute_shape(x)\n#         return self.kernel_fn(x_unf, self.weight, self.bias).view(x.shape[0], -1, h, w)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"cdcb7f6c9adffc8b1b2862773997e7f32726f5fc"},"cell_type":"code","source":"from torchvision import models\n\nprint_every = 20\ngpu_dtype = torch.cuda.FloatTensor\n\nout_1 = 32\nout_2 = 64\nout_3 = 128\nout_4 = 256\n\nk_size_1 = 3\npadding_1 = 1\n\n\nnum_epochs = 6\n\n\n\nfixed_model_base = nn.Sequential( # You fill this in!\n    #KernelConv2d(3, out_1, kernel_size=k_size_1),\n                Kerv2d(3, out_1, padding= padding_1, kernel_size=k_size_1, stride=1), # out_1-k_size_1+1 = 26\n                nn.ReLU(inplace=True),\n                nn.BatchNorm2d(out_1),\n    #KernelConv2d(out_1, out_1, kernel_size=k_size_1),\n                nn.Conv2d(out_1 , out_1, padding= padding_1, kernel_size=k_size_1, stride=1), #26 - 4 + 1 = 23\n                nn.ReLU(inplace=True),\n                nn.BatchNorm2d(out_1),\n    #KernelConv2d(out_1, out_1, kernel_size=k_size_1),\n                nn.Conv2d(out_1 , out_1, padding= padding_1, kernel_size=k_size_1, stride=1), # 23 -3 = 20\n                nn.ReLU(inplace=True),\n                nn.BatchNorm2d(out_1),\n    \n                nn.MaxPool2d(2, stride=2),\n    #KernelConv2d(out_1, out_2, kernel_size=k_size_1),\n                nn.Conv2d(out_1 , out_2, padding= padding_1, kernel_size=k_size_1, stride=1), # 20 -3 = 17\n                nn.ReLU(inplace=True),\n                nn.BatchNorm2d(out_2),\n    #KernelConv2d(out_2, out_2, kernel_size=k_size_1),\n                nn.Conv2d(out_2 , out_2, padding= padding_1, kernel_size=k_size_1, stride=1), # 17 -3 = 14\n                nn.ReLU(inplace=True), \n                nn.BatchNorm2d(out_2),\n    #KernelConv2d(out_2, out_2, kernel_size=k_size_1),\n                nn.Conv2d(out_2 , out_2, padding= padding_1, kernel_size=k_size_1, stride=1), # 17 -3 = 14\n                nn.ReLU(inplace=True),\n                nn.BatchNorm2d(out_2),\n    \n                nn.MaxPool2d(2, stride=2),\n    #KernelConv2d(out_2, out_3, kernel_size=k_size_1),\n                nn.Conv2d(out_2 , out_3, padding= padding_1, kernel_size=k_size_1, stride=1),\n                nn.ReLU(inplace=True),\n                nn.BatchNorm2d(out_3),\n    #KernelConv2d(out_3, out_3, kernel_size=k_size_1),\n                nn.Conv2d(out_3 , out_3, padding= padding_1, kernel_size=k_size_1, stride=1), # 17 -3 = 14\n                nn.ReLU(inplace=True),\n                nn.BatchNorm2d(out_3),\n    #KernelConv2d(out_3, out_3, kernel_size=k_size_1),\n                nn.Conv2d(out_3 , out_3, padding= padding_1, kernel_size=k_size_1, stride=1), # 17 -3 = 14\n                nn.ReLU(inplace=True),\n                nn.BatchNorm2d(out_3),\n    \n                nn.MaxPool2d(2, stride=2),\n    #KernelConv2d(out_3, out_4, kernel_size=k_size_1),\n                nn.Conv2d(out_3 , out_4, padding= padding_1, kernel_size=k_size_1, stride=1), # 17 -3 = 14\n                nn.ReLU(inplace=True),\n                nn.BatchNorm2d(out_4),\n    #KernelConv2d(out_4, out_4, kernel_size=k_size_1),\n                nn.Conv2d(out_4 , out_4, padding= padding_1, kernel_size=k_size_1, stride=1), # 17 -3 = 14\n                nn.ReLU(inplace=True),\n                nn.BatchNorm2d(out_4),\n    #KernelConv2d(out_4, out_4, kernel_size=k_size_1),\n                nn.Conv2d(out_4 , out_4, padding= padding_1, kernel_size=k_size_1, stride=1), # 17 -3 = 14\n                nn.ReLU(inplace=True),\n                nn.BatchNorm2d(out_4),\n                        \n    \n                nn.MaxPool2d(2, stride=2), #17/2 = 7\n                Flatten(),\n                \n                nn.Linear(9216,512 ), # affine layer\n                nn.ReLU(inplace=True),\n                nn.Linear(512,10), # affine layer\n                nn.ReLU(inplace=True),\n                nn.Linear(10,2), # affine layer\n            )\nfixed_model_gpu = fixed_model_base.type(gpu_dtype)\nprint(fixed_model_gpu)\nloss_fn = nn.modules.loss.CrossEntropyLoss()\noptimizer = optim.RMSprop(fixed_model_gpu.parameters(), lr = 1e-3)\n\ntrain(fixed_model_gpu, train_loader ,loss_fn, optimizer, num_epochs=num_epochs)\ncheck_accuracy(fixed_model_gpu, valid_loader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6b2f26fc0e0f59be1eba5774d5d68c1b6f581ef6"},"cell_type":"code","source":"print(avg_loss_list,acc_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6849f23812e7deb06c2500395f80015115bfce2f"},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.plot([i+1 for i in range((len(acc_list)))],acc_list)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Loss\")\nplt.plot([print_every*batch_size*(i+1)/len(tr) for i in range((len(avg_loss_list)))],avg_loss_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6088c735b19731d3c5cfdd28811ceca346cabc66"},"cell_type":"code","source":"fixed_model_gpu.eval()\npreds = []\nfor batch_i, (data, target) in enumerate(test_loader):\n    data, target = data.cuda(), target.cuda()\n    output = fixed_model_gpu(data)\n\n    pr = output[:,1].detach().cpu().numpy()\n    for i in pr:\n        preds.append(i)\n        \ntest_preds = pd.DataFrame({'imgs': test_set.image_files_list, 'preds': preds})\n\ntest_preds['imgs'] = test_preds['imgs'].apply(lambda x: x.split('.')[0])\n\ndata_to_submit = pd.read_csv('../input/histopathologic-cancer-detection/sample_submission.csv')\ndata_to_submit = pd.merge(data_to_submit, test_preds, left_on='id', right_on='imgs')\ndata_to_submit = data_to_submit[['id', 'preds']]\ndata_to_submit.columns = ['id', 'label']\ndata_to_submit.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4011f2fe1ba4c87cd64cc792ee37fd7977b65432"},"cell_type":"code","source":"data_to_submit.to_csv('csv_to_submit_knn.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"KNN without ReLu and Pooling\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print_every = 20\ngpu_dtype = torch.cuda.FloatTensor\n\nout_1 = 32\nout_2 = 64\nout_3 = 128\nout_4 = 256\n\nk_size_1 = 3\npadding_1 = 1\n\n\nnum_epochs = 6\n\n\n\nfixed_model_base = nn.Sequential( \n                Kerv2d(3, out_1, padding= padding_1, kernel_size=k_size_1, stride=1), # out_1-k_size_1+1 = 26\n               \n                nn.BatchNorm2d(out_1),\n                nn.Conv2d(out_1 , out_1, padding= padding_1, kernel_size=k_size_1, stride=1), #26 - 4 + 1 = 23\n                nn.BatchNorm2d(out_1),\n                nn.Conv2d(out_1 , out_1, padding= padding_1, kernel_size=k_size_1, stride=1), # 23 -3 = 20\n                nn.BatchNorm2d(out_1),\n    \n                nn.AvgPool2d(2, stride=2),\n                nn.Conv2d(out_1 , out_2, padding= padding_1, kernel_size=k_size_1, stride=1), # 20 -3 = 17\n                nn.BatchNorm2d(out_2),\n                nn.Conv2d(out_2 , out_2, padding= padding_1, kernel_size=k_size_1, stride=1), # 17 -3 = 14\n                nn.BatchNorm2d(out_2),\n                nn.Conv2d(out_2 , out_2, padding= padding_1, kernel_size=k_size_1, stride=1), # 17 -3 = 14\n                nn.BatchNorm2d(out_2),\n    \n                nn.AvgPool2d(2, stride=2),\n                nn.Conv2d(out_2 , out_3, padding= padding_1, kernel_size=k_size_1, stride=1),\n                nn.BatchNorm2d(out_3),\n                nn.Conv2d(out_3 , out_3, padding= padding_1, kernel_size=k_size_1, stride=1), # 17 -3 = 14\n                nn.BatchNorm2d(out_3),\n                nn.Conv2d(out_3 , out_3, padding= padding_1, kernel_size=k_size_1, stride=1), # 17 -3 = 14\n                nn.BatchNorm2d(out_3),\n                nn.AvgPool2d(2, stride=2),\n                nn.Conv2d(out_3 , out_4, padding= padding_1, kernel_size=k_size_1, stride=1), # 17 -3 = 14\n                nn.BatchNorm2d(out_4),\n                nn.Conv2d(out_4 , out_4, padding= padding_1, kernel_size=k_size_1, stride=1), # 17 -3 = 14\n                nn.BatchNorm2d(out_4),\n                nn.Conv2d(out_4 , out_4, padding= padding_1, kernel_size=k_size_1, stride=1), # 17 -3 = 14\n                nn.BatchNorm2d(out_4),\n                        \n    \n                nn.AvgPool2d(2, stride=2), #17/2 = 7\n                Flatten(),\n                \n#                 nn.Linear(9216,512 ), # affine layer\n#                 nn.ReLU(inplace=True),\n#                 nn.Linear(512,10), # affine layer\n#                 nn.ReLU(inplace=True),\n#                 nn.Linear(10,2), # affine layer\n            )\nfixed_model_gpu = fixed_model_base.type(gpu_dtype)\nprint(fixed_model_gpu)\nloss_fn = nn.modules.loss.CrossEntropyLoss()\noptimizer = optim.RMSprop(fixed_model_gpu.parameters(), lr = 1e-3)\n\ntrain(fixed_model_gpu, train_loader ,loss_fn, optimizer, num_epochs=num_epochs)\ncheck_accuracy(fixed_model_gpu, valid_loader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(avg_loss_list,acc_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot([i+1 for i in range((len(acc_list)))],acc_list)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Loss\")\nplt.plot([print_every*batch_size*(i+1)/len(tr) for i in range((len(avg_loss_list)))],avg_loss_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fixed_model_gpu.eval()\npreds = []\nfor batch_i, (data, target) in enumerate(test_loader):\n    data, target = data.cuda(), target.cuda()\n    output = fixed_model_gpu(data)\n\n    pr = output[:,1].detach().cpu().numpy()\n    for i in pr:\n        preds.append(i)\n        \ntest_preds = pd.DataFrame({'imgs': test_set.image_files_list, 'preds': preds})\n\ntest_preds['imgs'] = test_preds['imgs'].apply(lambda x: x.split('.')[0])\n\ndata_to_submit = pd.read_csv('../input/histopathologic-cancer-detection/sample_submission.csv')\ndata_to_submit = pd.merge(data_to_submit, test_preds, left_on='id', right_on='imgs')\ndata_to_submit = data_to_submit[['id', 'preds']]\ndata_to_submit.columns = ['id', 'label']\ndata_to_submit.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_to_submit.to_csv('csv_to_submit_knn_without_relu_pool.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Custom CNN:\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print_every = 20\ngpu_dtype = torch.cuda.FloatTensor\n\nout_1 = 32\nout_2 = 64\nout_3 = 128\nout_4 = 256\n\nk_size_1 = 3\npadding_1 = 1\n\n\nnum_epochs = 6\n\n\n\nfixed_model_base = nn.Sequential( \n                nn.Conv2d(3, out_1, padding= padding_1, kernel_size=k_size_1, stride=1), # out_1-k_size_1+1 = 26\n                nn.ReLU(inplace=True),\n                nn.BatchNorm2d(out_1),\n                nn.Conv2d(out_1 , out_1, padding= padding_1, kernel_size=k_size_1, stride=1), #26 - 4 + 1 = 23\n                nn.ReLU(inplace=True),\n                nn.BatchNorm2d(out_1),\n                nn.Conv2d(out_1 , out_1, padding= padding_1, kernel_size=k_size_1, stride=1), # 23 -3 = 20\n                nn.ReLU(inplace=True),\n                nn.BatchNorm2d(out_1),\n    \n                nn.MaxPool2d(2, stride=2),\n                nn.Conv2d(out_1 , out_2, padding= padding_1, kernel_size=k_size_1, stride=1), # 20 -3 = 17\n                nn.ReLU(inplace=True),\n                nn.BatchNorm2d(out_2),\n                nn.Conv2d(out_2 , out_2, padding= padding_1, kernel_size=k_size_1, stride=1), # 17 -3 = 14\n                nn.ReLU(inplace=True), \n                nn.BatchNorm2d(out_2),\n                nn.Conv2d(out_2 , out_2, padding= padding_1, kernel_size=k_size_1, stride=1), # 17 -3 = 14\n                nn.ReLU(inplace=True),\n                nn.BatchNorm2d(out_2),\n    \n                nn.MaxPool2d(2, stride=2),\n                nn.Conv2d(out_2 , out_3, padding= padding_1, kernel_size=k_size_1, stride=1),\n                nn.ReLU(inplace=True),\n                nn.BatchNorm2d(out_3),\n                nn.Conv2d(out_3 , out_3, padding= padding_1, kernel_size=k_size_1, stride=1), # 17 -3 = 14\n                nn.ReLU(inplace=True),\n                nn.BatchNorm2d(out_3),\n                nn.Conv2d(out_3 , out_3, padding= padding_1, kernel_size=k_size_1, stride=1), # 17 -3 = 14\n                nn.ReLU(inplace=True),\n                nn.BatchNorm2d(out_3),\n                nn.MaxPool2d(2, stride=2),\n                nn.Conv2d(out_3 , out_4, padding= padding_1, kernel_size=k_size_1, stride=1), # 17 -3 = 14\n                nn.ReLU(inplace=True),\n                nn.BatchNorm2d(out_4),\n                nn.Conv2d(out_4 , out_4, padding= padding_1, kernel_size=k_size_1, stride=1), # 17 -3 = 14\n                nn.ReLU(inplace=True),\n                nn.BatchNorm2d(out_4),\n                nn.Conv2d(out_4 , out_4, padding= padding_1, kernel_size=k_size_1, stride=1), # 17 -3 = 14\n                nn.ReLU(inplace=True),\n                nn.BatchNorm2d(out_4),\n                        \n    \n                nn.MaxPool2d(2, stride=2), #17/2 = 7\n                Flatten(),\n                \n                nn.Linear(9216,512 ), # affine layer\n                nn.ReLU(inplace=True),\n                nn.Linear(512,10), # affine layer\n                nn.ReLU(inplace=True),\n                nn.Linear(10,2), # affine layer\n            )\nfixed_model_gpu = fixed_model_base.type(gpu_dtype)\nprint(fixed_model_gpu)\nloss_fn = nn.modules.loss.CrossEntropyLoss()\noptimizer = optim.RMSprop(fixed_model_gpu.parameters(), lr = 1e-3)\n\ntrain(fixed_model_gpu, train_loader ,loss_fn, optimizer, num_epochs=num_epochs)\ncheck_accuracy(fixed_model_gpu, valid_loader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(avg_loss_list,acc_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot([i+1 for i in range((len(acc_list)))],acc_list)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Loss\")\nplt.plot([print_every*batch_size*(i+1)/len(tr) for i in range((len(avg_loss_list)))],avg_loss_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fixed_model_gpu.eval()\npreds = []\nfor batch_i, (data, target) in enumerate(test_loader):\n    data, target = data.cuda(), target.cuda()\n    output = fixed_model_gpu(data)\n\n    pr = output[:,1].detach().cpu().numpy()\n    for i in pr:\n        preds.append(i)\n        \ntest_preds = pd.DataFrame({'imgs': test_set.image_files_list, 'preds': preds})\n\ntest_preds['imgs'] = test_preds['imgs'].apply(lambda x: x.split('.')[0])\n\ndata_to_submit = pd.read_csv('../input/histopathologic-cancer-detection/sample_submission.csv')\ndata_to_submit = pd.merge(data_to_submit, test_preds, left_on='id', right_on='imgs')\ndata_to_submit = data_to_submit[['id', 'preds']]\ndata_to_submit.columns = ['id', 'label']\ndata_to_submit.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_to_submit.to_csv('csv_to_submit_cnn.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7a4baf77ec397302c7c35af473595f74b6cf240c"},"cell_type":"markdown","source":"#citation\n* data parsing and code for submittion are taken from: https://www.kaggle.com/artgor/simple-eda-and-model-in-pytorch"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}