{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ../input/pretrainedmodels/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4/ > /dev/null","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install efficientnet-pytorch","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport sys\n\nimport numpy as np\nimport pandas as pd\nimport torchvision #package consists of popular datasets, model architectures, and common image transformations for computer vision.\nimport torch.nn as nn #neural network model\nfrom tqdm import tqdm #progress bars to Python code \nfrom PIL import Image, ImageFile #adds support for opening, manipulating, and saving many different image file formats.\nfrom torch.utils.data import Dataset #An abstract class representing a Dataset.\nimport torch #pytorch\nfrom torchvision import transforms #image tranformation \nimport os #functions for interacting with the operating system.\nimport pretrainedmodels \nfrom efficientnet_pytorch import EfficientNet \ndevice = torch.device(\"cuda:0\") # used to send the model to selected GPU\nImageFile.LOAD_TRUNCATED_IMAGES = True\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from albumentations import (\n    PadIfNeeded,\n    HorizontalFlip,\n    VerticalFlip,    \n    CenterCrop,    \n    Crop,\n    Rotate,\n    Compose,\n    Transpose,\n    RandomRotate90,\n    ElasticTransform,\n    GridDistortion, \n    OpticalDistortion,\n    RandomSizedCrop,\n    OneOf,\n    #MultiplicativeNoise,\n    #CoarseDropout,\n    CLAHE,\n    RandomBrightnessContrast,    \n    Cutout,\n    RandomGamma,\n    Rotate,\n    ShiftScaleRotate ,\n    GaussNoise,\n    Blur,\n    #GlassBlur,\n    Normalize,\n    MotionBlur,\n    MedianBlur,   \n    IAAPiecewiseAffine,\n    GaussianBlur\n)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"class RetinopathyDatasetTest(Dataset):\n    def __init__(self, csv_file, transform):\n        self.data = pd.read_csv(csv_file)\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.data)  #returns the number of items in an object.\n\n    def __getitem__(self, idx):\n        img_name = os.path.join('../input/aptos2019-blindness-detection/test_images', self.data.loc[idx, 'id_code'] + '.png')\n        image = Image.open(img_name)\n        image = image.resize((224, 224), resample=Image.BILINEAR)\n        image = np.asarray( image, dtype=np.uint8 ) #image is converted into an array datatype integer\n        if self.transform:\n            result = self.transform(image=image)\n            image = result['image']\n        return {'image': transforms.ToTensor()(image)} #Data loader need the input in the form of tensor\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.nn as nn\nimport torch.nn.functional as F  #(uncomment if needed,but you likely already have it)\n\n#Mish - \"Mish: A Self Regularized Non-Monotonic Neural Activation Function\"\n#https://arxiv.org/abs/1908.08681v1\n#implemented for PyTorch / FastAI by lessw2020 \n#github: https://github.com/lessw2020/mish\n\nclass Mish(nn.Module):\n    def __init__(self):\n        super().__init__() #the function returns a temporary object that allows reference to a parent class \n\n    def forward(self, x):\n        #inlining this saves 1 second per epoch (V100 GPU) vs having a temp x and then returning x(!)\n        return x *( torch.tanh(F.softplus(x))) #provides support for the hyperbolic tangent function in PyTorch.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class AdaptiveConcatPool2d(nn.Module):   \n    \"Layer that concats `AdaptiveAvgPool2d` and `AdaptiveMaxPool2d`.\"\n    def __init__(self, sz):\n      super().__init__()\n      \"Output will be 2*sz or 2 if sz is None\"\n      self.output_size = sz or 1\n      self.ap = nn.AdaptiveAvgPool2d(self.output_size)\n      self.mp = nn.AdaptiveMaxPool2d(self.output_size)\n\n    def forward(self, x): return torch.cat([self.mp(x), self.ap(x)], 1)\n    \n    \n    #uses adaptive average pooling and adaptive max pooling and concatenates them both.\n    #Max pooling extracts the most important features like edges whereas, average pooling extracts features so smoothly.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Flatten(nn.Module):\n    \"Flatten `x` to a single dimension, often used at the end of a model. `full` for rank-1 tensor\"\n    def __init__(self, full:bool=False): \n      super().__init__()\n      self.full = full\n    def forward(self, x): return x.view(-1) if self.full else x.view(x.size(0), -1)\n    \n    #used to get a copy of an given array collapsed into one dimension.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.nn.parameter import Parameter\n\ndef gem(x, p=3, eps=1e-6):\n    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1./p)\n\nclass GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6):\n        super(GeM,self).__init__()\n        self.p = Parameter(torch.ones(1)*p)\n        self.eps = eps\n    def forward(self, x):\n        return gem(x, p=self.p, eps=self.eps)       \n    def __repr__(self):\n        return self.__class__.__name__ + '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist()[0]) + ', ' + 'eps=' + str(self.eps) + ')'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def convert_to_gem(model):\n    for child_name, child in model.named_children():\n        if isinstance(child, nn.AdaptiveAvgPool2d):\n            setattr(model, child_name, GeM())\n        else:\n            convert_to_gem(child)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def bn_drop_lin( n_in:int, n_out:int, actn:nn.Module,bn:bool=True, p:float=0.):\n    \"Sequence of batchnorm (if `bn`), dropout (with `p`) and linear (`n_in`,`n_out`) layers followed by `actn`.\"\n    layers = [nn.BatchNorm1d(n_in)] if bn else []\n    if p != 0: layers.append(nn.Dropout(p))\n    layers.append(nn.Linear(n_in, n_out))\n    if actn is not None: layers.append(actn)\n    return layers\n\n#Since we are computing the mean and standard deviation from a single batch as opposed to computing it from the entire data. \n#Batch normalization is done individually at each hidden neuron in the network.\n#concept of dropout in neural networks specifically how it helps to reduce overfitting and generalization error.\n#Dropout is a regularization technique that “drops out” or “deactivates” few neurons in the neural network randomly in order to avoid the problem of overfitting.\n#Each Linear Module computes output from input using a linear function, and holds internal Tensors for its weight and bias.\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Head(nn.Module):\n    def __init__(self, nc, n, ps=0.5):\n        super(Head,self).__init__()\n        layers = [AdaptiveConcatPool2d(1), Mish(), Flatten()] + \\\n            bn_drop_lin(nc*2, 512, Mish(),True, ps) + \\\n            bn_drop_lin(512, n, None, True, ps)\n        self.fc = nn.Sequential(*layers)\n        self._init_weight()\n        \n    def _init_weight(self):\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                torch.nn.init.kaiming_normal_(m.weight)\n            elif isinstance(m, nn.BatchNorm2d):\n                m.weight.data.fill_(1.0)\n                m.bias.data.zero_()\n        \n    def forward(self, x):\n        return self.fc(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"arch1 = pretrainedmodels.__dict__['se_resnext50_32x4d']\narch2 = 'efficientnet-b2'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DRModel1(nn.Module):\n    def __init__(self, arch1 = arch1 , pre=True):\n        super(DRModel1,self).__init__()\n        m = arch1(pretrained='imagenet') if pre else arch1(pretrained=None) #base model inherit\n        conv = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False) \n        #w = (m.layer0.conv1.weight.sum(1)).unsqueeze(1)\n        #conv.weight = nn.Parameter(w)\n        self.layer0 = nn.Sequential(conv, m.layer0.bn1, m.layer0.relu1, m.layer0.pool)\n        self.layer1 = m.layer1\n        self.layer2 = m.layer2\n        self.layer3 = m.layer3\n        self.layer4 = nn.Sequential(m.layer4[0], m.layer4[1], m.layer4[2]) #\n\n        \n        nc = self.layer4[-1].se_module.fc2.out_channels       # changes as per architecture\n        self.head = Head(nc,1)\n#         self.head1 = Head(nc,n[0])\n#         self.base_model = m\n#         self.head = Head(1000,1)\n#         convert_sigmoid_to_mish(self.base_model)\n#         convert_relu_to_mish(self.base_model)\n        \n        \n    def forward(self, x):    \n        x = self.layer0(x)\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n        \n        x = self.head(x)\n        \n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DRModel2(nn.Module):\n    def __init__(self, arch2 = arch2 , pre=True):\n        super(DRModel2,self).__init__()\n        m = EfficientNet.from_pretrained(arch2 ) if pre else EfficientNet.from_name(arch2)\n        conv = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False)\n        #w = (m.layer0.conv1.weight.sum(1)).unsqueeze(1)\n        #conv.weight = nn.Parameter(w)\n        #self.conv1=conv\n        self.base = m\n        self.base._fc = nn.Linear(1408,1,bias = True)\n        \n        #nc = self.layer4[-1].se_module.fc2.out_channels       # changes as per architecture\n        #self.head = Head(nc,1)\n#         self.head1 = Head(nc,n[0])\n#         self.base_model = m\n#         self.head = Head(1000,1)\n#         convert_sigmoid_to_mish(self.base_model)\n#         convert_relu_to_mish(self.base_model)\n        \n        \n    def forward(self, x):    \n        #x= self.conv1(x)\n        x = self.base(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model1 = DRModel1(pre=False)\n#convert_to_gem(model1)\nmodel1 = model1.to(device)\nmodel2 = DRModel2(pre=False)\nconvert_to_gem(model2)\nmodel2 = model2.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model1.load_state_dict(torch.load(\"../input/modelweights/model_S.pth\"))\nmodel2.load_state_dict(torch.load(\"../input/updatedmodelweights/model_Enew.pth\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model1.eval()\nmodel2.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_transforms = Compose([ \n    ShiftScaleRotate(p=1,rotate_limit=180, shift_limit=0.2,scale_limit=0.2),\n    Normalize(mean=(0.0692),std=(0.2051)),\n    #Flip(p=0.6),\n    GaussianBlur(blur_limit=5)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = RetinopathyDatasetTest(csv_file='../input/aptos2019-blindness-detection/sample_submission.csv',\n                                      transform=test_transforms)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds1 = np.zeros((len(test_dataset), 1))\ntk0 = tqdm(test_data_loader,total=int(len(test_data_loader)))\nfor i, x_batch in enumerate(tk0):\n    x_batch = x_batch[\"image\"]\n    pred = model1(x_batch.to(device))\n    preds1[i * 32:(i + 1) * 32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds2 = np.zeros((len(test_dataset), 1))\ntk1 = tqdm(test_data_loader , total=int(len(test_data_loader)))\nfor i, x_batch in enumerate(tk1):\n    x_batch = x_batch[\"image\"]\n    pred = model2(x_batch.to(device))\n    preds2[i * 32:(i + 1) * 32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds = (preds1 +preds2) / 2.0\n#test_preds = preds2\n#test_preds = preds1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"coef = [0.5, 1.5, 2.5, 3.5]\n\nfor i, pred in enumerate(test_preds):\n    if pred < coef[0]:\n        test_preds[i] = 0\n    elif pred >= coef[0] and pred < coef[1]:\n        test_preds[i] = 1\n    elif pred >= coef[1] and pred < coef[2]:\n        test_preds[i] = 2\n    elif pred >= coef[2] and pred < coef[3]:\n        test_preds[i] = 3\n    else:\n        test_preds[i] = 4\n\n\nsample = pd.read_csv(\"../input/aptos2019-blindness-detection/sample_submission.csv\")\nsample.diagnosis = test_preds.astype(int)\nsample.to_csv(r'C:\\Users\\Admin\\Desktop\\helpme.csv', index=False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"i=0\nj=0\nfor i in range(sample.shape[0]):\n    if sample.diagnosis[i] == 4:\n        j = j+1\n        \nprint (j)\ni","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample1 = pd.read_csv(\"../input/aptos2019-blindness-detection/train.csv\")\n#sample1.shape[0]\n#sample1.diagnosis = test_preds.astype(int)\n#sample1.to_csv(r'C:\\Users\\Admin\\Desktop\\fafa2.csv', index=False)\ni=0\nj=0\nfor i in range(sample1.shape[0]):\n    if sample1.diagnosis[i] == 4:\n        j = j+1\nprint(j)\nprint(j/sample1.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample.iloc[1435:1900,:]","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}