{"cells":[{"metadata":{},"cell_type":"markdown","source":"- https://www.kaggle.com/chanhu/eye-inference-num-class-1-ver3\n- https://www.kaggle.com/solomonk/ensemble-8-x-tta-2-models-008/notebook"},{"metadata":{},"cell_type":"markdown","source":"# 1 Submission 1\n\n- https://www.kaggle.com/chanhu/eye-inference-num-class-1-ver3"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nfrom os.path import isfile\nimport sys\nimport numpy as np\nimport pandas as pd \nimport time\nimport random\nimport urllib\nimport pickle\nfrom tqdm import tqdm_notebook as tqdm\nimport cv2\nfrom PIL import Image, ImageFilter\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn import metrics\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nimport torch\nimport torch.nn.init as init\nimport torch.nn as nn\nfrom torch.utils.data import Dataset\nfrom torchvision import transforms\nfrom torch.optim import Adam, SGD, RMSprop\nfrom torch.autograd import Variable\nimport torch.functional as F\nimport torch.nn.functional as F\nfrom torchvision import models\n\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"package_path = '../input/efficientnet-pytorch/efficientnet-pytorch/EfficientNet-PyTorch-master/'\nsys.path.append(package_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from efficientnet_pytorch import EfficientNet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nSEED = 620402\nseed_everything(SEED)\nTTA         = 3\nnum_classes = 1\nIMG_SIZE    = 256\n\ntest = '../input/aptos2019-blindness-detection/test_images/'\n\ndef expand_path(p):\n    p = str(p)\n    if isfile(test + p + \".png\"):\n        return test + (p + \".png\")\n    return p\n\ndef p_show(imgs, label_name=None, per_row=3):\n    n = len(imgs)\n    rows = (n + per_row - 1)//per_row\n    cols = min(per_row, n)\n    fig, axes = plt.subplots(rows,cols, figsize=(15,15))\n    for ax in axes.flatten(): ax.axis('off')\n    for i,(p, ax) in enumerate(zip(imgs, axes.flatten())): \n        img = Image.open(expand_path(p))\n        ax.imshow(img)\n        ax.set_title(train_df[train_df.id_code == p].diagnosis.values)\n        \ndef crop_image1(img,tol=7):\n    # img is image data\n    # tol  is tolerance\n        \n    mask = img>tol\n    return img[np.ix_(mask.any(1),mask.any(0))]\n\ndef crop_image_from_gray(img,tol=7):\n    if img.ndim ==2:\n        mask = img>tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    elif img.ndim==3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img>tol\n        \n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0): # image is too dark so that we crop out everything,\n            return img # return original image\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n    #         print(img1.shape,img2.shape,img3.shape)\n            img = np.stack([img1,img2,img3],axis=-1)\n    #         print(img.shape)\n        return img\n    \nclass MyDataset(Dataset):\n    \n    def __init__(self, dataframe, transform=None):\n        self.df = dataframe\n        self.transform = transform\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        \n        label = self.df.diagnosis.values[idx]\n        label = np.expand_dims(label, -1)\n        \n        p = self.df.id_code.values[idx]\n        p_path = expand_path(p)\n        image = cv2.imread(p_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image = crop_image_from_gray(image)\n        image = cv2.resize(image, (IMG_SIZE, IMG_SIZE))\n        image = cv2.addWeighted ( image,4, cv2.GaussianBlur( image , (0,0) , 30) ,-4 ,128)\n        image = transforms.ToPILImage()(image)\n        \n        if self.transform:\n            image = self.transform(image)\n        \n        return image, label\n\ntest_transform = transforms.Compose([\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomRotation((-120, 120)),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])\n\ntestset        = MyDataset(pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv'), \n                 transform=test_transform)\ntest_loader    = torch.utils.data.DataLoader(testset, batch_size=16, shuffle=False)\n\nmodel = EfficientNet.from_name('efficientnet-b0')\nin_features = model._fc.in_features\nmodel._fc = nn.Linear(in_features, num_classes)\nmodel.load_state_dict(torch.load('../input/enet-test/weight_best(3).pt'))\nmodel.cuda()\n\nfor param in model.parameters():\n    param.requires_grad = False\n    \nsample = pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')\n\ntest_pred = np.zeros((len(sample), 1))\nmodel.eval()\n\nfor _ in tqdm(range(TTA)):\n    with torch.no_grad():\n        for i, data in tqdm(enumerate(test_loader)):\n            images, _ = data\n            images = images.cuda()\n            pred = model(images)\n            test_pred[i * 16:(i + 1) * 16] += pred.detach().cpu().squeeze().numpy().reshape(-1, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"output = test_pred / TTA\noutput_1 = output\n\ncoef = [0.57, 1.37, 2.57, 3.57]\n\nfor i, pred in enumerate(output):\n    if pred < coef[0]:\n        output[i] = 0\n    elif pred >= coef[0] and pred < coef[1]:\n        output[i] = 1\n    elif pred >= coef[1] and pred < coef[2]:\n        output[i] = 2\n    elif pred >= coef[2] and pred < coef[3]:\n        output[i] = 3\n    else:\n        output[i] = 4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Results\nfrom IPython.display import display\nsubmission_1 = pd.DataFrame({'id_code' : pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv').id_code.values,\n                             'diagnosis' : np.squeeze(output).astype(int)})\n\ndisplay(submission_1.head())\n\nsubmission_1.to_csv('submission_1.csv', index=False)\nprint(os.listdir('./'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2 Submission 2\n\n- https://www.kaggle.com/solomonk/ensemble-8-x-tta-2-models-008/notebook"},{"metadata":{"trusted":true},"cell_type":"code","source":"%reset -f \n%matplotlib inline\n%config InlineBackend.figure_format = 'retina'\n\nimport PIL\nprint(PIL.PILLOW_VERSION)\nimport numpy as np\nfrom IPython.display import display\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nfrom collections import OrderedDict\nimport copy\nimport json\nfrom glob import glob\nimport fnmatch\nimport argparse\nimport pandas as pd\nimport os, sys, pdb, shutil, time, random\nfrom tqdm import tqdm_notebook as tqdm\nfrom shutil import copyfile\nfrom mpl_toolkits.axes_grid1 import ImageGrid\nfrom os import listdir, makedirs, getcwd, remove\nfrom os.path import isfile, join, abspath, exists, isdir, expanduser\n\nimport torch\nfrom torch import nn, optim\nimport torch.nn.functional as F\nfrom torchvision import datasets, transforms, models\nimport torchvision\nfrom torch.autograd import Variable\nfrom torch.optim import lr_scheduler\nimport torch.backends.cudnn as cudnn\nimport torchvision.datasets as dset\nimport torchvision.transforms as transforms\nfrom torch.autograd import Variable\nfrom torch.utils.data import Dataset, DataLoader\nimport torch.nn.functional as func\n\n# check if CUDA is available\ntrain_on_gpu = torch.cuda.is_available()\n\nif not train_on_gpu:\n    print('CUDA is not available.  Training on CPU ...')\nelse:\n    print('CUDA is available!  Training on GPU ...')\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nprint(device)\n\nfrom PIL import ImageFile\nImageFile.LOAD_TRUNCATED_IMAGES = True\n# device = torch.device(\"cuda:1\")\nImageFile.LOAD_TRUNCATED_IMAGES = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"! ls -la ../input/\n# #Organizing the dataset\ndata_dir = '../input/aptos2019-blindness-detection/'\ntrain_dir = data_dir + '/train_images/'\ntest_dir= data_dir + '/test_images/'\nnThreads = 4\nbatch_size = 32\nuse_gpu = torch.cuda.is_available()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class GenericDataset():\n    def __init__(self, labels, root_dir, subset=False, transform=None):\n        self.labels = labels\n        self.root_dir = root_dir\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.labels)\n\n    def __getitem__(self, idx):\n        img_name = self.labels.iloc[idx, 0]  # file name\n        fullname = join(self.root_dir, img_name)\n        image = Image.open(fullname).convert('RGB')\n        labels = self.labels.iloc[idx, 2]  # category_id\n        #         print (labels)\n        if self.transform:\n            image = self.transform(image)\n        return image, labels\n\n    @staticmethod\n    def find_classes(fullDir):\n        classes = [d for d in os.listdir(fullDir) if os.path.isdir(os.path.join(fullDir, d))]\n        classes.sort()\n        class_to_idx = {classes[i]: i for i in range(len(classes))}\n        idx_to_class = {v: k for k, v in class_to_idx.items()}\n\n        #     idx_to_class = dict(zip(range(len(classes)), classes))\n\n        train = []\n        for index, label in enumerate(classes):\n            path = fullDir + label + '/'\n            for file in listdir(path):\n                train.append(['{}/{}'.format(label, file), label, index])\n\n        df = pd.DataFrame(train, columns=['file', 'category', 'category_id', ])\n\n        return classes, class_to_idx, idx_to_class, df\n\n    @staticmethod\n    def find_classes_retino(fullDir):\n\n        df_labels = pd.read_csv(\"../input/aptos2019-blindness-detection/train.csv\", sep=',')\n        class_to_idx = {'0': 0, '1': 1, '2': 2, '3': 3, '4': 4}\n\n        idx_to_class = dict(zip(class_to_idx.values(), class_to_idx.keys()))\n        classes = ['0', '1', '2', '3', '4']\n\n        print('Sorted Classes: {}'.format(classes))\n        print('class_to_idx: {}'.format(class_to_idx))\n        print('num_to_class: {}'.format(idx_to_class))\n\n        train = []\n        for index, row in (df_labels.iterrows()):\n            id = (row['id_code'])\n            # currImage = os.path.join(fullDir, num_to_class[(int(row['melanoma']))] + '/' + id + '.jpg')\n            currImage_on_disk = os.path.join(fullDir, id + '.png')\n            if os.path.isfile(currImage_on_disk):\n                if (int(row['diagnosis'])) == 0:\n                    trait = '0'\n                elif (int(row['diagnosis'])) == 1:\n                    trait = '1'\n                elif (int(row['diagnosis'])) == 2:\n                    trait = '2'\n                elif (int(row['diagnosis'])) == 3:\n                    trait = '3'\n                elif (int(row['diagnosis'])) == 4:\n                    trait = '4'\n\n                train.append(['{}'.format(currImage_on_disk), trait, class_to_idx[trait]])\n        df = pd.DataFrame(train, columns=['file', 'category', 'category_id', ])\n        if os.path.isfile('full_retino_labels.csv'):\n            os.remove('full_retino_labels.csv')\n        df.to_csv('full_retino_labels.csv', index=None)\n        return classes, class_to_idx, idx_to_class, df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class GenericDatasetTTA():\n    def __init__(self, labels, root_dir, subset=False, transform=None,TTA=8):\n        self.labels = labels\n        self.root_dir = root_dir\n        self.transform = transform\n        self.TTA = TTA\n\n    def __len__(self):\n        return len(self.labels)\n\n    def __getitem__(self, idx):\n        img_name = self.labels.iloc[idx, 0]  # file name\n        fullname = join(self.root_dir, img_name)\n        image = Image.open(fullname).convert('RGB')\n        labels = self.labels.iloc[idx, 2]  # category_id\n        #         print (labels)\n        a = []\n        for _ in range(self.TTA):\n            image1 = self.transform(image)\n            a.append(image1)\n        image = torch.stack(a)\n        return image, labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classes, class_to_idx, idx_to_class, df = GenericDataset.find_classes_retino(train_dir)\ndisplay(df.head(5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls -la '../input/pth-best/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nResNet code gently borrowed from\nhttps://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py\n\"\"\"\nfrom __future__ import print_function, division, absolute_import\nfrom collections import OrderedDict\nimport math\n\nimport torch.nn as nn\nfrom torch.utils import model_zoo\n\n__all__ = ['SENet', 'senet154', 'se_resnet50', 'se_resnet101', 'se_resnet152',\n           'se_resnext50_32x4d', 'se_resnext101_32x4d']\n\npretrained_settings = {\n    'senet154': {\n        'imagenet': {\n            'url': 'http://data.lip6.fr/cadene/pretrainedmodels/senet154-c7b49a05.pth',\n            'input_space': 'RGB',\n            'input_size': [3, 224, 224],\n            'input_range': [0, 1],\n            'mean': [0.485, 0.456, 0.406],\n            'std': [0.229, 0.224, 0.225],\n            'num_classes': 1000\n        }\n    },\n    'se_resnet50': {\n        'imagenet': {\n            'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet50-ce0d4300.pth',\n            'input_space': 'RGB',\n            'input_size': [3, 224, 224],\n            'input_range': [0, 1],\n            'mean': [0.485, 0.456, 0.406],\n            'std': [0.229, 0.224, 0.225],\n            'num_classes': 1000\n        }\n    },\n    'se_resnet101': {\n        'imagenet': {\n            'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet101-7e38fcc6.pth',\n            'input_space': 'RGB',\n            'input_size': [3, 224, 224],\n            'input_range': [0, 1],\n            'mean': [0.485, 0.456, 0.406],\n            'std': [0.229, 0.224, 0.225],\n            'num_classes': 1000\n        }\n    },\n    'se_resnet152': {\n        'imagenet': {\n            'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet152-d17c99b7.pth',\n            'input_space': 'RGB',\n            'input_size': [3, 224, 224],\n            'input_range': [0, 1],\n            'mean': [0.485, 0.456, 0.406],\n            'std': [0.229, 0.224, 0.225],\n            'num_classes': 1000\n        }\n    },\n    'se_resnext50_32x4d': {\n        'imagenet': {\n            'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnext50_32x4d-a260b3a4.pth',\n            'input_space': 'RGB',\n            'input_size': [3, 224, 224],\n            'input_range': [0, 1],\n            'mean': [0.485, 0.456, 0.406],\n            'std': [0.229, 0.224, 0.225],\n            'num_classes': 1000\n        }\n    },\n    'se_resnext101_32x4d': {\n        'imagenet': {\n            'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnext101_32x4d-3b2fe3d8.pth',\n            'input_space': 'RGB',\n            'input_size': [3, 224, 224],\n            'input_range': [0, 1],\n            'mean': [0.485, 0.456, 0.406],\n            'std': [0.229, 0.224, 0.225],\n            'num_classes': 1000\n        }\n    },\n}\n\n\nclass SEModule(nn.Module):\n\n    def __init__(self, channels, reduction):\n        super(SEModule, self).__init__()\n        self.avg_pool = nn.AdaptiveAvgPool2d(1)\n        self.fc1 = nn.Conv2d(channels, channels // reduction, kernel_size=1,\n                             padding=0)\n        self.relu = nn.ReLU(inplace=True)\n        self.fc2 = nn.Conv2d(channels // reduction, channels, kernel_size=1,\n                             padding=0)\n        self.sigmoid = nn.Sigmoid()\n\n    def forward(self, x):\n        module_input = x\n        x = self.avg_pool(x)\n        x = self.fc1(x)\n        x = self.relu(x)\n        x = self.fc2(x)\n        x = self.sigmoid(x)\n        return module_input * x\n\n\nclass Bottleneck(nn.Module):\n    \"\"\"\n    Base class for bottlenecks that implements `forward()` method.\n    \"\"\"\n    def forward(self, x):\n        residual = x\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = self.relu(out)\n\n        out = self.conv3(out)\n        out = self.bn3(out)\n\n        if self.downsample is not None:\n            residual = self.downsample(x)\n\n        out = self.se_module(out) + residual\n        out = self.relu(out)\n\n        return out\n\n\nclass SEBottleneck(Bottleneck):\n    \"\"\"\n    Bottleneck for SENet154.\n    \"\"\"\n    expansion = 4\n\n    def __init__(self, inplanes, planes, groups, reduction, stride=1,\n                 downsample=None):\n        super(SEBottleneck, self).__init__()\n        self.conv1 = nn.Conv2d(inplanes, planes * 2, kernel_size=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(planes * 2)\n        self.conv2 = nn.Conv2d(planes * 2, planes * 4, kernel_size=3,\n                               stride=stride, padding=1, groups=groups,\n                               bias=False)\n        self.bn2 = nn.BatchNorm2d(planes * 4)\n        self.conv3 = nn.Conv2d(planes * 4, planes * 4, kernel_size=1,\n                               bias=False)\n        self.bn3 = nn.BatchNorm2d(planes * 4)\n        self.relu = nn.ReLU(inplace=True)\n        self.se_module = SEModule(planes * 4, reduction=reduction)\n        self.downsample = downsample\n        self.stride = stride\n\n\nclass SEResNetBottleneck(Bottleneck):\n    \"\"\"\n    ResNet bottleneck with a Squeeze-and-Excitation module. It follows Caffe\n    implementation and uses `stride=stride` in `conv1` and not in `conv2`\n    (the latter is used in the torchvision implementation of ResNet).\n    \"\"\"\n    expansion = 4\n\n    def __init__(self, inplanes, planes, groups, reduction, stride=1,\n                 downsample=None):\n        super(SEResNetBottleneck, self).__init__()\n        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False,\n                               stride=stride)\n        self.bn1 = nn.BatchNorm2d(planes)\n        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, padding=1,\n                               groups=groups, bias=False)\n        self.bn2 = nn.BatchNorm2d(planes)\n        self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)\n        self.bn3 = nn.BatchNorm2d(planes * 4)\n        self.relu = nn.ReLU(inplace=True)\n        self.se_module = SEModule(planes * 4, reduction=reduction)\n        self.downsample = downsample\n        self.stride = stride\n\n\nclass SEResNeXtBottleneck(Bottleneck):\n    \"\"\"\n    ResNeXt bottleneck type C with a Squeeze-and-Excitation module.\n    \"\"\"\n    expansion = 4\n\n    def __init__(self, inplanes, planes, groups, reduction, stride=1,\n                 downsample=None, base_width=4):\n        super(SEResNeXtBottleneck, self).__init__()\n        width = math.floor(planes * (base_width / 64)) * groups\n        self.conv1 = nn.Conv2d(inplanes, width, kernel_size=1, bias=False,\n                               stride=1)\n        self.bn1 = nn.BatchNorm2d(width)\n        self.conv2 = nn.Conv2d(width, width, kernel_size=3, stride=stride,\n                               padding=1, groups=groups, bias=False)\n        self.bn2 = nn.BatchNorm2d(width)\n        self.conv3 = nn.Conv2d(width, planes * 4, kernel_size=1, bias=False)\n        self.bn3 = nn.BatchNorm2d(planes * 4)\n        self.relu = nn.ReLU(inplace=True)\n        self.se_module = SEModule(planes * 4, reduction=reduction)\n        self.downsample = downsample\n        self.stride = stride\n\n\nclass SENet(nn.Module):\n\n    def __init__(self, block, layers, groups, reduction, dropout_p=0.2,\n                 inplanes=128, input_3x3=True, downsample_kernel_size=3,\n                 downsample_padding=1, num_classes=1000):\n        \"\"\"\n        Parameters\n        ----------\n        block (nn.Module): Bottleneck class.\n            - For SENet154: SEBottleneck\n            - For SE-ResNet models: SEResNetBottleneck\n            - For SE-ResNeXt models:  SEResNeXtBottleneck\n        layers (list of ints): Number of residual blocks for 4 layers of the\n            network (layer1...layer4).\n        groups (int): Number of groups for the 3x3 convolution in each\n            bottleneck block.\n            - For SENet154: 64\n            - For SE-ResNet models: 1\n            - For SE-ResNeXt models:  32\n        reduction (int): Reduction ratio for Squeeze-and-Excitation modules.\n            - For all models: 16\n        dropout_p (float or None): Drop probability for the Dropout layer.\n            If `None` the Dropout layer is not used.\n            - For SENet154: 0.2\n            - For SE-ResNet models: None\n            - For SE-ResNeXt models: None\n        inplanes (int):  Number of input channels for layer1.\n            - For SENet154: 128\n            - For SE-ResNet models: 64\n            - For SE-ResNeXt models: 64\n        input_3x3 (bool): If `True`, use three 3x3 convolutions instead of\n            a single 7x7 convolution in layer0.\n            - For SENet154: True\n            - For SE-ResNet models: False\n            - For SE-ResNeXt models: False\n        downsample_kernel_size (int): Kernel size for downsampling convolutions\n            in layer2, layer3 and layer4.\n            - For SENet154: 3\n            - For SE-ResNet models: 1\n            - For SE-ResNeXt models: 1\n        downsample_padding (int): Padding for downsampling convolutions in\n            layer2, layer3 and layer4.\n            - For SENet154: 1\n            - For SE-ResNet models: 0\n            - For SE-ResNeXt models: 0\n        num_classes (int): Number of outputs in `last_linear` layer.\n            - For all models: 1000\n        \"\"\"\n        super(SENet, self).__init__()\n        self.inplanes = inplanes\n        if input_3x3:\n            layer0_modules = [\n                ('conv1', nn.Conv2d(3, 64, 3, stride=2, padding=1,\n                                    bias=False)),\n                ('bn1', nn.BatchNorm2d(64)),\n                ('relu1', nn.ReLU(inplace=True)),\n                ('conv2', nn.Conv2d(64, 64, 3, stride=1, padding=1,\n                                    bias=False)),\n                ('bn2', nn.BatchNorm2d(64)),\n                ('relu2', nn.ReLU(inplace=True)),\n                ('conv3', nn.Conv2d(64, inplanes, 3, stride=1, padding=1,\n                                    bias=False)),\n                ('bn3', nn.BatchNorm2d(inplanes)),\n                ('relu3', nn.ReLU(inplace=True)),\n            ]\n        else:\n            layer0_modules = [\n                ('conv1', nn.Conv2d(3, inplanes, kernel_size=7, stride=2,\n                                    padding=3, bias=False)),\n                ('bn1', nn.BatchNorm2d(inplanes)),\n                ('relu1', nn.ReLU(inplace=True)),\n            ]\n        # To preserve compatibility with Caffe weights `ceil_mode=True`\n        # is used instead of `padding=1`.\n        layer0_modules.append(('pool', nn.MaxPool2d(3, stride=2,\n                                                    ceil_mode=True)))\n        self.layer0 = nn.Sequential(OrderedDict(layer0_modules))\n        self.layer1 = self._make_layer(\n            block,\n            planes=64,\n            blocks=layers[0],\n            groups=groups,\n            reduction=reduction,\n            downsample_kernel_size=1,\n            downsample_padding=0\n        )\n        self.layer2 = self._make_layer(\n            block,\n            planes=128,\n            blocks=layers[1],\n            stride=2,\n            groups=groups,\n            reduction=reduction,\n            downsample_kernel_size=downsample_kernel_size,\n            downsample_padding=downsample_padding\n        )\n        self.layer3 = self._make_layer(\n            block,\n            planes=256,\n            blocks=layers[2],\n            stride=2,\n            groups=groups,\n            reduction=reduction,\n            downsample_kernel_size=downsample_kernel_size,\n            downsample_padding=downsample_padding\n        )\n        self.layer4 = self._make_layer(\n            block,\n            planes=512,\n            blocks=layers[3],\n            stride=2,\n            groups=groups,\n            reduction=reduction,\n            downsample_kernel_size=downsample_kernel_size,\n            downsample_padding=downsample_padding\n        )\n        self.avg_pool = nn.AvgPool2d(7, stride=1)\n        self.dropout = nn.Dropout(dropout_p) if dropout_p is not None else None\n        self.last_linear = nn.Linear(512 * block.expansion, num_classes)\n\n    def _make_layer(self, block, planes, blocks, groups, reduction, stride=1,\n                    downsample_kernel_size=1, downsample_padding=0):\n        downsample = None\n        if stride != 1 or self.inplanes != planes * block.expansion:\n            downsample = nn.Sequential(\n                nn.Conv2d(self.inplanes, planes * block.expansion,\n                          kernel_size=downsample_kernel_size, stride=stride,\n                          padding=downsample_padding, bias=False),\n                nn.BatchNorm2d(planes * block.expansion),\n            )\n\n        layers = []\n        layers.append(block(self.inplanes, planes, groups, reduction, stride,\n                            downsample))\n        self.inplanes = planes * block.expansion\n        for i in range(1, blocks):\n            layers.append(block(self.inplanes, planes, groups, reduction))\n\n        return nn.Sequential(*layers)\n\n    def features(self, x):\n        x = self.layer0(x)\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n        return x\n\n    def logits(self, x):\n        x = self.avg_pool(x)\n        if self.dropout is not None:\n            x = self.dropout(x)\n        x = x.view(x.size(0), -1)\n        x = self.last_linear(x)\n        return x\n\n    def forward(self, x):\n        x = self.features(x)\n        x = self.logits(x)\n        return x\n\n\ndef initialize_pretrained_model(model, num_classes, settings):\n    assert num_classes == settings['num_classes'], \\\n        'num_classes should be {}, but is {}'.format(\n            settings['num_classes'], num_classes)\n    model.load_state_dict(model_zoo.load_url(settings['url']))\n    model.input_space = settings['input_space']\n    model.input_size = settings['input_size']\n    model.input_range = settings['input_range']\n    model.mean = settings['mean']\n    model.std = settings['std']\n\n\ndef senet154(num_classes=1000, pretrained='imagenet'):\n    model = SENet(SEBottleneck, [3, 8, 36, 3], groups=64, reduction=16,\n                  dropout_p=0.2, num_classes=num_classes)\n    if pretrained is not None:\n        settings = pretrained_settings['senet154'][pretrained]\n        initialize_pretrained_model(model, num_classes, settings)\n    return model\n\n\ndef se_resnet50(num_classes=1000, pretrained='imagenet'):\n    model = SENet(SEResNetBottleneck, [3, 4, 6, 3], groups=1, reduction=16,\n                  dropout_p=None, inplanes=64, input_3x3=False,\n                  downsample_kernel_size=1, downsample_padding=0,\n                  num_classes=num_classes)\n    if pretrained is not None:\n        settings = pretrained_settings['se_resnet50'][pretrained]\n        initialize_pretrained_model(model, num_classes, settings)\n    return model\n\n\ndef se_resnet101(num_classes=1000, pretrained='imagenet'):\n    model = SENet(SEResNetBottleneck, [3, 4, 23, 3], groups=1, reduction=16,\n                  dropout_p=None, inplanes=64, input_3x3=False,\n                  downsample_kernel_size=1, downsample_padding=0,\n                  num_classes=num_classes)\n    if pretrained is not None:\n        settings = pretrained_settings['se_resnet101'][pretrained]\n        initialize_pretrained_model(model, num_classes, settings)\n    return model\n\n\ndef se_resnet152(num_classes=1000, pretrained='imagenet'):\n    model = SENet(SEResNetBottleneck, [3, 8, 36, 3], groups=1, reduction=16,\n                  dropout_p=None, inplanes=64, input_3x3=False,\n                  downsample_kernel_size=1, downsample_padding=0,\n                  num_classes=num_classes)\n    if pretrained is not None:\n        settings = pretrained_settings['se_resnet152'][pretrained]\n        initialize_pretrained_model(model, num_classes, settings)\n    return model\n\n\ndef se_resnext50_32x4d(num_classes=1000, pretrained='imagenet'):\n    model = SENet(SEResNeXtBottleneck, [3, 4, 6, 3], groups=32, reduction=16,\n                  dropout_p=None, inplanes=64, input_3x3=False,\n                  downsample_kernel_size=1, downsample_padding=0,\n                  num_classes=num_classes)\n    if pretrained is not None:\n        settings = pretrained_settings['se_resnext50_32x4d'][pretrained]\n        initialize_pretrained_model(model, num_classes, settings)\n    return model\n\n\ndef se_resnext101_32x4d(num_classes=1000, pretrained='imagenet'):\n    model = SENet(SEResNeXtBottleneck, [3, 4, 23, 3], groups=32, reduction=16,\n                  dropout_p=None, inplanes=64, input_3x3=False,\n                  downsample_kernel_size=1, downsample_padding=0,\n                  num_classes=num_classes)\n    if pretrained is not None:\n        settings = pretrained_settings['se_resnext101_32x4d'][pretrained]\n        initialize_pretrained_model(model, num_classes, settings)\n    return model\n\nfrom functools import partial\n\nimport torch\nimport numpy as np\n# import pretrainedmodels\nfrom torch import nn\nfrom torch.nn import functional as F\n\n\nclass Flatten(nn.Module):\n    def forward(self, x):\n        return x.view(x.size(0), -1)\n\n\ndef create_net(net_cls, pretrained: bool):\n    net = net_cls(pretrained=pretrained)\n    return net\n\n\ndef get_head(nf: int, n_classes):\n    model = nn.Sequential(\n        nn.AdaptiveAvgPool2d(1),\n        Flatten(),\n        # nn.Dropout(p=0.25),\n        nn.Linear(nf, n_classes)\n    )\n    return model\n\n\ndef init_weights(model):\n    for i, module in enumerate(model):\n        if isinstance(module, (nn.BatchNorm1d, nn.BatchNorm2d)):\n            if module.weight is not None:\n                nn.init.uniform_(module.weight)\n            if module.bias is not None:\n                nn.init.constant_(module.bias, 0)\n        if isinstance(module, (nn.Linear, nn.Conv2d, nn.Conv1d)):\n            if getattr(module, \"weight_v\", None) is not None:\n                print(\"Initing linear with weight normalization\")\n                assert model[i].weight_g is not None\n            else:\n                nn.init.kaiming_normal_(module.weight)\n                print(\"Initing linear\")\n            if module.bias is not None:\n                nn.init.constant_(module.bias, 0)\n    return model\n\n\ndef get_seresnet_model(arch: str = \"se_resnext50_32x4d\", n_classes: int = 10, pretrained: bool = False):\n    full = se_resnext50_32x4d(\n        pretrained='imagenet' if pretrained else None)\n    model = nn.Sequential(\n        nn.Sequential(full.layer0, full.layer1, full.layer2, full.layer3[:3]),\n        nn.Sequential(full.layer3[3:], full.layer4),\n        get_head(2048, n_classes))\n    print(\" | \".join([\n        \"{:,d}\".format(np.sum([p.numel() for p in x.parameters()])) for x in model]))\n    if pretrained:\n        init_weights(model[-1])\n        return model\n    return init_weights(model)\n\n\nclass Swish(nn.Module):\n    def forward(self, x):\n        \"\"\" Swish activation function \"\"\"\n        return x * torch.sigmoid(x)\n    \n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.model_zoo as model_zoo\nimport os\nimport sys\n\n\npretrained_settings = {\n    'inceptionv4': {\n        'imagenet': {\n            'url': 'http://data.lip6.fr/cadene/pretrainedmodels/inceptionv4-8e4777a0.pth',\n            'input_space': 'RGB',\n            'input_size': [3, 299, 299],\n            'input_range': [0, 1],\n            'mean': [0.5, 0.5, 0.5],\n            'std': [0.5, 0.5, 0.5],\n            'num_classes': 1000\n        },\n        'imagenet+background': {\n            'url': 'http://data.lip6.fr/cadene/pretrainedmodels/inceptionv4-8e4777a0.pth',\n            'input_space': 'RGB',\n            'input_size': [3, 299, 299],\n            'input_range': [0, 1],\n            'mean': [0.5, 0.5, 0.5],\n            'std': [0.5, 0.5, 0.5],\n            'num_classes': 1001\n        }\n    }\n}\n\n\nclass BasicConv2d(nn.Module):\n\n    def __init__(self, in_planes, out_planes, kernel_size, stride, padding=0):\n        super(BasicConv2d, self).__init__()\n        self.conv = nn.Conv2d(in_planes, out_planes,\n                              kernel_size=kernel_size, stride=stride,\n                              padding=padding, bias=False) # verify bias false\n        self.bn = nn.BatchNorm2d(out_planes,\n                                 eps=0.001, # value found in tensorflow\n                                 momentum=0.1, # default pytorch value\n                                 affine=True)\n        self.relu = nn.ReLU(inplace=True)\n\n    def forward(self, x):\n        x = self.conv(x)\n        x = self.bn(x)\n        x = self.relu(x)\n        return x\n\n\nclass Mixed_3a(nn.Module):\n\n    def __init__(self):\n        super(Mixed_3a, self).__init__()\n        self.maxpool = nn.MaxPool2d(3, stride=2)\n        self.conv = BasicConv2d(64, 96, kernel_size=3, stride=2)\n\n    def forward(self, x):\n        x0 = self.maxpool(x)\n        x1 = self.conv(x)\n        out = torch.cat((x0, x1), 1)\n        return out\n\n\nclass Mixed_4a(nn.Module):\n\n    def __init__(self):\n        super(Mixed_4a, self).__init__()\n\n        self.branch0 = nn.Sequential(\n            BasicConv2d(160, 64, kernel_size=1, stride=1),\n            BasicConv2d(64, 96, kernel_size=3, stride=1)\n        )\n\n        self.branch1 = nn.Sequential(\n            BasicConv2d(160, 64, kernel_size=1, stride=1),\n            BasicConv2d(64, 64, kernel_size=(1,7), stride=1, padding=(0,3)),\n            BasicConv2d(64, 64, kernel_size=(7,1), stride=1, padding=(3,0)),\n            BasicConv2d(64, 96, kernel_size=(3,3), stride=1)\n        )\n\n    def forward(self, x):\n        x0 = self.branch0(x)\n        x1 = self.branch1(x)\n        out = torch.cat((x0, x1), 1)\n        return out\n\n\nclass Mixed_5a(nn.Module):\n\n    def __init__(self):\n        super(Mixed_5a, self).__init__()\n        self.conv = BasicConv2d(192, 192, kernel_size=3, stride=2)\n        self.maxpool = nn.MaxPool2d(3, stride=2)\n\n    def forward(self, x):\n        x0 = self.conv(x)\n        x1 = self.maxpool(x)\n        out = torch.cat((x0, x1), 1)\n        return out\n\n\nclass Inception_A(nn.Module):\n\n    def __init__(self):\n        super(Inception_A, self).__init__()\n        self.branch0 = BasicConv2d(384, 96, kernel_size=1, stride=1)\n\n        self.branch1 = nn.Sequential(\n            BasicConv2d(384, 64, kernel_size=1, stride=1),\n            BasicConv2d(64, 96, kernel_size=3, stride=1, padding=1)\n        )\n\n        self.branch2 = nn.Sequential(\n            BasicConv2d(384, 64, kernel_size=1, stride=1),\n            BasicConv2d(64, 96, kernel_size=3, stride=1, padding=1),\n            BasicConv2d(96, 96, kernel_size=3, stride=1, padding=1)\n        )\n\n        self.branch3 = nn.Sequential(\n            nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False),\n            BasicConv2d(384, 96, kernel_size=1, stride=1)\n        )\n\n    def forward(self, x):\n        x0 = self.branch0(x)\n        x1 = self.branch1(x)\n        x2 = self.branch2(x)\n        x3 = self.branch3(x)\n        out = torch.cat((x0, x1, x2, x3), 1)\n        return out\n\n\nclass Reduction_A(nn.Module):\n\n    def __init__(self):\n        super(Reduction_A, self).__init__()\n        self.branch0 = BasicConv2d(384, 384, kernel_size=3, stride=2)\n\n        self.branch1 = nn.Sequential(\n            BasicConv2d(384, 192, kernel_size=1, stride=1),\n            BasicConv2d(192, 224, kernel_size=3, stride=1, padding=1),\n            BasicConv2d(224, 256, kernel_size=3, stride=2)\n        )\n\n        self.branch2 = nn.MaxPool2d(3, stride=2)\n\n    def forward(self, x):\n        x0 = self.branch0(x)\n        x1 = self.branch1(x)\n        x2 = self.branch2(x)\n        out = torch.cat((x0, x1, x2), 1)\n        return out\n\n\nclass Inception_B(nn.Module):\n\n    def __init__(self):\n        super(Inception_B, self).__init__()\n        self.branch0 = BasicConv2d(1024, 384, kernel_size=1, stride=1)\n\n        self.branch1 = nn.Sequential(\n            BasicConv2d(1024, 192, kernel_size=1, stride=1),\n            BasicConv2d(192, 224, kernel_size=(1,7), stride=1, padding=(0,3)),\n            BasicConv2d(224, 256, kernel_size=(7,1), stride=1, padding=(3,0))\n        )\n\n        self.branch2 = nn.Sequential(\n            BasicConv2d(1024, 192, kernel_size=1, stride=1),\n            BasicConv2d(192, 192, kernel_size=(7,1), stride=1, padding=(3,0)),\n            BasicConv2d(192, 224, kernel_size=(1,7), stride=1, padding=(0,3)),\n            BasicConv2d(224, 224, kernel_size=(7,1), stride=1, padding=(3,0)),\n            BasicConv2d(224, 256, kernel_size=(1,7), stride=1, padding=(0,3))\n        )\n\n        self.branch3 = nn.Sequential(\n            nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False),\n            BasicConv2d(1024, 128, kernel_size=1, stride=1)\n        )\n\n    def forward(self, x):\n        x0 = self.branch0(x)\n        x1 = self.branch1(x)\n        x2 = self.branch2(x)\n        x3 = self.branch3(x)\n        out = torch.cat((x0, x1, x2, x3), 1)\n        return out\n\n\nclass Reduction_B(nn.Module):\n\n    def __init__(self):\n        super(Reduction_B, self).__init__()\n\n        self.branch0 = nn.Sequential(\n            BasicConv2d(1024, 192, kernel_size=1, stride=1),\n            BasicConv2d(192, 192, kernel_size=3, stride=2)\n        )\n\n        self.branch1 = nn.Sequential(\n            BasicConv2d(1024, 256, kernel_size=1, stride=1),\n            BasicConv2d(256, 256, kernel_size=(1,7), stride=1, padding=(0,3)),\n            BasicConv2d(256, 320, kernel_size=(7,1), stride=1, padding=(3,0)),\n            BasicConv2d(320, 320, kernel_size=3, stride=2)\n        )\n\n        self.branch2 = nn.MaxPool2d(3, stride=2)\n\n    def forward(self, x):\n        x0 = self.branch0(x)\n        x1 = self.branch1(x)\n        x2 = self.branch2(x)\n        out = torch.cat((x0, x1, x2), 1)\n        return out\n\n\nclass Inception_C(nn.Module):\n\n    def __init__(self):\n        super(Inception_C, self).__init__()\n\n        self.branch0 = BasicConv2d(1536, 256, kernel_size=1, stride=1)\n\n        self.branch1_0 = BasicConv2d(1536, 384, kernel_size=1, stride=1)\n        self.branch1_1a = BasicConv2d(384, 256, kernel_size=(1,3), stride=1, padding=(0,1))\n        self.branch1_1b = BasicConv2d(384, 256, kernel_size=(3,1), stride=1, padding=(1,0))\n\n        self.branch2_0 = BasicConv2d(1536, 384, kernel_size=1, stride=1)\n        self.branch2_1 = BasicConv2d(384, 448, kernel_size=(3,1), stride=1, padding=(1,0))\n        self.branch2_2 = BasicConv2d(448, 512, kernel_size=(1,3), stride=1, padding=(0,1))\n        self.branch2_3a = BasicConv2d(512, 256, kernel_size=(1,3), stride=1, padding=(0,1))\n        self.branch2_3b = BasicConv2d(512, 256, kernel_size=(3,1), stride=1, padding=(1,0))\n\n        self.branch3 = nn.Sequential(\n            nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False),\n            BasicConv2d(1536, 256, kernel_size=1, stride=1)\n        )\n\n    def forward(self, x):\n        x0 = self.branch0(x)\n\n        x1_0 = self.branch1_0(x)\n        x1_1a = self.branch1_1a(x1_0)\n        x1_1b = self.branch1_1b(x1_0)\n        x1 = torch.cat((x1_1a, x1_1b), 1)\n\n        x2_0 = self.branch2_0(x)\n        x2_1 = self.branch2_1(x2_0)\n        x2_2 = self.branch2_2(x2_1)\n        x2_3a = self.branch2_3a(x2_2)\n        x2_3b = self.branch2_3b(x2_2)\n        x2 = torch.cat((x2_3a, x2_3b), 1)\n\n        x3 = self.branch3(x)\n\n        out = torch.cat((x0, x1, x2, x3), 1)\n        return out\n\n\nclass InceptionV4(nn.Module):\n\n    def __init__(self, num_classes=1001):\n        super(InceptionV4, self).__init__()\n        # Special attributs\n        self.input_space = None\n        self.input_size = (299, 299, 3)\n        self.mean = None\n        self.std = None\n        # Modules\n        self.features = nn.Sequential(\n            BasicConv2d(3, 32, kernel_size=3, stride=2),\n            BasicConv2d(32, 32, kernel_size=3, stride=1),\n            BasicConv2d(32, 64, kernel_size=3, stride=1, padding=1),\n            Mixed_3a(),\n            Mixed_4a(),\n            Mixed_5a(),\n            Inception_A(),\n            Inception_A(),\n            Inception_A(),\n            Inception_A(),\n            Reduction_A(), # Mixed_6a\n            Inception_B(),\n            Inception_B(),\n            Inception_B(),\n            Inception_B(),\n            Inception_B(),\n            Inception_B(),\n            Inception_B(),\n            Reduction_B(), # Mixed_7a\n            Inception_C(),\n            Inception_C(),\n            Inception_C()\n        )\n        self.last_linear = nn.Linear(1536, num_classes)\n\n    def logits(self, features):\n        #Allows image of any size to be processed\n        adaptiveAvgPoolWidth = features.shape[2]\n        x = F.avg_pool2d(features, kernel_size=adaptiveAvgPoolWidth)\n        x = x.view(x.size(0), -1)\n        x = self.last_linear(x)\n        return x\n\n    def forward(self, input):\n        x = self.features(input)\n        x = self.logits(x)\n        return x\n\n\ndef inceptionv4(num_classes=1000, pretrained='imagenet'):\n    if pretrained:\n        settings = pretrained_settings['inceptionv4'][pretrained]\n        assert num_classes == settings['num_classes'], \\\n            \"num_classes should be {}, but is {}\".format(settings['num_classes'], num_classes)\n\n        # both 'imagenet'&'imagenet+background' are loaded from same parameters\n        model = InceptionV4(num_classes=1001)\n        model.load_state_dict(model_zoo.load_url(settings['url']))\n\n        if pretrained == 'imagenet':\n            new_last_linear = nn.Linear(1536, 1000)\n            new_last_linear.weight.data = model.last_linear.weight.data[1:]\n            new_last_linear.bias.data = model.last_linear.bias.data[1:]\n            model.last_linear = new_last_linear\n\n        model.input_space = settings['input_space']\n        model.input_size = settings['input_size']\n        model.input_range = settings['input_range']\n        model.mean = settings['mean']\n        model.std = settings['std']\n    else:\n        model = InceptionV4(num_classes=num_classes)\n    return model  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"! ls -la ../input/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nReproduced from https://github.com/DeepVoltaire/AutoAugment\nMIT License\n\"\"\"\n\nfrom PIL import Image, ImageEnhance, ImageOps\nimport numpy as np\nimport random\n\n\nclass ImageNetPolicy(object):\n    \"\"\" Randomly choose one of the best 24 Sub-policies on ImageNet.\n\n        Example:\n        >>> policy = ImageNetPolicy()\n        >>> transformed = policy(image)\n\n        Example as a PyTorch Transform:\n        >>> transform=transforms.Compose([\n        >>>     transforms.Resize(256),\n        >>>     ImageNetPolicy(),\n        >>>     transforms.ToTensor()])\n    \"\"\"\n\n    def __init__(self, fillcolor=(128, 128, 128)):\n        self.policies = [\n            SubPolicy(0.4, \"posterize\", 8, 0.6, \"rotate\", 9, fillcolor),\n            SubPolicy(0.6, \"solarize\", 5, 0.6, \"autocontrast\", 5, fillcolor),\n            SubPolicy(0.8, \"equalize\", 8, 0.6, \"equalize\", 3, fillcolor),\n            SubPolicy(0.6, \"posterize\", 7, 0.6, \"posterize\", 6, fillcolor),\n            SubPolicy(0.4, \"equalize\", 7, 0.2, \"solarize\", 4, fillcolor),\n\n            SubPolicy(0.4, \"equalize\", 4, 0.8, \"rotate\", 8, fillcolor),\n            SubPolicy(0.6, \"solarize\", 3, 0.6, \"equalize\", 7, fillcolor),\n            SubPolicy(0.8, \"posterize\", 5, 1.0, \"equalize\", 2, fillcolor),\n            SubPolicy(0.2, \"rotate\", 3, 0.6, \"solarize\", 8, fillcolor),\n            SubPolicy(0.6, \"equalize\", 8, 0.4, \"posterize\", 6, fillcolor),\n\n            SubPolicy(0.8, \"rotate\", 8, 0.4, \"color\", 0, fillcolor),\n            SubPolicy(0.4, \"rotate\", 9, 0.6, \"equalize\", 2, fillcolor),\n            SubPolicy(0.0, \"equalize\", 7, 0.8, \"equalize\", 8, fillcolor),\n            SubPolicy(0.6, \"invert\", 4, 1.0, \"equalize\", 8, fillcolor),\n            SubPolicy(0.6, \"color\", 4, 1.0, \"contrast\", 8, fillcolor),\n\n            SubPolicy(0.8, \"rotate\", 8, 1.0, \"color\", 2, fillcolor),\n            SubPolicy(0.8, \"color\", 8, 0.8, \"solarize\", 7, fillcolor),\n            SubPolicy(0.4, \"sharpness\", 7, 0.6, \"invert\", 8, fillcolor),\n            SubPolicy(0.6, \"shearX\", 5, 1.0, \"equalize\", 9, fillcolor),\n            SubPolicy(0.4, \"color\", 0, 0.6, \"equalize\", 3, fillcolor),\n\n            SubPolicy(0.4, \"equalize\", 7, 0.2, \"solarize\", 4, fillcolor),\n            SubPolicy(0.6, \"solarize\", 5, 0.6, \"autocontrast\", 5, fillcolor),\n            SubPolicy(0.6, \"invert\", 4, 1.0, \"equalize\", 8, fillcolor),\n            SubPolicy(0.6, \"color\", 4, 1.0, \"contrast\", 8, fillcolor)\n        ]\n\n    def __call__(self, img):\n        policy_idx = random.randint(0, len(self.policies) - 1)\n        return self.policies[policy_idx](img)\n\n    def __repr__(self):\n        return \"AutoAugment ImageNet Policy\"\n\n\nclass CIFAR10Policy(object):\n    \"\"\" Randomly choose one of the best 25 Sub-policies on CIFAR10.\n\n        Example:\n        >>> policy = CIFAR10Policy()\n        >>> transformed = policy(image)\n\n        Example as a PyTorch Transform:\n        >>> transform=transforms.Compose([\n        >>>     transforms.Resize(256),\n        >>>     CIFAR10Policy(),\n        >>>     transforms.ToTensor()])\n    \"\"\"\n\n    def __init__(self, fillcolor=(128, 128, 128)):\n        self.policies = [\n            SubPolicy(0.1, \"invert\", 7, 0.2, \"contrast\", 6, fillcolor),\n            SubPolicy(0.7, \"rotate\", 2, 0.3, \"translateX\", 9, fillcolor),\n            SubPolicy(0.8, \"sharpness\", 1, 0.9, \"sharpness\", 3, fillcolor),\n            SubPolicy(0.5, \"shearY\", 8, 0.7, \"translateY\", 9, fillcolor),\n            SubPolicy(0.5, \"autocontrast\", 8, 0.9, \"equalize\", 2, fillcolor),\n\n            SubPolicy(0.2, \"shearY\", 7, 0.3, \"posterize\", 7, fillcolor),\n            SubPolicy(0.4, \"color\", 3, 0.6, \"brightness\", 7, fillcolor),\n            SubPolicy(0.3, \"sharpness\", 9, 0.7, \"brightness\", 9, fillcolor),\n            SubPolicy(0.6, \"equalize\", 5, 0.5, \"equalize\", 1, fillcolor),\n            SubPolicy(0.6, \"contrast\", 7, 0.6, \"sharpness\", 5, fillcolor),\n\n            SubPolicy(0.7, \"color\", 7, 0.5, \"translateX\", 8, fillcolor),\n            SubPolicy(0.3, \"equalize\", 7, 0.4, \"autocontrast\", 8, fillcolor),\n            SubPolicy(0.4, \"translateY\", 3, 0.2, \"sharpness\", 6, fillcolor),\n            SubPolicy(0.9, \"brightness\", 6, 0.2, \"color\", 8, fillcolor),\n            SubPolicy(0.5, \"solarize\", 2, 0.0, \"invert\", 3, fillcolor),\n\n            SubPolicy(0.2, \"equalize\", 0, 0.6, \"autocontrast\", 0, fillcolor),\n            SubPolicy(0.2, \"equalize\", 8, 0.8, \"equalize\", 4, fillcolor),\n            SubPolicy(0.9, \"color\", 9, 0.6, \"equalize\", 6, fillcolor),\n            SubPolicy(0.8, \"autocontrast\", 4, 0.2, \"solarize\", 8, fillcolor),\n            SubPolicy(0.1, \"brightness\", 3, 0.7, \"color\", 0, fillcolor),\n\n            SubPolicy(0.4, \"solarize\", 5, 0.9, \"autocontrast\", 3, fillcolor),\n            SubPolicy(0.9, \"translateY\", 9, 0.7, \"translateY\", 9, fillcolor),\n            SubPolicy(0.9, \"autocontrast\", 2, 0.8, \"solarize\", 3, fillcolor),\n            SubPolicy(0.8, \"equalize\", 8, 0.1, \"invert\", 3, fillcolor),\n            SubPolicy(0.7, \"translateY\", 9, 0.9, \"autocontrast\", 1, fillcolor)\n        ]\n\n    def __call__(self, img):\n        policy_idx = random.randint(0, len(self.policies) - 1)\n        return self.policies[policy_idx](img)\n\n    def __repr__(self):\n        return \"AutoAugment CIFAR10 Policy\"\n\n\nclass SVHNPolicy(object):\n    \"\"\" Randomly choose one of the best 25 Sub-policies on SVHN.\n\n        Example:\n        >>> policy = SVHNPolicy()\n        >>> transformed = policy(image)\n\n        Example as a PyTorch Transform:\n        >>> transform=transforms.Compose([\n        >>>     transforms.Resize(256),\n        >>>     SVHNPolicy(),\n        >>>     transforms.ToTensor()])\n    \"\"\"\n\n    def __init__(self, fillcolor=(128, 128, 128)):\n        self.policies = [\n            SubPolicy(0.9, \"shearX\", 4, 0.2, \"invert\", 3, fillcolor),\n            SubPolicy(0.9, \"shearY\", 8, 0.7, \"invert\", 5, fillcolor),\n            SubPolicy(0.6, \"equalize\", 5, 0.6, \"solarize\", 6, fillcolor),\n            SubPolicy(0.9, \"invert\", 3, 0.6, \"equalize\", 3, fillcolor),\n            SubPolicy(0.6, \"equalize\", 1, 0.9, \"rotate\", 3, fillcolor),\n\n            SubPolicy(0.9, \"shearX\", 4, 0.8, \"autocontrast\", 3, fillcolor),\n            SubPolicy(0.9, \"shearY\", 8, 0.4, \"invert\", 5, fillcolor),\n            SubPolicy(0.9, \"shearY\", 5, 0.2, \"solarize\", 6, fillcolor),\n            SubPolicy(0.9, \"invert\", 6, 0.8, \"autocontrast\", 1, fillcolor),\n            SubPolicy(0.6, \"equalize\", 3, 0.9, \"rotate\", 3, fillcolor),\n\n            SubPolicy(0.9, \"shearX\", 4, 0.3, \"solarize\", 3, fillcolor),\n            SubPolicy(0.8, \"shearY\", 8, 0.7, \"invert\", 4, fillcolor),\n            SubPolicy(0.9, \"equalize\", 5, 0.6, \"translateY\", 6, fillcolor),\n            SubPolicy(0.9, \"invert\", 4, 0.6, \"equalize\", 7, fillcolor),\n            SubPolicy(0.3, \"contrast\", 3, 0.8, \"rotate\", 4, fillcolor),\n\n            SubPolicy(0.8, \"invert\", 5, 0.0, \"translateY\", 2, fillcolor),\n            SubPolicy(0.7, \"shearY\", 6, 0.4, \"solarize\", 8, fillcolor),\n            SubPolicy(0.6, \"invert\", 4, 0.8, \"rotate\", 4, fillcolor),\n            SubPolicy(0.3, \"shearY\", 7, 0.9, \"translateX\", 3, fillcolor),\n            SubPolicy(0.1, \"shearX\", 6, 0.6, \"invert\", 5, fillcolor),\n\n            SubPolicy(0.7, \"solarize\", 2, 0.6, \"translateY\", 7, fillcolor),\n            SubPolicy(0.8, \"shearY\", 4, 0.8, \"invert\", 8, fillcolor),\n            SubPolicy(0.7, \"shearX\", 9, 0.8, \"translateY\", 3, fillcolor),\n            SubPolicy(0.8, \"shearY\", 5, 0.7, \"autocontrast\", 3, fillcolor),\n            SubPolicy(0.7, \"shearX\", 2, 0.1, \"invert\", 5, fillcolor)\n        ]\n\n    def __call__(self, img):\n        policy_idx = random.randint(0, len(self.policies) - 1)\n        return self.policies[policy_idx](img)\n\n    def __repr__(self):\n        return \"AutoAugment SVHN Policy\"\n\n\nclass SubPolicy(object):\n    def __init__(self, p1, operation1, magnitude_idx1, p2, operation2, magnitude_idx2, fillcolor=(128, 128, 128)):\n        ranges = {\n            \"shearX\": np.linspace(0, 0.3, 10),\n            \"shearY\": np.linspace(0, 0.3, 10),\n            \"translateX\": np.linspace(0, 150 / 331, 10),\n            \"translateY\": np.linspace(0, 150 / 331, 10),\n            \"rotate\": np.linspace(0, 30, 10),\n            \"color\": np.linspace(0.0, 0.9, 10),\n            \"posterize\": np.round(np.linspace(8, 4, 10), 0).astype(np.int),\n            \"solarize\": np.linspace(256, 0, 10),\n            \"contrast\": np.linspace(0.0, 0.9, 10),\n            \"sharpness\": np.linspace(0.0, 0.9, 10),\n            \"brightness\": np.linspace(0.0, 0.9, 10),\n            \"autocontrast\": [0] * 10,\n            \"equalize\": [0] * 10,\n            \"invert\": [0] * 10\n        }\n\n        # from https://stackoverflow.com/questions/5252170/specify-image-filling-color-when-rotating-in-python-with-pil-and-setting-expand\n        def rotate_with_fill(img, magnitude):\n            rot = img.convert(\"RGBA\").rotate(magnitude)\n            return Image.composite(rot, Image.new(\"RGBA\", rot.size, (128,) * 4), rot).convert(img.mode)\n\n        func = {\n            \"shearX\": lambda img, magnitude: img.transform(\n                img.size, Image.AFFINE, (1, magnitude * random.choice([-1, 1]), 0, 0, 1, 0),\n                Image.BICUBIC, fillcolor=fillcolor),\n            \"shearY\": lambda img, magnitude: img.transform(\n                img.size, Image.AFFINE, (1, 0, 0, magnitude * random.choice([-1, 1]), 1, 0),\n                Image.BICUBIC, fillcolor=fillcolor),\n            \"translateX\": lambda img, magnitude: img.transform(\n                img.size, Image.AFFINE, (1, 0, magnitude * img.size[0] * random.choice([-1, 1]), 0, 1, 0),\n                fillcolor=fillcolor),\n            \"translateY\": lambda img, magnitude: img.transform(\n                img.size, Image.AFFINE, (1, 0, 0, 0, 1, magnitude * img.size[1] * random.choice([-1, 1])),\n                fillcolor=fillcolor),\n            \"rotate\": lambda img, magnitude: rotate_with_fill(img, magnitude),\n            # \"rotate\": lambda img, magnitude: img.rotate(magnitude * random.choice([-1, 1])),\n            \"color\": lambda img, magnitude: ImageEnhance.Color(img).enhance(1 + magnitude * random.choice([-1, 1])),\n            \"posterize\": lambda img, magnitude: ImageOps.posterize(img, magnitude),\n            \"solarize\": lambda img, magnitude: ImageOps.solarize(img, magnitude),\n            \"contrast\": lambda img, magnitude: ImageEnhance.Contrast(img).enhance(\n                1 + magnitude * random.choice([-1, 1])),\n            \"sharpness\": lambda img, magnitude: ImageEnhance.Sharpness(img).enhance(\n                1 + magnitude * random.choice([-1, 1])),\n            \"brightness\": lambda img, magnitude: ImageEnhance.Brightness(img).enhance(\n                1 + magnitude * random.choice([-1, 1])),\n            \"autocontrast\": lambda img, magnitude: ImageOps.autocontrast(img),\n            \"equalize\": lambda img, magnitude: ImageOps.equalize(img),\n            \"invert\": lambda img, magnitude: ImageOps.invert(img)\n        }\n\n        # self.name = \"{}_{:.2f}_and_{}_{:.2f}\".format(\n        #     operation1, ranges[operation1][magnitude_idx1],\n        #     operation2, ranges[operation2][magnitude_idx2])\n        self.p1 = p1\n        self.operation1 = func[operation1]\n        self.magnitude1 = ranges[operation1][magnitude_idx1]\n        self.p2 = p2\n        self.operation2 = func[operation2]\n        self.magnitude2 = ranges[operation2][magnitude_idx2]\n\n    def __call__(self, img):\n        if random.random() < self.p1: img = self.operation1(img, self.magnitude1)\n        if random.random() < self.p2: img = self.operation2(img, self.magnitude2)\n        return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Write a function that loads a checkpoint and rebuilds the model\n\nImageFile.LOAD_TRUNCATED_IMAGES = True\nimport pandas as pd\nimport cv2\n\ndef crop_image1(img, tol=7):\n    # img is image data\n    # tol  is tolerance\n\n    mask = img > tol\n    return img[np.ix_(mask.any(1), mask.any(0))]\n\n\ndef crop_image_from_gray(img, tol=7):\n    if img.ndim == 2:\n        mask = img > tol\n        return img[np.ix_(mask.any(1), mask.any(0))]\n    elif img.ndim == 3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img > tol\n\n        check_shape = img[:, :, 0][np.ix_(mask.any(1), mask.any(0))].shape[0]\n        if (check_shape == 0):  # image is too dark so that we crop out everything,\n            return img  # return original image\n        else:\n            img1 = img[:, :, 0][np.ix_(mask.any(1), mask.any(0))]\n            img2 = img[:, :, 1][np.ix_(mask.any(1), mask.any(0))]\n            img3 = img[:, :, 2][np.ix_(mask.any(1), mask.any(0))]\n            #         print(img1.shape,img2.shape,img3.shape)\n            img = np.stack([img1, img2, img3], axis=-1)\n        #         print(img.shape)\n        return img\n\n    \ndef load_checkpoint(filepath, model_name):\n    checkpoint = torch.load(filepath)\n#     print (checkpoint)\n    model=None\n   \n    if model_name == 'seresnext50':\n        model = get_seresnet_model(arch=\"se_resnext50_32x4d\",n_classes=len(classes), pretrained=False)\n    \n    elif model_name == 'seresnext101':\n        model = get_seresnet_model(arch=\"se_resnext101_32x4d\",n_classes=len(classes), pretrained=False)\n#         print (model)\n    \n    elif model_name == 'senet154':\n        model = senet154(num_classes=1000, pretrained=None)\n        model.last_linear = nn.Linear(model.last_linear.in_features, len(classes))   \n\n        \n    model = torch.nn.DataParallel(model, device_ids=list(range(1)))\n    model = model.cuda()                    \n    model.load_state_dict(checkpoint['state_dict'], strict=False)    \n    return model, checkpoint['class_to_idx'], checkpoint['idx_to_class']\n\n# Get index to class mapping\nloaded_model1, class_to_idx, idx_to_class = load_checkpoint('../input/pth-best/seresnext50_MixUpSoftmaxLoss_0.8571428571428571','seresnext50')\nloaded_model2, class_to_idx, idx_to_class = load_checkpoint('../input/pth-best/seresnext50_MixUpSoftmaxLoss_0.8454545454545455','seresnext50')\nloaded_model3, class_to_idx, idx_to_class = load_checkpoint('../input/pth-best/seresnext50_MixUpSoftmaxLoss_0.8487394957983193','seresnext50')\n\n# loaded_model3, class_to_idx, idx_to_class = load_checkpoint('../input/pth-best/seresnext50_MixUpSoftmaxLoss_0.8571428571428571','seresnext50')\n# loaded_model4, class_to_idx, idx_to_class = load_checkpoint('../input/pth-best/seresnext50_MixUpSoftmaxLoss_0.8487394957983193','seresnext50')\n\nmdls = [loaded_model1, loaded_model2, loaded_model3]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm \nimport pandas as pd \n# from torchvision.transforms import *\n\nfrom PIL import Image, ImageDraw\nimport numpy as np\nimport torch\n\nimport torchvision\nimport random\nfrom PIL import Image, ImageOps\nimport numpy as np\nimport numbers\nimport math\nimport torch\nimport torch\nimport random\nimport PIL.ImageEnhance as ie\nimport PIL.Image as im\nimport PIL \n\nimport cv2\nfrom PIL import ImageFile\nImageFile.LOAD_TRUNCATED_IMAGES = True\nimport torchvision\nfrom torchvision import transforms, datasets, models\nimport numpy \n\nAUG_IMG_SIZE=512 \n\ndef david(image_path):    \n    image = Image.open(image_path).convert('RGB')\n    image = cv2.cvtColor(numpy.array (image), cv2.COLOR_BGR2RGB)\n    image = crop_image_from_gray(image)\n    image = cv2.resize(image, (AUG_IMG_SIZE, AUG_IMG_SIZE))\n    image = cv2.addWeighted(image, 4, cv2.GaussianBlur(image, (0, 0), 30), -4, 128)\n\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    image = Image.fromarray(image)\n    return image \n\ndef nodavid(image_path):    \n    image = Image.open(image_path).convert('RGB')    \n    return image \n\n        \n# pil \ntensor_transform_norm = transforms.Compose([ \n            transforms.ToTensor(),\n            transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),\n])\n\ntest_transform_tsr= transforms.Compose([\n        transforms.Resize((AUG_IMG_SIZE,AUG_IMG_SIZE),interpolation=Image.NEAREST),\n        transforms.CenterCrop(AUG_IMG_SIZE), \n        tensor_transform_norm\n    ])    \n\ntrain_transform_tsr = transforms.Compose([\n    #         transforms.ToPILImage(),\n    transforms.Resize((AUG_IMG_SIZE, AUG_IMG_SIZE), interpolation=Image.NEAREST),\n    transforms.CenterCrop(AUG_IMG_SIZE),        \n    transforms.RandomApply([                \n        ImageNetPolicy(),\n        transforms.RandomHorizontalFlip(),\n        transforms.RandomRotation(20, resample=PIL.Image.BICUBIC),\n        transforms.RandomAffine(0, translate=(\n            0.2, 0.2), resample=PIL.Image.BICUBIC),\n        transforms.RandomAffine(0, shear=20, resample=PIL.Image.BICUBIC),\n        transforms.RandomAffine(0, scale=(0.8, 1.2),\n                                resample=PIL.Image.BICUBIC)\n    ],p=0.85),\n    tensor_transform_norm,\n])\n\nSEED=620402\n\ndef seed_everything(seed=SEED):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(int(seed))\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n#     torch.backends.cudnn.deterministic = True\n\ndef predictOne(image_path, topk=1): \n    preds = []\n        \n    image=david(image_path)    \n    image_orig=image               \n    image = test_transform_tsr(image_orig)\n    image = image.unsqueeze(0)    \n    image.cuda()    \n    \n    for m in mdls:\n        preds.append(m.forward(Variable(image)).data.cpu().numpy()[0])                 \n    \n    num_tta = 5    \n    for tta in range(num_tta):        \n        seed_everything (SEED+102*tta) \n        image = train_transform_tsr(image_orig)\n        image = image.unsqueeze(0)    \n        image.cuda()\n        for m in mdls:\n            preds.append(m.forward(Variable(image)).data.cpu().numpy()[0])\n\n    preds = numpy.mean(numpy.array(preds),axis=0)\n    pobabilities = numpy.exp(preds)   \n#     pobabilities = torch.exp(output).data.cpu().numpy()[0]    \n    top_idx = np.argsort(pobabilities)[-topk:][::-1] \n    top_class = [idx_to_class[x] for x in top_idx]\n    top_probability = pobabilities[top_idx]\n    return top_class[0]\n\ndef testModel(test_dir):        \n    for m in mdls:                \n        m.cuda()    \n        m.eval()        \n    \n    topk=1\n    \n    columns = ['id_code', 'diagnosis']\n    df_pred = pd.DataFrame(data=np.zeros((0, len(columns))), columns=columns)\n\n    for index, row in tqdm(sample_submission.iterrows()):\n\n        currImage=os.path.join(test_dir, row['id_code'])\n        currImage=currImage + \".png\"\n        if os.path.isfile(currImage):  \n            with torch.no_grad():\n                df_pred = df_pred.append({'id_code': row['id_code'], 'diagnosis': predictOne (currImage)}, ignore_index=True)    \n    return df_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nsample_submission = pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')\nsample_submission.columns = ['id_code', 'diagnosis']\nsample_submission.head(3)\n\nsubmission_2 = testModel(test_dir)\nsubmission_2.to_csv(\"submission_2.csv\", columns=('id_code', 'diagnosis'), index=None)  ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}