{"cells":[{"metadata":{},"cell_type":"markdown","source":"voting from data\n\n\n\n1.https://www.kaggle.com/chanhu/eye-inference-num-class-1-ver3 \n\n2.https://www.kaggle.com/abhishek/pytorch-inference-kernel-lazy-tta\n\n3.https://www.kaggle.com/ratan123/aptos-2019-keras-baseline\n\n4.https://www.kaggle.com/drhabib/starter-kernel-for-0-79\n\n5.https://www.kaggle.com/filemide/xception-0-757/output?scriptVersionId=18755249\n\nthanks for sharing"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"1.https://www.kaggle.com/chanhu/eye-inference-num-class-1-ver3 "},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\nimport matplotlib.pyplot as plt\nfrom os.path import isfile\nimport torch.nn.init as init\nimport torch\nimport torch.nn as nn\nfrom PIL import Image, ImageFilter\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom torch.utils.data import Dataset\nfrom torchvision import transforms\nfrom torch.optim import Adam, SGD, RMSprop\nimport time\nfrom torch.autograd import Variable\nimport torch.functional as F\nfrom tqdm import tqdm\nfrom sklearn import metrics\nimport urllib\nimport pickle\nimport cv2\nimport torch.nn.functional as F\nfrom torchvision import models\nimport seaborn as sns\nimport random\nimport sys","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"package_path = '../input/efficientnet/efficientnet-pytorch/EfficientNet-PyTorch/'\nsys.path.append(package_path)\nfrom efficientnet_pytorch import EfficientNet\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nseed_everything(1234)\nTTA         = 5\nnum_classes = 1\nIMG_SIZE    = 256\ntest = '../input/aptos2019-blindness-detection/test_images/'\ndef expand_path(p):\n    p = str(p)\n    if isfile(test + p + \".png\"):\n        return test + (p + \".png\")\n    return p\n\ndef p_show(imgs, label_name=None, per_row=3):\n    n = len(imgs)\n    rows = (n + per_row - 1)//per_row\n    cols = min(per_row, n)\n    fig, axes = plt.subplots(rows,cols, figsize=(15,15))\n    for ax in axes.flatten(): ax.axis('off')\n    for i,(p, ax) in enumerate(zip(imgs, axes.flatten())): \n        img = Image.open(expand_path(p))\n        ax.imshow(img)\n        ax.set_title(train_df[train_df.id_code == p].diagnosis.values)\ndef crop_image1(img,tol=7):\n    # img is image data\n    # tol  is tolerance\n        \n    mask = img>tol\n    return img[np.ix_(mask.any(1),mask.any(0))]\n\ndef crop_image_from_gray(img,tol=7):\n    if img.ndim ==2:\n        mask = img>tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    elif img.ndim==3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img>tol\n        \n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0): # image is too dark so that we crop out everything,\n            return img # return original image\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n    #         print(img1.shape,img2.shape,img3.shape)\n            img = np.stack([img1,img2,img3],axis=-1)\n    #         print(img.shape)\n        return img\nclass MyDataset(Dataset):\n    \n    def __init__(self, dataframe, transform=None):\n        self.df = dataframe\n        self.transform = transform\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        \n        label = self.df.diagnosis.values[idx]\n        label = np.expand_dims(label, -1)\n        \n        p = self.df.id_code.values[idx]\n        p_path = expand_path(p)\n        image = cv2.imread(p_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image = crop_image_from_gray(image)\n        image = cv2.resize(image, (IMG_SIZE, IMG_SIZE))\n        image = cv2.addWeighted ( image,4, cv2.GaussianBlur( image , (0,0) , 30) ,-4 ,128)\n        image = transforms.ToPILImage()(image)\n        \n        if self.transform:\n            image = self.transform(image)\n        \n        return image, label\ntest_transform = transforms.Compose([\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomRotation((-120, 120)),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])\n\ntestset        = MyDataset(pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv'), \n                 transform=test_transform)\ntest_loader    = torch.utils.data.DataLoader(testset, batch_size=16, shuffle=False)\nmodel = EfficientNet.from_name('efficientnet-b0')\nin_features = model._fc.in_features\nmodel._fc = nn.Linear(in_features, num_classes)\nmodel.load_state_dict(torch.load('../input/enet-test/weight_best(3).pt'))\nmodel.cuda()\nfor param in model.parameters():\n    param.requires_grad = False\nsample = pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')\ntest_pred = np.zeros((len(sample), 1))\nmodel.eval()\n\nfor _ in range(TTA):\n    with torch.no_grad():\n        for i, data in tqdm(enumerate(test_loader)):\n            images, _ = data\n            images = images.cuda()\n            pred = model(images)\n            test_pred[i * 16:(i + 1) * 16] += pred.detach().cpu().squeeze().numpy().reshape(-1, 1)\n\noutput = test_pred / TTA\npreds1 = output.copy()\ncoef = [0.57, 1.37, 2.57, 3.57]\nfor i, pred in enumerate(output):\n    if pred < coef[0]:\n        output[i] = 0\n    elif pred >= coef[0] and pred < coef[1]:\n        output[i] = 1\n    elif pred >= coef[1] and pred < coef[2]:\n        output[i] = 2\n    elif pred >= coef[2] and pred < coef[3]:\n        output[i] = 3\n    else:\n        output[i] = 4\nsubmission1 = pd.DataFrame({'id_code':pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv').id_code.values,\n                          'diagnosis':np.squeeze(output).astype(int)})","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"2.https://www.kaggle.com/abhishek/pytorch-inference-kernel-lazy-tta"},{"metadata":{"trusted":true},"cell_type":"code","source":"package_dir = \"../input/pretrained-models/pretrained-models/pretrained-models.pytorch-master/\"\nsys.path.insert(0, package_dir)\nimport torchvision\nimport torch.nn as nn\nfrom tqdm import tqdm\nfrom PIL import Image, ImageFile\nfrom torch.utils.data import Dataset\nimport torch\nfrom torchvision import transforms\nimport pretrainedmodels\n\ndevice = torch.device(\"cuda:0\")\nImageFile.LOAD_TRUNCATED_IMAGES = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RetinopathyDatasetTest(Dataset):\n    def __init__(self, csv_file, transform):\n        self.data = pd.read_csv(csv_file)\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        img_name = os.path.join('../input/aptos2019-blindness-detection/test_images', self.data.loc[idx, 'id_code'] + '.png')\n        image = Image.open(img_name)\n        image = self.transform(image)\n        return {'image': image}\nmodel = pretrainedmodels.__dict__['resnet101'](pretrained=None)\n\nmodel.avg_pool = nn.AdaptiveAvgPool2d(1)\nmodel.last_linear = nn.Sequential(\n                          nn.BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n                          nn.Dropout(p=0.25),\n                          nn.Linear(in_features=2048, out_features=2048, bias=True),\n                          nn.ReLU(),\n                          nn.BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n                          nn.Dropout(p=0.5),\n                          nn.Linear(in_features=2048, out_features=1, bias=True),\n                         )\nmodel.load_state_dict(torch.load(\"../input/mmmodel/model.bin\"))\nmodel = model.to(device)\nfor param in model.parameters():\n    param.requires_grad = False\n\nmodel.eval()\n\ntest_transform = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.RandomHorizontalFlip(),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n])\ntest_dataset = RetinopathyDatasetTest(csv_file='../input/aptos2019-blindness-detection/sample_submission.csv',\n                                      transform=test_transform)\ntest_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)\ntest_preds1 = np.zeros((len(test_dataset), 1))\ntk0 = tqdm(test_data_loader)\nfor i, x_batch in enumerate(tk0):\n    x_batch = x_batch[\"image\"]\n    pred = model(x_batch.to(device))\n    test_preds1[i * 32:(i + 1) * 32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)\ntest_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)\ntest_preds2 = np.zeros((len(test_dataset), 1))\ntk0 = tqdm(test_data_loader)\nfor i, x_batch in enumerate(tk0):\n    x_batch = x_batch[\"image\"]\n    pred = model(x_batch.to(device))\n    test_preds2[i * 32:(i + 1) * 32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)\ntest_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)\ntest_preds3 = np.zeros((len(test_dataset), 1))\ntk0 = tqdm(test_data_loader)\nfor i, x_batch in enumerate(tk0):\n    x_batch = x_batch[\"image\"]\n    pred = model(x_batch.to(device))\n    test_preds3[i * 32:(i + 1) * 32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)\ntest_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)\ntest_preds4 = np.zeros((len(test_dataset), 1))\ntk0 = tqdm(test_data_loader)\nfor i, x_batch in enumerate(tk0):\n    x_batch = x_batch[\"image\"]\n    pred = model(x_batch.to(device))\n    test_preds4[i * 32:(i + 1) * 32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)\ntest_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)\ntest_preds5 = np.zeros((len(test_dataset), 1))\ntk0 = tqdm(test_data_loader)\nfor i, x_batch in enumerate(tk0):\n    x_batch = x_batch[\"image\"]\n    pred = model(x_batch.to(device))\n    test_preds5[i * 32:(i + 1) * 32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)\ntest_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)\ntest_preds6 = np.zeros((len(test_dataset), 1))\ntk0 = tqdm(test_data_loader)\nfor i, x_batch in enumerate(tk0):\n    x_batch = x_batch[\"image\"]\n    pred = model(x_batch.to(device))\n    test_preds6[i * 32:(i + 1) * 32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)\ntest_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)\ntest_preds7 = np.zeros((len(test_dataset), 1))\ntk0 = tqdm(test_data_loader)\nfor i, x_batch in enumerate(tk0):\n    x_batch = x_batch[\"image\"]\n    pred = model(x_batch.to(device))\n    test_preds7[i * 32:(i + 1) * 32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)\ntest_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)\ntest_preds8 = np.zeros((len(test_dataset), 1))\ntk0 = tqdm(test_data_loader)\nfor i, x_batch in enumerate(tk0):\n    x_batch = x_batch[\"image\"]\n    pred = model(x_batch.to(device))\n    test_preds8[i * 32:(i + 1) * 32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)\ntest_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)\ntest_preds9 = np.zeros((len(test_dataset), 1))\ntk0 = tqdm(test_data_loader)\nfor i, x_batch in enumerate(tk0):\n    x_batch = x_batch[\"image\"]\n    pred = model(x_batch.to(device))\n    test_preds9[i * 32:(i + 1) * 32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)\ntest_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)\ntest_preds10 = np.zeros((len(test_dataset), 1))\ntk0 = tqdm(test_data_loader)\nfor i, x_batch in enumerate(tk0):\n    x_batch = x_batch[\"image\"]\n    pred = model(x_batch.to(device))\n    test_preds10[i * 32:(i + 1) * 32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)\ntest_preds = (test_preds1 + test_preds2 + test_preds3 + test_preds4 + test_preds5\n             + test_preds6 + test_preds7 + test_preds8 + test_preds9 + test_preds10) / 10.0\ncoef = [0.5, 1.5, 2.5, 3.5]\n\npreds2 = test_preds.copy()\nfor i, pred in enumerate(test_preds):\n    if pred < coef[0]:\n        test_preds[i] = 0\n    elif pred >= coef[0] and pred < coef[1]:\n        test_preds[i] = 1\n    elif pred >= coef[1] and pred < coef[2]:\n        test_preds[i] = 2\n    elif pred >= coef[2] and pred < coef[3]:\n        test_preds[i] = 3\n    else:\n        test_preds[i] = 4\nsubmission2 = pd.read_csv(\"../input/aptos2019-blindness-detection/sample_submission.csv\")\nsubmission2.diagnosis = test_preds.astype(int)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"3.https://www.kaggle.com/ratan123/aptos-2019-keras-baseline"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Sequential, load_model\nfrom keras.layers import (Activation, Dropout, Flatten, Dense, GlobalMaxPooling2D,\n                          BatchNormalization, Input, Conv2D, GlobalAveragePooling2D,concatenate,Concatenate)\nfrom keras.callbacks import ModelCheckpoint\nfrom keras import metrics\nfrom keras.optimizers import Adam \nfrom keras import backend as K\nimport keras\nfrom keras.models import Model\nimport matplotlib.pyplot as plt\nimport skimage.io\nfrom skimage.transform import resize\nfrom imgaug import augmenters as iaa\nfrom tqdm import tqdm\nimport PIL\nfrom PIL import Image, ImageOps\nimport cv2\nfrom sklearn.utils import class_weight, shuffle\nfrom keras.losses import binary_crossentropy, categorical_crossentropy\n#from keras.applications.resnet50 import preprocess_input\nfrom keras.applications.densenet import DenseNet121,DenseNet169\nimport keras.backend as K\nimport tensorflow as tf\nfrom sklearn.metrics import f1_score, fbeta_score, cohen_kappa_score\nfrom keras.utils import Sequence\nfrom keras.utils import to_categorical\nfrom sklearn.model_selection import train_test_split\nimport imgaug as ia\n\nWORKERS = 2\nCHANNEL = 3\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nSIZE = 300\nNUM_CLASSES = 5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\ndf_test = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')\nx = df_train['id_code']\ny = df_train['diagnosis']\n\nx, y = shuffle(x, y, random_state=8)\ny = to_categorical(y, num_classes=NUM_CLASSES)\ntrain_x, valid_x, train_y, valid_y = train_test_split(x, y, test_size=0.15,\n                                                      stratify=y, random_state=8)\nsometimes = lambda aug: iaa.Sometimes(0.5, aug)\nseq = iaa.Sequential(\n        [\n            # apply the following augmenters to most images\n            iaa.Fliplr(0.5), # horizontally flip 50% of all images\n            iaa.Flipud(0.2), # vertically flip 20% of all images\n            sometimes(iaa.Affine(\n                scale={\"x\": (0.9, 1.1), \"y\": (0.9, 1.1)}, # scale images to 80-120% of their size, individually per axis\n                translate_percent={\"x\": (-0.1, 0.1), \"y\": (-0.1, 0.1)}, # translate by -20 to +20 percent (per axis)\n                rotate=(-10, 10), # rotate by -45 to +45 degrees\n                shear=(-5, 5), # shear by -16 to +16 degrees\n                order=[0, 1], # use nearest neighbour or bilinear interpolation (fast)\n                cval=(0, 255), # if mode is constant, use a cval between 0 and 255\n                mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)\n            )),\n            # execute 0 to 5 of the following (less important) augmenters per image\n            # don't execute all of them, as that would often be way too strong\n            iaa.SomeOf((0, 5),\n                [\n                    sometimes(iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))), # convert images into their superpixel representation\n                    iaa.OneOf([\n                        iaa.GaussianBlur((0, 1.0)), # blur images with a sigma between 0 and 3.0\n                        iaa.AverageBlur(k=(3, 5)), # blur image using local means with kernel sizes between 2 and 7\n                        iaa.MedianBlur(k=(3, 5)), # blur image using local medians with kernel sizes between 2 and 7\n                    ]),\n                    iaa.Sharpen(alpha=(0, 1.0), lightness=(0.9, 1.1)), # sharpen images\n                    iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)), # emboss images\n                    # search either for all edges or for directed edges,\n                    # blend the result with the original image using a blobby mask\n                    iaa.SimplexNoiseAlpha(iaa.OneOf([\n                        iaa.EdgeDetect(alpha=(0.5, 1.0)),\n                        iaa.DirectedEdgeDetect(alpha=(0.5, 1.0), direction=(0.0, 1.0)),\n                    ])),\n                    iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.01*255), per_channel=0.5), # add gaussian noise to images\n                    iaa.OneOf([\n                        iaa.Dropout((0.01, 0.05), per_channel=0.5), # randomly remove up to 10% of the pixels\n                        iaa.CoarseDropout((0.01, 0.03), size_percent=(0.01, 0.02), per_channel=0.2),\n                    ]),\n                    iaa.Invert(0.01, per_channel=True), # invert color channels\n                    iaa.Add((-2, 2), per_channel=0.5), # change brightness of images (by -10 to 10 of original value)\n                    iaa.AddToHueAndSaturation((-1, 1)), # change hue and saturation\n                    # either change the brightness of the whole image (sometimes\n                    # per channel) or change the brightness of subareas\n                    iaa.OneOf([\n                        iaa.Multiply((0.9, 1.1), per_channel=0.5),\n                        iaa.FrequencyNoiseAlpha(\n                            exponent=(-1, 0),\n                            first=iaa.Multiply((0.9, 1.1), per_channel=True),\n                            second=iaa.ContrastNormalization((0.9, 1.1))\n                        )\n                    ]),\n                    sometimes(iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)), # move pixels locally around (with random strengths)\n                    sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))), # sometimes move parts of the image around\n                    sometimes(iaa.PerspectiveTransform(scale=(0.01, 0.1)))\n                ],\n                random_order=True\n            )\n        ],\n        random_order=True)\nclass My_Generator(Sequence):\n\n    def __init__(self, image_filenames, labels,\n                 batch_size, is_train=True,\n                 mix=False, augment=False):\n        self.image_filenames, self.labels = image_filenames, labels\n        self.batch_size = batch_size\n        self.is_train = is_train\n        self.is_augment = augment\n        if(self.is_train):\n            self.on_epoch_end()\n        self.is_mix = mix\n\n    def __len__(self):\n        return int(np.ceil(len(self.image_filenames) / float(self.batch_size)))\n\n    def __getitem__(self, idx):\n        batch_x = self.image_filenames[idx * self.batch_size:(idx + 1) * self.batch_size]\n        batch_y = self.labels[idx * self.batch_size:(idx + 1) * self.batch_size]\n\n        if(self.is_train):\n            return self.train_generate(batch_x, batch_y)\n        return self.valid_generate(batch_x, batch_y)\n\n    def on_epoch_end(self):\n        if(self.is_train):\n            self.image_filenames, self.labels = shuffle(self.image_filenames, self.labels)\n        else:\n            pass\n    \n    def mix_up(self, x, y):\n        lam = np.random.beta(0.2, 0.4)\n        ori_index = np.arange(int(len(x)))\n        index_array = np.arange(int(len(x)))\n        np.random.shuffle(index_array)        \n        \n        mixed_x = lam * x[ori_index] + (1 - lam) * x[index_array]\n        mixed_y = lam * y[ori_index] + (1 - lam) * y[index_array]\n        \n        return mixed_x, mixed_y\n\n    def train_generate(self, batch_x, batch_y):\n        batch_images = []\n        for (sample, label) in zip(batch_x, batch_y):\n            img = cv2.imread('../input/aptos2019-blindness-detection/train_images/'+sample+'.png')\n            img = cv2.resize(img, (SIZE, SIZE))\n            if(self.is_augment):\n                img = seq.augment_image(img)\n            batch_images.append(img)\n        batch_images = np.array(batch_images, np.float32) / 255\n        batch_y = np.array(batch_y, np.float32)\n        if(self.is_mix):\n            batch_images, batch_y = self.mix_up(batch_images, batch_y)\n        return batch_images, batch_y\n\n    def valid_generate(self, batch_x, batch_y):\n        batch_images = []\n        for (sample, label) in zip(batch_x, batch_y):\n            img = cv2.imread('../input/aptos2019-blindness-detection/train_images/'+sample+'.png')\n            img = cv2.resize(img, (SIZE, SIZE))\n            batch_images.append(img)\n        batch_images = np.array(batch_images, np.float32) / 255\n        batch_y = np.array(batch_y, np.float32)\n        return batch_images, batch_y\ndef create_model(input_shape, n_out):\n    input_tensor = Input(shape=input_shape)\n    base_model = DenseNet121(include_top=False,\n                   weights=None,\n                   input_tensor=input_tensor)\n    base_model.load_weights(\"../input/densenet-keras/DenseNet-BC-121-32-no-top.h5\")\n    x = GlobalAveragePooling2D()(base_model.output)\n    x = Dropout(0.5)(x)\n    x = Dense(1024, activation='relu')(x)\n    x = Dropout(0.5)(x)\n    final_output = Dense(n_out, activation='softmax', name='final_output')(x)\n    model = Model(input_tensor, final_output) \n    return model\n# create callbacks list\nfrom keras.callbacks import (ModelCheckpoint, LearningRateScheduler,\n                             EarlyStopping, ReduceLROnPlateau,CSVLogger)\n\nepochs = 30; batch_size = 32\ncheckpoint = ModelCheckpoint('../working/densenet_.h5', monitor='val_loss', verbose=1, \n                             save_best_only=True, mode='min', save_weights_only = True)\nreduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=4, \n                                   verbose=1, mode='auto', epsilon=0.0001)\nearly = EarlyStopping(monitor=\"val_loss\", \n                      mode=\"min\", \n                      patience=9)\n\ncsv_logger = CSVLogger(filename='../working/training_log.csv',\n                       separator=',',\n                       append=True)\n\ntrain_generator = My_Generator(train_x, train_y, 128, is_train=True)\ntrain_mixup = My_Generator(train_x, train_y, batch_size, is_train=True, mix=False, augment=True)\nvalid_generator = My_Generator(valid_x, valid_y, batch_size, is_train=False)\n\nmodel = create_model(\n    input_shape=(SIZE,SIZE,3), \n    n_out=NUM_CLASSES)\n# reference link: https://www.kaggle.com/christofhenkel/weighted-kappa-loss-for-keras-tensorflow\ndef kappa_loss(y_true, y_pred, y_pow=2, eps=1e-12, N=5, bsize=32, name='kappa'):\n    \"\"\"A continuous differentiable approximation of discrete kappa loss.\n        Args:\n            y_pred: 2D tensor or array, [batch_size, num_classes]\n            y_true: 2D tensor or array,[batch_size, num_classes]\n            y_pow: int,  e.g. y_pow=2\n            N: typically num_classes of the model\n            bsize: batch_size of the training or validation ops\n            eps: a float, prevents divide by zero\n            name: Optional scope/name for op_scope.\n        Returns:\n            A tensor with the kappa loss.\"\"\"\n\n    with tf.name_scope(name):\n        y_true = tf.to_float(y_true)\n        repeat_op = tf.to_float(tf.tile(tf.reshape(tf.range(0, N), [N, 1]), [1, N]))\n        repeat_op_sq = tf.square((repeat_op - tf.transpose(repeat_op)))\n        weights = repeat_op_sq / tf.to_float((N - 1) ** 2)\n    \n        pred_ = y_pred ** y_pow\n        try:\n            pred_norm = pred_ / (eps + tf.reshape(tf.reduce_sum(pred_, 1), [-1, 1]))\n        except Exception:\n            pred_norm = pred_ / (eps + tf.reshape(tf.reduce_sum(pred_, 1), [bsize, 1]))\n    \n        hist_rater_a = tf.reduce_sum(pred_norm, 0)\n        hist_rater_b = tf.reduce_sum(y_true, 0)\n    \n        conf_mat = tf.matmul(tf.transpose(pred_norm), y_true)\n    \n        nom = tf.reduce_sum(weights * conf_mat)\n        denom = tf.reduce_sum(weights * tf.matmul(\n            tf.reshape(hist_rater_a, [N, 1]), tf.reshape(hist_rater_b, [1, N])) /\n                              tf.to_float(bsize))\n    \n        return nom*0.5 / (denom + eps) + categorical_crossentropy(y_true, y_pred)*0.5\nfrom keras.callbacks import Callback\nclass QWKEvaluation(Callback):\n    def __init__(self, validation_data=(), batch_size=64, interval=1):\n        super(Callback, self).__init__()\n\n        self.interval = interval\n        self.batch_size = batch_size\n        self.valid_generator, self.y_val = validation_data\n        self.history = []\n\n    def on_epoch_end(self, epoch, logs={}):\n        if epoch % self.interval == 0:\n            y_pred = self.model.predict_generator(generator=self.valid_generator,\n                                                  steps=np.ceil(float(len(self.y_val)) / float(self.batch_size)),\n                                                  workers=1, use_multiprocessing=False,\n                                                  verbose=1)\n            def flatten(y):\n                return np.argmax(y, axis=1).reshape(-1)\n            \n            score = cohen_kappa_score(flatten(self.y_val),\n                                      flatten(y_pred),\n                                      labels=[0,1,2,3,4],\n                                      weights='quadratic')\n            print(\"\\n epoch: %d - QWK_score: %.6f \\n\" % (epoch+1, score))\n            self.history.append(score)\n            if score >= max(self.history):\n                print('saving checkpoint: ', score)\n                self.model.save('../working/densenet_bestqwk.h5')\n\nqwk = QWKEvaluation(validation_data=(valid_generator, valid_y),\n                    batch_size=batch_size, interval=1)\n# warm up model\nfor layer in model.layers:\n    layer.trainable = False\n\nfor i in range(-3,0):\n    model.layers[i].trainable = True\n\nmodel.compile(\n    loss='categorical_crossentropy',\n    optimizer=Adam(1e-3))\n\n# model.fit_generator(\n#     train_generator,\n#     steps_per_epoch=np.ceil(float(len(train_y)) / float(128)),\n#     epochs=2,\n#     workers=WORKERS, use_multiprocessing=True,\n#     verbose=1,\n#     callbacks=[qwk])\nfor layer in model.layers:\n    layer.trainable = True\ncallbacks_list = [checkpoint, csv_logger, reduceLROnPlat, early, qwk]\nmodel.compile(loss='categorical_crossentropy',\n            # loss=kappa_loss,\n            optimizer=Adam(lr=1e-4))\n# model.fit_generator(\n#     train_mixup,\n#     steps_per_epoch=np.ceil(float(len(train_x)) / float(batch_size)),\n#     validation_data=valid_generator,\n#     validation_steps=np.ceil(float(len(valid_x)) / float(batch_size)),\n#     epochs=epochs,\n#     verbose=1,\n#     workers=1, use_multiprocessing=False,\n#     callbacks=callbacks_list)\nsubmission3 = pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')\nmodel.load_weights('../input/keras-base/m_aptos_vote/densenet_bestqwk.h5')\npredicted = []\n# reference:https://www.kaggle.com/CVxTz/cnn-starter-nasnet-mobile-0-9709-lb \nfor i, name in tqdm(enumerate(submission3['id_code'])):\n    path = os.path.join('../input/aptos2019-blindness-detection/test_images/', name+'.png')\n    image = cv2.imread(path)\n    image = cv2.resize(image, (SIZE, SIZE))\n    X = np.array((image[np.newaxis])/255)\n    score_predict=((model.predict(X).ravel()*model.predict(X[:, ::-1, :, :]).ravel()*model.predict(X[:, ::-1, ::-1, :]).ravel()*model.predict(X[:, :, ::-1, :]).ravel())**0.25).tolist()\n    label_predict = np.argmax(score_predict)\n    predicted.append(label_predict)\nsubmission3['diagnosis'] = predicted","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"4.https://www.kaggle.com/drhabib/starter-kernel-for-0-79"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nimport torch\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom matplotlib import style\nimport seaborn as sns\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom joblib import load, dump\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.metrics import confusion_matrix\nfrom fastai import *\nfrom fastai.vision import *\nfrom fastai.callbacks import *\nfrom torchvision import models as md\nfrom torch import nn\nfrom torch.nn import functional as F\nimport re\nimport math\nimport collections\nfrom functools import partial\nfrom torch.utils import model_zoo\nfrom sklearn import metrics\nfrom collections import Counter\nimport json","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Parameters for the entire model (stem, all blocks, and head)\nGlobalParams = collections.namedtuple('GlobalParams', [\n    'batch_norm_momentum', 'batch_norm_epsilon', 'dropout_rate',\n    'num_classes', 'width_coefficient', 'depth_coefficient',\n    'depth_divisor', 'min_depth', 'drop_connect_rate', 'image_size'])\n\n\n# Parameters for an individual model block\nBlockArgs = collections.namedtuple('BlockArgs', [\n    'kernel_size', 'num_repeat', 'input_filters', 'output_filters',\n    'expand_ratio', 'id_skip', 'stride', 'se_ratio'])\n\n\n# Change namedtuple defaults\nGlobalParams.__new__.__defaults__ = (None,) * len(GlobalParams._fields)\nBlockArgs.__new__.__defaults__ = (None,) * len(BlockArgs._fields)\n\n\ndef relu_fn(x):\n    \"\"\" Swish activation function \"\"\"\n    return x * torch.sigmoid(x)\n\n\ndef round_filters(filters, global_params):\n    \"\"\" Calculate and round number of filters based on depth multiplier. \"\"\"\n    multiplier = global_params.width_coefficient\n    if not multiplier:\n        return filters\n    divisor = global_params.depth_divisor\n    min_depth = global_params.min_depth\n    filters *= multiplier\n    min_depth = min_depth or divisor\n    new_filters = max(min_depth, int(filters + divisor / 2) // divisor * divisor)\n    if new_filters < 0.9 * filters:  # prevent rounding by more than 10%\n        new_filters += divisor\n    return int(new_filters)\n\n\ndef round_repeats(repeats, global_params):\n    \"\"\" Round number of filters based on depth multiplier. \"\"\"\n    multiplier = global_params.depth_coefficient\n    if not multiplier:\n        return repeats\n    return int(math.ceil(multiplier * repeats))\n\n\ndef drop_connect(inputs, p, training):\n    \"\"\" Drop connect. \"\"\"\n    if not training: return inputs\n    batch_size = inputs.shape[0]\n    keep_prob = 1 - p\n    random_tensor = keep_prob\n    random_tensor += torch.rand([batch_size, 1, 1, 1], dtype=inputs.dtype, device=inputs.device)\n    binary_tensor = torch.floor(random_tensor)\n    output = inputs / keep_prob * binary_tensor\n    return output\n\n\ndef get_same_padding_conv2d(image_size=None):\n    \"\"\" Chooses static padding if you have specified an image size, and dynamic padding otherwise.\n        Static padding is necessary for ONNX exporting of models. \"\"\"\n    if image_size is None:\n        return Conv2dDynamicSamePadding\n    else:\n        return partial(Conv2dStaticSamePadding, image_size=image_size)\n\nclass Conv2dDynamicSamePadding(nn.Conv2d):\n    \"\"\" 2D Convolutions like TensorFlow, for a dynamic image size \"\"\"\n    def __init__(self, in_channels, out_channels, kernel_size, stride=1, dilation=1, groups=1, bias=True):\n        super().__init__(in_channels, out_channels, kernel_size, stride, 0, dilation, groups, bias)\n        self.stride = self.stride if len(self.stride) == 2 else [self.stride[0]]*2\n\n    def forward(self, x):\n        ih, iw = x.size()[-2:]\n        kh, kw = self.weight.size()[-2:]\n        sh, sw = self.stride\n        oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)\n        pad_h = max((oh - 1) * self.stride[0] + (kh - 1) * self.dilation[0] + 1 - ih, 0)\n        pad_w = max((ow - 1) * self.stride[1] + (kw - 1) * self.dilation[1] + 1 - iw, 0)\n        if pad_h > 0 or pad_w > 0:\n            x = F.pad(x, [pad_w//2, pad_w - pad_w//2, pad_h//2, pad_h - pad_h//2])\n        return F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)\n\n\nclass Conv2dStaticSamePadding(nn.Conv2d):\n    \"\"\" 2D Convolutions like TensorFlow, for a fixed image size\"\"\"\n    def __init__(self, in_channels, out_channels, kernel_size, image_size=None, **kwargs):\n        super().__init__(in_channels, out_channels, kernel_size, **kwargs)\n        self.stride = self.stride if len(self.stride) == 2 else [self.stride[0]] * 2\n\n        # Calculate padding based on image size and save it\n        assert image_size is not None\n        ih, iw = image_size if type(image_size) == list else [image_size, image_size]\n        kh, kw = self.weight.size()[-2:]\n        sh, sw = self.stride\n        oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)\n        pad_h = max((oh - 1) * self.stride[0] + (kh - 1) * self.dilation[0] + 1 - ih, 0)\n        pad_w = max((ow - 1) * self.stride[1] + (kw - 1) * self.dilation[1] + 1 - iw, 0)\n        if pad_h > 0 or pad_w > 0:\n            self.static_padding = nn.ZeroPad2d((pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2))\n        else:\n            self.static_padding = Identity()\n\n    def forward(self, x):\n        x = self.static_padding(x)\n        x = F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)\n        return x\n\n\nclass Identity(nn.Module):\n    def __init__(self,):\n        super(Identity, self).__init__()\n\n    def forward(self, input):\n        return input\n\n\n########################################################################\n############## HELPERS FUNCTIONS FOR LOADING MODEL PARAMS ##############\n########################################################################\n\n\ndef efficientnet_params(model_name):\n    \"\"\" Map EfficientNet model name to parameter coefficients. \"\"\"\n    params_dict = {\n        # Coefficients:   width,depth,res,dropout\n        'efficientnet-b0': (1.0, 1.0, 224, 0.2),\n        'efficientnet-b1': (1.0, 1.1, 240, 0.2),\n        'efficientnet-b2': (1.1, 1.2, 260, 0.3),\n        'efficientnet-b3': (1.2, 1.4, 300, 0.3),\n        'efficientnet-b4': (1.4, 1.8, 380, 0.4),\n        'efficientnet-b5': (1.6, 2.2, 456, 0.4),\n        'efficientnet-b6': (1.8, 2.6, 528, 0.5),\n        'efficientnet-b7': (2.0, 3.1, 600, 0.5),\n    }\n    return params_dict[model_name]\n\n\nclass BlockDecoder(object):\n    \"\"\" Block Decoder for readability, straight from the official TensorFlow repository \"\"\"\n\n    @staticmethod\n    def _decode_block_string(block_string):\n        \"\"\" Gets a block through a string notation of arguments. \"\"\"\n        assert isinstance(block_string, str)\n\n        ops = block_string.split('_')\n        options = {}\n        for op in ops:\n            splits = re.split(r'(\\d.*)', op)\n            if len(splits) >= 2:\n                key, value = splits[:2]\n                options[key] = value\n\n        # Check stride\n        assert (('s' in options and len(options['s']) == 1) or\n                (len(options['s']) == 2 and options['s'][0] == options['s'][1]))\n\n        return BlockArgs(\n            kernel_size=int(options['k']),\n            num_repeat=int(options['r']),\n            input_filters=int(options['i']),\n            output_filters=int(options['o']),\n            expand_ratio=int(options['e']),\n            id_skip=('noskip' not in block_string),\n            se_ratio=float(options['se']) if 'se' in options else None,\n            stride=[int(options['s'][0])])\n\n    @staticmethod\n    def _encode_block_string(block):\n        \"\"\"Encodes a block to a string.\"\"\"\n        args = [\n            'r%d' % block.num_repeat,\n            'k%d' % block.kernel_size,\n            's%d%d' % (block.strides[0], block.strides[1]),\n            'e%s' % block.expand_ratio,\n            'i%d' % block.input_filters,\n            'o%d' % block.output_filters\n        ]\n        if 0 < block.se_ratio <= 1:\n            args.append('se%s' % block.se_ratio)\n        if block.id_skip is False:\n            args.append('noskip')\n        return '_'.join(args)\n\n    @staticmethod\n    def decode(string_list):\n        \"\"\"\n        Decodes a list of string notations to specify blocks inside the network.\n\n        :param string_list: a list of strings, each string is a notation of block\n        :return: a list of BlockArgs namedtuples of block args\n        \"\"\"\n        assert isinstance(string_list, list)\n        blocks_args = []\n        for block_string in string_list:\n            blocks_args.append(BlockDecoder._decode_block_string(block_string))\n        return blocks_args\n\n    @staticmethod\n    def encode(blocks_args):\n        \"\"\"\n        Encodes a list of BlockArgs to a list of strings.\n\n        :param blocks_args: a list of BlockArgs namedtuples of block args\n        :return: a list of strings, each string is a notation of block\n        \"\"\"\n        block_strings = []\n        for block in blocks_args:\n            block_strings.append(BlockDecoder._encode_block_string(block))\n        return block_strings\n\n\ndef efficientnet(width_coefficient=None, depth_coefficient=None, dropout_rate=0.2,\n                 drop_connect_rate=0.2, image_size=None, num_classes=1000):\n    \"\"\" Creates a efficientnet model. \"\"\"\n\n    blocks_args = [\n        'r1_k3_s11_e1_i32_o16_se0.25', 'r2_k3_s22_e6_i16_o24_se0.25',\n        'r2_k5_s22_e6_i24_o40_se0.25', 'r3_k3_s22_e6_i40_o80_se0.25',\n        'r3_k5_s11_e6_i80_o112_se0.25', 'r4_k5_s22_e6_i112_o192_se0.25',\n        'r1_k3_s11_e6_i192_o320_se0.25',\n    ]\n    blocks_args = BlockDecoder.decode(blocks_args)\n\n    global_params = GlobalParams(\n        batch_norm_momentum=0.99,\n        batch_norm_epsilon=1e-3,\n        dropout_rate=dropout_rate,\n        drop_connect_rate=drop_connect_rate,\n        # data_format='channels_last',  # removed, this is always true in PyTorch\n        num_classes=num_classes,\n        width_coefficient=width_coefficient,\n        depth_coefficient=depth_coefficient,\n        depth_divisor=8,\n        min_depth=None,\n        image_size=image_size,\n    )\n\n    return blocks_args, global_params\n\n\ndef get_model_params(model_name, override_params):\n    \"\"\" Get the block args and global params for a given model \"\"\"\n    if model_name.startswith('efficientnet'):\n        w, d, s, p = efficientnet_params(model_name)\n        # note: all models have drop connect rate = 0.2\n        blocks_args, global_params = efficientnet(\n            width_coefficient=w, depth_coefficient=d, dropout_rate=p, image_size=s)\n    else:\n        raise NotImplementedError('model name is not pre-defined: %s' % model_name)\n    if override_params:\n        # ValueError will be raised here if override_params has fields not included in global_params.\n        global_params = global_params._replace(**override_params)\n    return blocks_args, global_params\n\n\nurl_map = {\n    'efficientnet-b0': 'http://storage.googleapis.com/public-models/efficientnet-b0-08094119.pth',\n    'efficientnet-b1': 'http://storage.googleapis.com/public-models/efficientnet-b1-dbc7070a.pth',\n    'efficientnet-b2': 'http://storage.googleapis.com/public-models/efficientnet-b2-27687264.pth',\n    'efficientnet-b3': 'http://storage.googleapis.com/public-models/efficientnet-b3-c8376fa2.pth',\n    'efficientnet-b4': 'http://storage.googleapis.com/public-models/efficientnet-b4-e116e8b3.pth',\n    'efficientnet-b5': 'http://storage.googleapis.com/public-models/efficientnet-b5-586e6cc6.pth',\n}\n\ndef load_pretrained_weights(model, model_name, load_fc=True):\n    \"\"\" Loads pretrained weights, and downloads if loading for the first time. \"\"\"\n    state_dict = model_zoo.load_url(url_map[model_name])\n    if load_fc:\n        model.load_state_dict(state_dict)\n    else:\n        state_dict.pop('_fc.weight')\n        state_dict.pop('_fc.bias')\n        res = model.load_state_dict(state_dict, strict=False)\n        assert str(res.missing_keys) == str(['_fc.weight', '_fc.bias']), 'issue loading pretrained weights'\n    print('Loaded pretrained weights for {}'.format(model_name))\n    \n    \nclass MBConvBlock(nn.Module):\n    \"\"\"\n    Mobile Inverted Residual Bottleneck Block\n\n    Args:\n        block_args (namedtuple): BlockArgs, see above\n        global_params (namedtuple): GlobalParam, see above\n\n    Attributes:\n        has_se (bool): Whether the block contains a Squeeze and Excitation layer.\n    \"\"\"\n\n    def __init__(self, block_args, global_params):\n        super().__init__()\n        self._block_args = block_args\n        self._bn_mom = 1 - global_params.batch_norm_momentum\n        self._bn_eps = global_params.batch_norm_epsilon\n        self.has_se = (self._block_args.se_ratio is not None) and (0 < self._block_args.se_ratio <= 1)\n        self.id_skip = block_args.id_skip  # skip connection and drop connect\n\n        # Get static or dynamic convolution depending on image size\n        Conv2d = get_same_padding_conv2d(image_size=global_params.image_size)\n\n        # Expansion phase\n        inp = self._block_args.input_filters  # number of input channels\n        oup = self._block_args.input_filters * self._block_args.expand_ratio  # number of output channels\n        if self._block_args.expand_ratio != 1:\n            self._expand_conv = Conv2d(in_channels=inp, out_channels=oup, kernel_size=1, bias=False)\n            self._bn0 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)\n\n        # Depthwise convolution phase\n        k = self._block_args.kernel_size\n        s = self._block_args.stride\n        self._depthwise_conv = Conv2d(\n            in_channels=oup, out_channels=oup, groups=oup,  # groups makes it depthwise\n            kernel_size=k, stride=s, bias=False)\n        self._bn1 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)\n\n        # Squeeze and Excitation layer, if desired\n        if self.has_se:\n            num_squeezed_channels = max(1, int(self._block_args.input_filters * self._block_args.se_ratio))\n            self._se_reduce = Conv2d(in_channels=oup, out_channels=num_squeezed_channels, kernel_size=1)\n            self._se_expand = Conv2d(in_channels=num_squeezed_channels, out_channels=oup, kernel_size=1)\n\n        # Output phase\n        final_oup = self._block_args.output_filters\n        self._project_conv = Conv2d(in_channels=oup, out_channels=final_oup, kernel_size=1, bias=False)\n        self._bn2 = nn.BatchNorm2d(num_features=final_oup, momentum=self._bn_mom, eps=self._bn_eps)\n\n    def forward(self, inputs, drop_connect_rate=None):\n        \"\"\"\n        :param inputs: input tensor\n        :param drop_connect_rate: drop connect rate (float, between 0 and 1)\n        :return: output of block\n        \"\"\"\n\n        # Expansion and Depthwise Convolution\n        x = inputs\n        if self._block_args.expand_ratio != 1:\n            x = relu_fn(self._bn0(self._expand_conv(inputs)))\n        x = relu_fn(self._bn1(self._depthwise_conv(x)))\n\n        # Squeeze and Excitation\n        if self.has_se:\n            x_squeezed = F.adaptive_avg_pool2d(x, 1)\n            x_squeezed = self._se_expand(relu_fn(self._se_reduce(x_squeezed)))\n            x = torch.sigmoid(x_squeezed) * x\n\n        x = self._bn2(self._project_conv(x))\n\n        # Skip connection and drop connect\n        input_filters, output_filters = self._block_args.input_filters, self._block_args.output_filters\n        if self.id_skip and self._block_args.stride == 1 and input_filters == output_filters:\n            if drop_connect_rate:\n                x = drop_connect(x, p=drop_connect_rate, training=self.training)\n            x = x + inputs  # skip connection\n        return x\n\n\nclass EfficientNet(nn.Module):\n    \"\"\"\n    An EfficientNet model. Most easily loaded with the .from_name or .from_pretrained methods\n\n    Args:\n        blocks_args (list): A list of BlockArgs to construct blocks\n        global_params (namedtuple): A set of GlobalParams shared between blocks\n\n    Example:\n        model = EfficientNet.from_pretrained('efficientnet-b0')\n\n    \"\"\"\n\n    def __init__(self, blocks_args=None, global_params=None):\n        super().__init__()\n        assert isinstance(blocks_args, list), 'blocks_args should be a list'\n        assert len(blocks_args) > 0, 'block args must be greater than 0'\n        self._global_params = global_params\n        self._blocks_args = blocks_args\n\n        # Get static or dynamic convolution depending on image size\n        Conv2d = get_same_padding_conv2d(image_size=global_params.image_size)\n\n        # Batch norm parameters\n        bn_mom = 1 - self._global_params.batch_norm_momentum\n        bn_eps = self._global_params.batch_norm_epsilon\n\n        # Stem\n        in_channels = 3  # rgb\n        out_channels = round_filters(32, self._global_params)  # number of output channels\n        self._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias=False)\n        self._bn0 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)\n\n        # Build blocks\n        self._blocks = nn.ModuleList([])\n        for block_args in self._blocks_args:\n\n            # Update block input and output filters based on depth multiplier.\n            block_args = block_args._replace(\n                input_filters=round_filters(block_args.input_filters, self._global_params),\n                output_filters=round_filters(block_args.output_filters, self._global_params),\n                num_repeat=round_repeats(block_args.num_repeat, self._global_params)\n            )\n\n            # The first block needs to take care of stride and filter size increase.\n            self._blocks.append(MBConvBlock(block_args, self._global_params))\n            if block_args.num_repeat > 1:\n                block_args = block_args._replace(input_filters=block_args.output_filters, stride=1)\n            for _ in range(block_args.num_repeat - 1):\n                self._blocks.append(MBConvBlock(block_args, self._global_params))\n\n        # Head\n        in_channels = block_args.output_filters  # output of final block\n        out_channels = round_filters(1280, self._global_params)\n        self._conv_head = Conv2d(in_channels, out_channels, kernel_size=1, bias=False)\n        self._bn1 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)\n\n        # Final linear layer\n        self._dropout = self._global_params.dropout_rate\n        self._fc = nn.Linear(out_channels, self._global_params.num_classes)\n\n    def extract_features(self, inputs):\n        \"\"\" Returns output of the final convolution layer \"\"\"\n\n        # Stem\n        x = relu_fn(self._bn0(self._conv_stem(inputs)))\n\n        # Blocks\n        for idx, block in enumerate(self._blocks):\n            drop_connect_rate = self._global_params.drop_connect_rate\n            if drop_connect_rate:\n                drop_connect_rate *= float(idx) / len(self._blocks)\n            x = block(x, drop_connect_rate=drop_connect_rate)\n\n        # Head\n        x = relu_fn(self._bn1(self._conv_head(x)))\n\n        return x\n\n    def forward(self, inputs):\n        \"\"\" Calls extract_features to extract features, applies final linear layer, and returns logits. \"\"\"\n\n        # Convolution layers\n        x = self.extract_features(inputs)\n\n        # Pooling and final linear layer\n        x = F.adaptive_avg_pool2d(x, 1).squeeze(-1).squeeze(-1)\n        if self._dropout:\n            x = F.dropout(x, p=self._dropout, training=self.training)\n        x = self._fc(x)\n        return x\n\n    @classmethod\n    def from_name(cls, model_name, override_params=None):\n        cls._check_model_name_is_valid(model_name)\n        blocks_args, global_params = get_model_params(model_name, override_params)\n        return EfficientNet(blocks_args, global_params)\n\n    @classmethod\n    def from_pretrained(cls, model_name, num_classes=1000):\n        model = EfficientNet.from_name(model_name, override_params={'num_classes': num_classes})\n        return model\n\n    @classmethod\n    def get_image_size(cls, model_name):\n        cls._check_model_name_is_valid(model_name)\n        _, _, res, _ = efficientnet_params(model_name)\n        return res\n\n    @classmethod\n    def _check_model_name_is_valid(cls, model_name, also_need_pretrained_weights=False):\n        \"\"\" Validates model name. None that pretrained weights are only available for\n        the first four models (efficientnet-b{i} for i in 0,1,2,3) at the moment. \"\"\"\n        num_models = 4 if also_need_pretrained_weights else 8\n        valid_models = ['efficientnet_b'+str(i) for i in range(num_models)]\n        if model_name.replace('-','_') not in valid_models:\n            raise ValueError('model_name should be one of: ' + ', '.join(valid_models))\n#making model\nmd_ef = EfficientNet.from_pretrained('efficientnet-b5', num_classes=1)\n#copying weighst to the local directory \n!mkdir models\n!cp '../input/kaggle-public-copy/abcdef.pth' 'models'\ndef get_df():\n    base_image_dir = os.path.join('..', 'input/aptos2019-blindness-detection/')\n    train_dir = os.path.join(base_image_dir,'train_images/')\n    df = pd.read_csv(os.path.join(base_image_dir, 'train.csv'))\n    df['path'] = df['id_code'].map(lambda x: os.path.join(train_dir,'{}.png'.format(x)))\n    df = df.drop(columns=['id_code'])\n    df = df.sample(frac=1).reset_index(drop=True) #shuffle dataframe\n    test_df = pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')\n    return df, test_df\n\ndf, test_df = get_df()\n#you can play around with tfms and image sizes\nbs = 64\nsz = 224\ntfms = get_transforms(do_flip=True,flip_vert=True)\ndata = (ImageList.from_df(df=df,path='./',cols='path') \n        .split_by_rand_pct(0.2) \n        .label_from_df(cols='diagnosis',label_cls=FloatList) \n        .transform(tfms,size=sz,resize_method=ResizeMethod.SQUISH,padding_mode='zeros') \n        .databunch(bs=bs,num_workers=4) \n        .normalize(imagenet_stats)  \n       )\ndef qk(y_pred, y):\n    return torch.tensor(cohen_kappa_score(torch.round(y_pred), y, weights='quadratic'), device='cuda:0')\nlearn = Learner(data, \n                md_ef, \n                metrics = [qk], \n                model_dir=\"models\").to_fp16()\n\nlearn.data.add_test(ImageList.from_df(test_df,\n                                      '../input/aptos2019-blindness-detection',\n                                      folder='test_images',\n                                      suffix='.png'))\nlearn.load('abcdef');\n#https://www.kaggle.com/abhishek/optimizer-for-quadratic-weighted-kappa\nclass OptimizedRounder(object):\n    def __init__(self):\n        self.coef_ = 0\n\n    def _kappa_loss(self, coef, X, y):\n        X_p = np.copy(X)\n        for i, pred in enumerate(X_p):\n            if pred < coef[0]:\n                X_p[i] = 0\n            elif pred >= coef[0] and pred < coef[1]:\n                X_p[i] = 1\n            elif pred >= coef[1] and pred < coef[2]:\n                X_p[i] = 2\n            elif pred >= coef[2] and pred < coef[3]:\n                X_p[i] = 3\n            else:\n                X_p[i] = 4\n\n        ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')\n        return -ll\n\n    def fit(self, X, y):\n        loss_partial = partial(self._kappa_loss, X=X, y=y)\n        initial_coef = [0.5, 1.5, 2.5, 3.5]\n        self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')\n        print(-loss_partial(self.coef_['x']))\n\n    def predict(self, X, coef):\n        X_p = np.copy(X)\n        for i, pred in enumerate(X_p):\n            if pred < coef[0]:\n                X_p[i] = 0\n            elif pred >= coef[0] and pred < coef[1]:\n                X_p[i] = 1\n            elif pred >= coef[1] and pred < coef[2]:\n                X_p[i] = 2\n            elif pred >= coef[2] and pred < coef[3]:\n                X_p[i] = 3\n            else:\n                X_p[i] = 4\n        return X_p\n\n    def coefficients(self):\n        return self.coef_['x']\n\ncoefficients=[0.5, 1.5, 2.5, 3.5]\nopt = OptimizedRounder()\npreds4,y = learn.get_preds(DatasetType.Test)\ntst_pred = opt.predict(preds4, coefficients)\ntest_df.diagnosis = tst_pred.astype(int)\nsubmission4 = test_df.copy()\nsubmission4.to_csv('submission4.csv',index=False)\nprint ('done')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission4.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# xecption"},{"metadata":{"trusted":true,"_kg_hide-output":true,"collapsed":true},"cell_type":"code","source":"import os\nfrom keras.applications.xception import Xception\nfrom keras.models import Model\nfrom keras.layers import Input, Activation, Dropout, Flatten, Dense, GlobalAveragePooling2D\nfrom keras import optimizers\nfrom keras.callbacks import ModelCheckpoint\n\nfrom keras.utils import np_utils\n\nimport cv2\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\n\nimg = np.loadtxt(\"../input/aptos2019-blindness-detection/train.csv\",       # 読み込みたいファイルのパス\n                  delimiter=\",\",    # ファイルの区切り文字\n                  skiprows=1,    # 先頭の何行を無視するか（指定した行数までは読み込まない）\n                  usecols=(0), # 読み込みたい列番号\n                  dtype = \"str\"\n                 )\nimg\n\nlabel = np.loadtxt(\"../input/aptos2019-blindness-detection/train.csv\",       # 読み込みたいファイルのパス\n                  delimiter=\",\",    # ファイルの区切り文字\n                  skiprows=1,    # 先頭の何行を無視するか（指定した行数までは読み込まない）\n                  usecols=(1), # 読み込みたい列番号\n                  dtype = \"int\"\n                 )\nlabel\n\nimg_label_trains = []\nimg_label_validations = []\n\nfor i in range(3):\n    data_train, data_test, labels_train, labels_test = train_test_split(img, label, train_size=0.75,random_state=i*5,stratify=label)\n    \n    img_label_train = np.stack([data_train, labels_train],axis=1)\n    img_label_validation = np.stack([data_test, labels_test],axis=1)\n    \n    img_label_trains.append(img_label_train)\n    img_label_validations.append(img_label_validation)\n    \nimg_width, img_height = 299, 299\nnum_train = int(len(data_train))\nnum_val = int(len(data_test))\nbatch_size = 4\nprint(num_train, num_val)\nabs_path = \"../input/aptos2019-blindness-detection/train_images/\"\n\ndef vertical_flip(image, rate=0.5):\n    if np.random.rand() < rate:\n        image = image[::-1, :, :]\n    return image\n\ndef horizontal_flip(image):\n    image = image[:, ::-1, :]\n    return image\n\ndef image_translation(img):\n    params = np.random.randint(-150, 151)\n    if not isinstance(params, list):\n        params = [params, params]\n    rows, cols, ch = img.shape\n\n    M = np.float32([[1, 0, params[0]], [0, 1, params[1]]])\n    dst = cv2.warpAffine(img, M, (cols, rows))\n    return dst\n\ndef image_shear(img):\n    params = np.random.randint(-5, 6)*0.1\n    rows, cols, ch = img.shape\n    factor = params*(-1.0)\n    M = np.float32([[1, factor, 0], [0, 1, 0]])\n    dst = cv2.warpAffine(img, M, (cols, rows))\n    return dst\n\ndef image_rotation(img):\n    params = np.random.randint(-30, 31)\n    rows, cols, ch = img.shape\n    M = cv2.getRotationMatrix2D((cols/2, rows/2), params, 1)\n    dst = cv2.warpAffine(img, M, (cols, rows))\n    return dst\n\ndef image_contrast(img):\n    params = np.random.randint(5, 20)*0.1\n    alpha = params\n    new_img = cv2.multiply(img, np.array([alpha]))                    # mul_img = img*alpha\n    #new_img = cv2.add(mul_img, beta)                                  # new_img = img*alpha + beta\n  \n    return new_img\n\ndef image_blur(img):\n    params = params = np.random.randint(1, 21)\n    blur = []\n    if params == 1:\n        blur = cv2.blur(img, (3, 3))\n    if params == 2:\n        blur = cv2.blur(img, (4, 4))\n    if params == 3:\n        blur = cv2.blur(img, (5, 5))\n    if params == 4:\n        blur = cv2.GaussianBlur(img, (3, 3), 0)\n    if params == 5:\n        blur = cv2.GaussianBlur(img, (5, 5), 0)\n    if params == 6:\n        blur = cv2.GaussianBlur(img, (7, 7), 0)\n    if params == 7:\n        blur = cv2.medianBlur(img, 3)\n    if params == 8:\n        blur = cv2.medianBlur(img, 5)\n    if params == 9:\n        blur = cv2.blur(img, (6, 6))\n    if params == 10:\n        blur = cv2.bilateralFilter(img, 9, 75, 75)\n    if params > 10:\n        blur = img\n        \n    return blur\n\ndef image_brightness2(img):\n    params = np.random.randint(-21, 22)\n    beta = params\n    b, g, r = cv2.split(img)\n    b = cv2.add(b, beta)\n    g = cv2.add(g, beta)\n    r = cv2.add(r, beta)\n    new_img = cv2.merge((b, g, r))\n    return new_img\n\n\ndef get_random_data(image_lines_1, abs_path, img_width, img_height):\n    image_file = abs_path + image_lines_1[0] + \".png\"\n    label = np.eye(5)[int(image_lines_1[1])]\n    \n    seed_image = cv2.imread(image_file)\n    seed_image = cv2.cvtColor(seed_image, cv2.COLOR_BGR2RGB)\n    seed_image = cv2.resize(seed_image, dsize=(img_width, img_height))\n    \n    seed_image = vertical_flip(seed_image)\n    seed_image = horizontal_flip(seed_image)\n    seed_image = image_shear(seed_image)\n    seed_image = image_rotation(seed_image)\n    seed_image = image_contrast(seed_image)\n    seed_image = image_blur(seed_image)\n    seed_image = image_brightness2(seed_image)\n    \n    seed_image = seed_image / 255\n    \n    return seed_image, label\n\n\ndef data_generator(image_lines, batch_size, abs_path, img_width, img_height):\n    '''data generator for fit_generator'''\n    n = len(image_lines)\n    i = 0\n    while True:\n        image_data = []\n        label_data = []\n        for b in range(batch_size):\n            if i==0:\n                np.random.shuffle(image_lines)\n            image, label = get_random_data(image_lines[i], abs_path, img_width, img_height)\n            image_data.append(image)\n            label_data.append(label)\n            i = (i+1) % n\n        image_data = np.array(image_data)\n        label_data = np.array(label_data)\n        yield image_data, label_data\n\ndef data_generator_wrapper(image_lines, batch_size, abs_path, img_width, img_height):\n    n = len(image_lines)\n    if n==0 or batch_size<=0: return None\n    return data_generator(image_lines, batch_size, abs_path, img_width, img_height)\n\n\nmodels = []\n\nfor i in range(3):\n\n    input_tensor = Input(shape=(img_height, img_width, 3))\n\n    xception_model = Xception(include_top=False, weights=None, input_tensor=input_tensor)\n\n    xception_model.load_weights(\"../input/keras-pretrained-models/xception_weights_tf_dim_ordering_tf_kernels_notop.h5\")\n\n    x = xception_model.output\n    x = GlobalAveragePooling2D()(x)\n    x = Dense(1024, activation='relu')(x)\n    x = Dropout(0.3)(x)\n    outputs = Dense(5, activation='softmax')(x)\n\n    model = Model(inputs=xception_model.input, outputs=outputs)\n    \n    model.compile(optimizer=optimizers.SGD(lr=0.001,momentum=0.9),\n              loss='categorical_crossentropy',\n              metrics=['accuracy'])\n\n    model.summary()\n    \n    models.append(model)\n    \n\nmodelpath = '/kaggle/input/xception-0-757/'\n\n\nimg_test = np.loadtxt(\"../input/aptos2019-blindness-detection/test.csv\",       # 読み込みたいファイルのパス\n                  delimiter=\",\",    # ファイルの区切り文字\n                  skiprows=1,    # 先頭の何行を無視するか（指定した行数までは読み込まない）\n#                  usecols=(1), # 読み込みたい列番号\n                  dtype = \"str\"\n                 )\n\n\n\nmodels[0].load_weights(\"/kaggle/input/xception-0-757/best_weight0.h5\")\nmodels[1].load_weights(\"/kaggle/input/xception-0-757/best_weight1.h5\")\nmodels[2].load_weights(\"/kaggle/input/xception-0-757/best_weight2.h5\")\n\ntest_abs_path = \"../input/aptos2019-blindness-detection/test_images/\"\n\ndata = []\nfor i in range(len(img_test)):\n    image_file = test_abs_path + img_test[i] + \".png\"\n    seed_image = cv2.imread(image_file)\n    seed_image = cv2.cvtColor(seed_image, cv2.COLOR_BGR2RGB)\n    seed_image = cv2.resize(seed_image, dsize=(img_width, img_height))\n    seed_image = np.expand_dims(seed_image, axis=0)\n    seed_image = seed_image / 255\n    predict1 = models[0].predict(seed_image)\n    predict2 = models[1].predict(seed_image)\n    predict3 = models[2].predict(seed_image)\n    predict_mean = (predict1+predict2+predict3)/3\n    x = np.array([img_test[i], np.argmax(predict_mean)])\n    data.append(x)\n    \ndata = np.array(data)\n\ncolumns = ['id_code', 'diagnosis']\nname = 'sample'\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"d = pd.DataFrame(data=data, columns=columns, dtype='str')\nd['diagnosis'] = d['diagnosis'].astype(int)\nd.to_csv(\"submission_xce_.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"d.head(5)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission1.head(4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission2.head(4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission3.head(4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission4.head(4)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# blend"},{"metadata":{"trusted":true},"cell_type":"code","source":"wei = [0.4, 0.6]\nker = [submission3, d]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"numClass = 5\nsubemp = np.zeros((ker[0].shape[0],numClass))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(len(ker)):\n    subemp[ker[i].index, ker[i].diagnosis.tolist()] += wei[i]\nprint(subemp)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subKER = pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')\nsubKER['diagnosis'] = subemp.argmax(1).astype(int)\nsubKER.to_csv('submissionKER.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subKER.head(5)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"score = [0.777, 0.758, 0.749, 0.783]\n# weight = [0.38, 0.15,0.07, 0.40]\nweight = [0.29, 0.16, 0.09,0.06, 0.40]\nsubData = [submission1, submission2, d, submission3, submission4]\npredsData = [preds1, preds2, preds4]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# np.sum([0.38, 0.16,0.06, 0.40])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"numClass = 5\nsubTemp = np.zeros((subData[0].shape[0],numClass))\nfor i in range(len(subData)):\n    subTemp[subData[i].index, subData[i].diagnosis.tolist()] += weight[i]\nprint(subTemp)\nsub = pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')\nsub['diagnosis'] = subTemp.argmax(1).astype(int)\nsub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"numClass = 5\nsubTemp = np.zeros((subData[0].shape[0],numClass))\nfor i in range(len(subData)):\n    subTemp[subData[i].index, subData[i].diagnosis.tolist()] += weight[i]\nprint(subTemp)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subTemp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')\nsub['diagnosis'] = subTemp.argmax(1).astype(int)\nsub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# score = [0.777, 0.758, 0.783]\n# weight = [0.35, 0.2, 0.45]\n# predsData = weight[0]*preds1 + weight[1]*preds2 + weight[2]*preds4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# coef = [0.5, 1.5, 2.5, 3.5]\n# for i, pred in enumerate(predsData):\n#     if pred < coef[0]:\n#         predsData[i] = 0\n#     elif pred >= coef[0] and pred < coef[1]:\n#         predsData[i] = 1\n#     elif pred >= coef[1] and pred < coef[2]:\n#         predsData[i] = 2\n#     elif pred >= coef[2] and pred < coef[3]:\n#         predsData[i] = 3\n#     else:\n#         predsData[i] = 4\n# submission = pd.read_csv(\"../input/aptos2019-blindness-detection/sample_submission.csv\")\n# submission.diagnosis = predsData.astype(int)\n# submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}