{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Aptos 2019 - Best models blend (top 2-3%)\n\nSimple weighted blend of a EfficientNet B3 with 256px images + EfficientNet B3 with 300px images and DenseNet101 with 320px images. Achieved 0.926 on the private leaderboard.\n\nSee [this](https://www.kaggle.com/c/aptos2019-blindness-detection/discussion/107947) post for details about solution."},{"metadata":{"trusted":true},"cell_type":"code","source":"import time\nimport sys\nimport os\n\nimport cv2                  \n         \nfrom random import shuffle  \nfrom zipfile import ZipFile\n\n# Ignore  the warnings\nimport warnings\nwarnings.filterwarnings('always')\nwarnings.filterwarnings('ignore')\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom matplotlib import style\nimport seaborn as sns\nimport numpy as np  \nfrom tqdm import tqdm \nfrom sklearn.utils import shuffle\nfrom sklearn.metrics import confusion_matrix, cohen_kappa_score\nfrom fastai import *\nfrom fastai.vision import *\nfrom fastai.callbacks import Callback\nfrom fastai.callbacks import SaveModelCallback, EarlyStoppingCallback, ReduceLROnPlateauCallback\n\nstyle.use('fivethirtyeight')\nsns.set(style='whitegrid',color_codes=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Shared functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed=999):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\n\ndef get_label(diagnosis):\n    return ','.join([str(i) for i in range(diagnosis + 1)])\n\n\ndef get_train_df(seed, num_zeros=4000):\n    val_preds_id = pd.read_csv('../input/bd-peter-and-lex-validation-set/val.csv')['id_code']\n\n    df_train = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\n    df_test = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')\n\n    df_train['is_valid'] = False\n    # df_train.loc[df_train.id_code.isin(val_preds_id), 'is_valid'] = True\n    df_train.id_code = '../input/aptos2019-blindness-detection/train_images/' + df_train.id_code + '.png'\n\n    df_train.columns = ['image_path', 'diagnosis', 'is_valid']\n\n    extra_training_df = pd.read_csv('../input/diabetic-retinopathy-resized/trainLabels.csv')\n    extra_training_df['is_valid'] = False\n    # extra_training_df.loc[extra_training_df.image.isin(val_preds_id), 'is_valid'] = True\n    extra_training_df.image = '../input/diabetic-retinopathy-resized/resized_train/resized_train/' + extra_training_df.image + '.jpeg'\n    extra_training_df.columns = ['image_path', 'diagnosis', 'is_valid']\n    \n    test_labels_15_df = pd.read_csv('../input/resized-2015-2019-blindness-detection-images/labels/testLabels15.csv')\n    del test_labels_15_df['Usage']\n    test_labels_15_df.columns = ['image_id', 'diagnosis']\n    test_labels_15_df['dataset_id'] = 'test_labels_15'\n    test_labels_15_df['image_path'] = '../input/resized-2015-2019-blindness-detection-images/resized test 15/' + test_labels_15_df.image_id + '.jpg'\n    test_labels_15_df['is_valid'] = True\n    test_labels_15_df = test_labels_15_df[['image_path', 'diagnosis', 'is_valid']]\n\n    df_train = pd.concat([\n        df_train,\n        extra_training_df[(extra_training_df.diagnosis == 0) & (extra_training_df.is_valid)],\n        extra_training_df[(extra_training_df.diagnosis == 0) & ~(extra_training_df.is_valid)].sample(n=num_zeros, random_state=seed),\n        extra_training_df[extra_training_df.diagnosis == 1],\n        extra_training_df[extra_training_df.diagnosis == 2],\n        extra_training_df[extra_training_df.diagnosis == 3],\n        extra_training_df[extra_training_df.diagnosis == 4],\n        pd.concat([\n            test_labels_15_df[test_labels_15_df.diagnosis == 0].sample(n=7900, random_state=420),\n            test_labels_15_df[test_labels_15_df.diagnosis != 0]\n        ]).sample(n=10_000, random_state=420),\n    ]).sample(frac=1, random_state=seed)\n\n    df_train['label'] = df_train.diagnosis.apply(get_label)\n    \n    return df_train\n\n\ndef make_or_preds(model_name, learner, model_path, expected_val):\n    learn.load(model_path);\n\n    val_items = learn.data.valid_dl.dataset.items\n    val_preds, val_y = learn.get_preds(ds_type=DatasetType.Valid)\n    metric = cohen_kappa_score(val_y.argmax(1).numpy(), get_output_preds((val_preds > 0.5).numpy()), weights='quadratic')\n\n    raw_preds = pd.DataFrame(val_preds.numpy())\n    raw_preds.columns = ['x_0', 'x_1', 'x_2', 'x_3', 'x_4']\n\n    val_preds_df = pd.concat([\n        pd.DataFrame({\n            'id_code': [v.split('/')[-1].split('.')[0] for v in val_items],\n            'diagnosis': val_y.argmax(1).numpy(),\n            'preds': get_output_preds((val_preds > 0.5).numpy())\n        }),\n        raw_preds\n    ], axis=1)\n\n    val_preds_df.to_csv(f'{model_name}_val_preds.csv', index=False)\n\n    test_items = learn.data.test_dl.dataset.items\n\n    test_preds, __ = learn.get_preds(ds_type=DatasetType.Test)\n\n    raw_test_preds = pd.DataFrame(test_preds.numpy())\n    raw_test_preds.columns = ['x_0', 'x_1', 'x_2', 'x_3', 'x_4']\n\n    test_preds_df = pd.concat([\n        pd.DataFrame({\n            'id_code': [v.split('/')[-1].split('.')[0] for v in test_items],\n            'preds': get_output_preds((test_preds > 0.5).numpy())\n        }),\n        raw_test_preds\n    ], axis=1)\n\n    test_preds_df.to_csv(f'{model_name}_test_preds.csv', index=False)\n\n    print(f'Val kappa score: {metric} (expected: {expected_val})')\n\n\ndef avg_tta_score(model_name):\n    no_flip = pd.read_csv(f'{model_name}_val_preds.csv').sort_values('id_code')\n    flip = pd.read_csv(f'{model_name}-flip_val_preds.csv').sort_values('id_code')\n\n    val_preds_avg = no_flip[['x_0', 'x_1', 'x_2', 'x_3', 'x_4']].values * 0.5 + flip[['x_0', 'x_1', 'x_2', 'x_3', 'x_4']].values * 0.5\n\n    return cohen_kappa_score(flip.diagnosis, get_output_preds((val_preds_avg > 0.5)), weights='quadratic')\n\n\nclass ConfusionMatrix(Callback):\n    \"Computes the confusion matrix.\"\n\n    def on_train_begin(self, **kwargs):\n        self.n_classes = 0\n\n    def on_epoch_begin(self, **kwargs):\n        self.cm = None\n\n    def on_batch_end(self, last_output:Tensor, last_target:Tensor, **kwargs):\n        preds = torch.tensor(get_preds((torch.sigmoid(last_output) > 0.5).cpu().numpy()))\n        \n        targs = torch.tensor(get_preds(last_target.cpu().numpy()))\n\n        if self.n_classes == 0:\n            self.n_classes = last_output.shape[-1]\n            self.x = torch.arange(0, self.n_classes)\n        \n        cm = ((preds==self.x[:, None]) & (targs==self.x[:, None, None])).sum(dim=2, dtype=torch.float32)\n        if self.cm is None: self.cm =  cm\n        else:               self.cm += cm\n\n    def on_epoch_end(self, **kwargs):\n        self.metric = self.cm\n\n\n@dataclass\nclass KappaScore(ConfusionMatrix):\n    \"Compute the rate of agreement (Cohens Kappa).\"\n    weights:Optional[str]=None      # None, `linear`, or `quadratic`\n\n    def on_epoch_end(self, last_metrics, **kwargs):\n        sum0 = self.cm.sum(dim=0)\n        sum1 = self.cm.sum(dim=1)\n        expected = torch.einsum('i,j->ij', (sum0, sum1)) / sum0.sum()\n        if self.weights is None:\n            w = torch.ones((self.n_classes, self.n_classes))\n            w[self.x, self.x] = 0\n        elif self.weights == \"linear\" or self.weights == \"quadratic\":\n            w = torch.zeros((self.n_classes, self.n_classes))\n            w += torch.arange(self.n_classes, dtype=torch.float)\n            w = torch.abs(w - torch.t(w)) if self.weights == \"linear\" else (w - torch.t(w)) ** 2\n        else: raise ValueError('Unknown weights. Expected None, \"linear\", or \"quadratic\".')\n        k = torch.sum(w * self.cm) / torch.sum(w * expected)\n        return add_metrics(last_metrics, 1-k)\n\n\nclass FlattenedLoss():\n    \"Same as `func`, but flattens input and target.\"\n    def __init__(self, func, *args, axis:int=-1, floatify:bool=False, is_2d:bool=True, **kwargs):\n        self.func,self.axis,self.floatify,self.is_2d = func(*args,**kwargs),axis,floatify,is_2d\n        functools.update_wrapper(self, self.func)\n\n    def __repr__(self): return f\"FlattenedLoss of {self.func}\"\n    @property\n    def reduction(self): return self.func.reduction\n    @reduction.setter\n    def reduction(self, v): self.func.reduction = v\n\n    def __call__(self, input:Tensor, target:Tensor, **kwargs)->Rank0Tensor:\n        input = input.transpose(self.axis,-1).contiguous()\n        target = target.transpose(self.axis,-1).contiguous()\n        if self.floatify: target = target.float()\n            \n        # Label smoothing experiment\n        target = (target * 0.9 + 0.05)\n        target[:,0] = 1\n\n        input = input.view(-1,input.shape[-1]) if self.is_2d else input.view(-1)\n        return self.func.__call__(input, target.view(-1), **kwargs)\n\n\ndef LabelSmoothBCEWithLogitsFlat(*args, axis:int=-1, floatify:bool=True, **kwargs):\n    \"Same as `nn.BCEWithLogitsLoss`, but flattens input and target.\"\n    return FlattenedLoss(nn.BCEWithLogitsLoss, *args, axis=axis, floatify=floatify, is_2d=False, **kwargs)\n\n\nclass ReconstructFixMultiCategoryList(MultiCategoryList):\n    def reconstruct(self, t):\n        try:\n            return super().reconstruct(t)\n        except Exception as e:\n            return FloatItem(np.log(t))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_df = pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## EfficientNet b3"},{"metadata":{"trusted":true},"cell_type":"code","source":"SIGMA_X = 10\n\nsys.path.append('../input/efficientnet-pytorch/efficientnet-pytorch/EfficientNet-PyTorch-master')\nimport efficientnet_pytorch\n\ndef get_data(seed, size, bs, tfms=((), ())):\n    df_train = get_train_df(seed)\n    ImageList.open = lambda self, fn: open_img(self, fn, size=size)\n    data = (\n        ImageList.from_df(\n            path='./',\n            df=df_train,\n            folder='.'\n        )\n    )\n    data = (data.split_from_df('is_valid')\n            .label_from_df('label', label_delim=',', label_cls=ReconstructFixMultiCategoryList)\n            .transform(\n                tfms,\n                resize_method=ResizeMethod.NO,\n                padding_mode='zeros')\n            .databunch(bs=bs)\n            .normalize(imagenet_stats))\n    data.add_test(ImageList.from_df(sample_df, '../input/aptos2019-blindness-detection', folder='test_images', suffix='.png'))\n    return data\n\n\n# To remove irregularities along the circular boundary of the image\nPARAM = 96\ndef Radius_Reduction(img, PARAM):\n    h,w,c=img.shape\n    Frame=np.zeros((h,w,c),dtype=np.uint8)\n    cv2.circle(Frame,(int(math.floor(w/2)),int(math.floor(h/2))),int(math.floor((h*PARAM)/float(2*100))), (255,255,255), -1)\n    Frame1=cv2.cvtColor(Frame, cv2.COLOR_BGR2GRAY)\n    img1 =cv2.bitwise_and(img,img,mask=Frame1)\n    return img1\n\n\ndef info_image(im):\n    # Compute the center (cx, cy) and radius of the eye\n    cy = im.shape[0]//2\n    midline = im[cy,:]\n    midline = np.where(midline>midline.mean()/3)[0]\n    if len(midline)>im.shape[1]//2:\n        x_start, x_end = np.min(midline), np.max(midline)\n    else: # This actually rarely happens p~1/10000\n        x_start, x_end = im.shape[1]//10, 9*im.shape[1]//10\n    cx = (x_start + x_end)/2\n    r = (x_end - x_start)/2\n    return cx, cy, r\n\n\ndef subtract_median_bg_image(im):\n    k = np.max(im.shape)//20*2+1\n    bg = cv2.medianBlur(im, k)\n    return cv2.addWeighted (im, 4, bg, -4, 128)\n\n\ndef get_output_preds(arr):\n    mask = arr == 0\n    return np.clip(np.where(mask.any(1), mask.argmax(1), 5) - 1, 0, 4)\n\n\ndef get_efficientnet(name, pretrained, model_path):\n    \"\"\"Constructs a EfficientNetB5 model for FastAI.\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n    \"\"\"\n    model = efficientnet_pytorch.EfficientNet.from_name(f'efficientnet-{name}', override_params={'num_classes': 5})\n    if pretrained:\n        model_state = torch.load(model_path)\n        # load original weights apart from its head\n        if '_fc.weight' in model_state.keys():\n            model_state.pop('_fc.weight')\n            model_state.pop('_fc.bias')\n            res = model.load_state_dict(model_state, strict=False)\n            print('Loaded pretrained')\n            assert str(res.missing_keys) == str(['_fc.weight', '_fc.bias']), 'issue loading pretrained weights'\n        else:\n            # A basic remapping is required\n            from collections import OrderedDict\n            mapping = { i:o for i,o in zip(model_state.keys(), model.state_dict().keys()) }\n            mapped_model_state = OrderedDict([\n                (mapping[k], v) for k,v in model_state.items() if not mapping[k].startswith('_fc')\n            ])\n            res = model.load_state_dict(mapped_model_state, strict=False)\n\n    return model\n\n\ndef crop_image_from_gray(img, tol=7):\n    if img.ndim == 2:\n        mask = img>tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    elif img.ndim==3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img > tol\n        \n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0): # image is too dark so that we crop out everything,\n            return img # return original image\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n            img = np.stack([img1,img2,img3],axis=-1)\n        return img\n\n\ndef load_ben_color(path, img_size, sigmaX=SIGMA_X):\n    image = cv2.imread(path)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    image = crop_image_from_gray(image)\n    image = cv2.resize(image, (img_size, img_size))\n    image = cv2.addWeighted (image, 4, cv2.GaussianBlur(image, (0,0), sigmaX) , -4 ,128)\n    return Image(pil2tensor(image, np.float32).div_(255))\n\n\ndef resize_image(im, img_size):\n    # Crops, resizes and potentially augments the image to IMG_SIZE\n    cx, cy, r = info_image(im)\n    scaling = img_size/(2*r)\n    rotation = 0\n    M = cv2.getRotationMatrix2D((cx,cy), rotation, scaling)\n    M[0,2] -= cx - img_size/2\n    M[1,2] -= cy - img_size/2\n    return cv2.warpAffine(im, M, (img_size, img_size)) # This is the most important line\n\n\ndef open_img(self, fn, size):\n    \"Open image in `fn`, subclass and overwrite for custom behavior.\"\n    image = cv2.imread(fn)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    image = resize_image(image, size)\n    # image = subtract_median_bg_image(image)\n    image = Radius_Reduction(image, PARAM)\n    return Image(pil2tensor(image, np.float32).div_(255))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## EfficientNet B03"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = get_efficientnet('b3', True, '../input/efficientnet-pytorch/efficientnet-b3-c8376fa2.pth')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed_everything(422)\n\ndata = get_data(seed=437, size=256, bs=64)\nlearn = Learner(data, model, model_dir=\".\", callback_fns=[BnFreeze])\n\nmake_or_preds(\n    model_name='bd-efficientnet-b03-2015val-psu3-sz-256',\n    learner=learn,\n    model_path='../input/bd-efficientnet-b03-2015val-psu3-sz-256/best_model',\n    expected_val=0.7273654273342949\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed_everything(422)\n\ndata = get_data(seed=437, size=256, bs=64, tfms=((), (flip_lr(p=1))))\nlearn = Learner(data, model, model_dir=\".\", callback_fns=[BnFreeze])\n\nmake_or_preds(\n    model_name='bd-efficientnet-b03-2015val-psu3-sz-256-flip',\n    learner=learn,\n    model_path='../input/bd-efficientnet-b03-2015val-psu3-sz-256/best_model',\n    expected_val=0.7273654273342949\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"avg_tta_score('bd-efficientnet-b03-2015val-psu3-sz-256')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed_everything(420)\n\ndata = get_data(seed=420, size=300, bs=64)\nlearn = Learner(data, model, model_dir=\".\", callback_fns=[BnFreeze])\n\nmake_or_preds(\n    model_name='bd-efficientnet-b03-2015val-psu3-sz-300',\n    learner=learn,\n    model_path='../input/bd-efficientnet-b03-2015val-psu3-sz-300/best_model',\n    expected_val=0.7433966054032339\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed_everything(420)\n\ndata = get_data(seed=420, size=300, bs=64, tfms=((), (flip_lr(p=1))))\nlearn = Learner(data, model, model_dir=\".\", callback_fns=[BnFreeze])\n\nmake_or_preds(\n    model_name='bd-efficientnet-b03-2015val-psu3-sz-300-flip',\n    learner=learn,\n    model_path='../input/bd-efficientnet-b03-2015val-psu3-sz-300/best_model',\n    expected_val=0.7433966054032339\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"avg_tta_score('bd-efficientnet-b03-2015val-psu3-sz-300')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## DenseNet201"},{"metadata":{"trusted":true},"cell_type":"code","source":"MODEL_NAME = 'densenet201_change_zeros_ord_reg_label_smoothing'\n\nIMG_SIZE = 320\nBS = 64\nSEED = 423\nSIGMA_X = 10\n\n\nseed_everything(SEED)\n\n\ndef subtract_gaussian_bg_image(im):\n    # k = np.max(im.shape)/10\n    bg = cv2.GaussianBlur(im ,(0,0) , SIGMA_X)\n    return cv2.addWeighted (im, 4, bg, -4, 128)\n\n\ndef open_img(self, fn, size):\n    \"Open image in `fn`, subclass and overwrite for custom behavior.\"\n    image = cv2.imread(fn)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    image = resize_image(image, size)\n\n    # changing line here.\n    image = subtract_gaussian_bg_image(image)\n    image = Radius_Reduction(image, PARAM)\n    return Image(pil2tensor(image, np.float32).div_(255))\n\n\ndef get_data(seed, size=IMG_SIZE, bs=BS, tfms=((), ())):\n    df_train = get_train_df(seed)\n    \n    ImageList.open = lambda self, fn: open_img(self, fn, size=size)\n\n    data = (\n        ImageList.from_df(\n            path='./',\n            df=df_train,\n            folder='.'\n        )\n    )\n    data = (data.split_from_df('is_valid')\n            .label_from_df('label', label_delim=',', label_cls=ReconstructFixMultiCategoryList)\n            .transform(\n                tfms,\n                resize_method=ResizeMethod.NO,\n                padding_mode='zeros')\n            .databunch(bs=bs)\n            .normalize(imagenet_stats))\n    data.add_test(ImageList.from_df(sample_df, '../input/aptos2019-blindness-detection', folder='test_images', suffix='.png'))\n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = get_data(seed=0, size=IMG_SIZE, bs=BS)\nlearn = cnn_learner(data, models.densenet201, model_dir=\".\", lin_ftrs=[2048], callback_fns=[BnFreeze], pretrained=False)\n\nmake_or_preds(\n    model_name='bd-densenet201-2015val-psu3-2019-val',\n    learner=learn,\n    model_path='../input/bd-densenet201-2015val-psu3-2019-val/best_model',\n    expected_val=0.690059677041891\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = get_data(seed=0, size=IMG_SIZE, bs=BS, tfms=((), (flip_lr(p=1))))\nlearn = cnn_learner(data, models.densenet201, model_dir=\".\", lin_ftrs=[2048], callback_fns=[BnFreeze], pretrained=False)\n\nmake_or_preds(\n    model_name='bd-densenet201-2015val-psu3-2019-val-flip',\n    learner=learn,\n    model_path='../input/bd-densenet201-2015val-psu3-2019-val/best_model',\n    expected_val=0.690059677041891\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"avg_tta_score('bd-densenet201-2015val-psu3-2019-val')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Weighted Blend"},{"metadata":{"trusted":true},"cell_type":"code","source":"columns = ['x_0', 'x_1', 'x_2', 'x_3', 'x_4']\n\neff_b3_300_val_preds_no_flip = pd.read_csv('bd-efficientnet-b03-2015val-psu3-sz-300_val_preds.csv').sort_values('id_code')\neff_b3_300_val_preds_flip = pd.read_csv('bd-efficientnet-b03-2015val-psu3-sz-300-flip_val_preds.csv').sort_values('id_code')\n\neff_b3_300_val_preds = (eff_b3_300_val_preds_flip[columns] + eff_b3_300_val_preds_no_flip[columns]) / 2\n\neff_b3_256_val_preds_no_flip = pd.read_csv('bd-efficientnet-b03-2015val-psu3-sz-256_val_preds.csv').sort_values('id_code')\neff_b3_256_val_preds_flip = pd.read_csv('bd-efficientnet-b03-2015val-psu3-sz-256_val_preds.csv').sort_values('id_code')\n\neff_b3_256_val_preds = (eff_b3_256_val_preds_flip[columns] + eff_b3_256_val_preds_no_flip[columns]) / 2\n\ndensenet_201_val_preds_no_flip = pd.read_csv('bd-densenet201-2015val-psu3-2019-val_val_preds.csv').sort_values('id_code')\ndensenet_201_val_preds_flip = pd.read_csv('bd-densenet201-2015val-psu3-2019-val-flip_val_preds.csv').sort_values('id_code')\n\ndensenet_201_val_preds = (densenet_201_val_preds_flip[columns] + densenet_201_val_preds_no_flip[columns]) / 2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_preds_avg = (\n    eff_b3_300_val_preds.values * 0.4 +\n    eff_b3_256_val_preds.values * 0.4 +\n    densenet_201_val_preds.values * 0.2\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = []\nthres = []\nfor i in np.arange(0, 1, 0.005):\n    thres.append(i)\n    preds.append(cohen_kappa_score(\n        eff_b3_300_val_preds_flip.sort_values('id_code').diagnosis,\n        get_output_preds((val_preds_avg > i)), weights='quadratic'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_preds_and_threshold = sorted(list(zip(preds, thres)), key=lambda x: x[0], reverse=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_preds_and_threshold[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_thresh = 0.45","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Make submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"columns = ['x_0', 'x_1', 'x_2', 'x_3', 'x_4']\n\neff_b3_300_test_preds_no_flip = pd.read_csv('bd-efficientnet-b03-2015val-psu3-sz-300_test_preds.csv')\neff_b3_300_test_preds_flip = pd.read_csv('bd-efficientnet-b03-2015val-psu3-sz-300-flip_test_preds.csv')\n\neff_b3_300_test_preds = (eff_b3_300_test_preds_flip[columns] + eff_b3_300_test_preds_no_flip[columns]) / 2\n\neff_b3_256_test_preds_no_flip = pd.read_csv('bd-efficientnet-b03-2015val-psu3-sz-256_test_preds.csv')\neff_b3_256_test_preds_flip = pd.read_csv('bd-efficientnet-b03-2015val-psu3-sz-256-flip_test_preds.csv')\n\neff_b3_256_test_preds = (eff_b3_256_test_preds_flip[columns] + eff_b3_256_test_preds_no_flip[columns]) / 2\n\ndensenet_201_test_preds_no_flip = pd.read_csv('bd-densenet201-2015val-psu3-2019-val_test_preds.csv').sort_values('id_code')\ndensenet_201_test_preds_flip = pd.read_csv('bd-densenet201-2015val-psu3-2019-val-flip_test_preds.csv').sort_values('id_code')\n\ndensenet_201_test_preds = (densenet_201_test_preds_flip[columns] + densenet_201_test_preds_no_flip[columns]) / 2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds_avg = (\n    eff_b3_300_test_preds.values * 0.4 +\n    eff_b3_256_test_preds.values * 0.4 +\n    densenet_201_test_preds.values * 0.2\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_df.diagnosis = get_output_preds((test_preds_avg > best_thresh)) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.read_csv('submission.csv').head(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.read_csv('submission.csv').diagnosis.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}