{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import os, sys\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport skimage.io\nfrom skimage.transform import resize\nfrom imgaug import augmenters as iaa\nfrom tqdm import tqdm\nimport PIL\nfrom PIL import Image, ImageOps\nimport cv2\nimport multiprocessing as mp\nimport warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.model_zoo as model_zoo\n\nimport re\nimport math\nfrom copy import deepcopy","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def load_pretrained(model, default_cfg, num_classes=1000, in_chans=3, filter_fn=None):\n    if 'url' not in default_cfg or not default_cfg['url']:\n        logging.warning(\"Pretrained model URL is invalid, using random initialization.\")\n        return\n\n    state_dict = model_zoo.load_url(default_cfg['url'])\n\n    if in_chans == 1:\n        conv1_name = default_cfg['first_conv']\n        logging.info('Converting first conv (%s) from 3 to 1 channel' % conv1_name)\n        conv1_weight = state_dict[conv1_name + '.weight']\n        state_dict[conv1_name + '.weight'] = conv1_weight.sum(dim=1, keepdim=True)\n    elif in_chans != 3:\n        assert False, \"Invalid in_chans for pretrained weights\"\n\n    strict = True\n    classifier_name = default_cfg['classifier']\n    if num_classes == 1000 and default_cfg['num_classes'] == 1001:\n        # special case for imagenet trained models with extra background class in pretrained weights\n        classifier_weight = state_dict[classifier_name + '.weight']\n        state_dict[classifier_name + '.weight'] = classifier_weight[1:]\n        classifier_bias = state_dict[classifier_name + '.bias']\n        state_dict[classifier_name + '.bias'] = classifier_bias[1:]\n    elif num_classes != default_cfg['num_classes']:\n        # completely discard fully connected for all other differences between pretrained and created model\n        del state_dict[classifier_name + '.weight']\n        del state_dict[classifier_name + '.bias']\n        strict = False\n\n    if filter_fn is not None:\n        state_dict = filter_fn(state_dict)\n\n    model.load_state_dict(state_dict, strict=strict)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def adaptive_pool_feat_mult(pool_type='avg'):\n    if pool_type == 'catavgmax':\n        return 2\n    else:\n        return 1\n\n\ndef adaptive_avgmax_pool2d(x, output_size=1):\n    x_avg = F.adaptive_avg_pool2d(x, output_size)\n    x_max = F.adaptive_max_pool2d(x, output_size)\n    return 0.5 * (x_avg + x_max)\n\n\ndef adaptive_catavgmax_pool2d(x, output_size=1):\n    x_avg = F.adaptive_avg_pool2d(x, output_size)\n    x_max = F.adaptive_max_pool2d(x, output_size)\n    return torch.cat((x_avg, x_max), 1)\n\n\ndef select_adaptive_pool2d(x, pool_type='avg', output_size=1):\n    \"\"\"Selectable global pooling function with dynamic input kernel size\n    \"\"\"\n    if pool_type == 'avg':\n        x = F.adaptive_avg_pool2d(x, output_size)\n    elif pool_type == 'avgmax':\n        x = adaptive_avgmax_pool2d(x, output_size)\n    elif pool_type == 'catavgmax':\n        x = adaptive_catavgmax_pool2d(x, output_size)\n    elif pool_type == 'max':\n        x = F.adaptive_max_pool2d(x, output_size)\n    else:\n        assert False, 'Invalid pool type: %s' % pool_type\n    return x\n\n\nclass AdaptiveAvgMaxPool2d(nn.Module):\n    def __init__(self, output_size=1):\n        super(AdaptiveAvgMaxPool2d, self).__init__()\n        self.output_size = output_size\n\n    def forward(self, x):\n        return adaptive_avgmax_pool2d(x, self.output_size)\n\n\nclass AdaptiveCatAvgMaxPool2d(nn.Module):\n    def __init__(self, output_size=1):\n        super(AdaptiveCatAvgMaxPool2d, self).__init__()\n        self.output_size = output_size\n\n    def forward(self, x):\n        return adaptive_catavgmax_pool2d(x, self.output_size)\n\n\nclass SelectAdaptivePool2d(nn.Module):\n    \"\"\"Selectable global pooling layer with dynamic input kernel size\n    \"\"\"\n    def __init__(self, output_size=1, pool_type='avg'):\n        super(SelectAdaptivePool2d, self).__init__()\n        self.output_size = output_size\n        self.pool_type = pool_type\n        if pool_type == 'avgmax':\n            self.pool = AdaptiveAvgMaxPool2d(output_size)\n        elif pool_type == 'catavgmax':\n            self.pool = AdaptiveCatAvgMaxPool2d(output_size)\n        elif pool_type == 'max':\n            self.pool = nn.AdaptiveMaxPool2d(output_size)\n        else:\n            if pool_type != 'avg':\n                assert False, 'Invalid pool type: %s' % pool_type\n            self.pool = nn.AdaptiveAvgPool2d(output_size)\n\n    def forward(self, x):\n        return self.pool(x)\n\n    def feat_mult(self):\n        return adaptive_pool_feat_mult(self.pool_type)\n\n    def __repr__(self):\n        return self.__class__.__name__ + ' (' \\\n               + 'output_size=' + str(self.output_size) \\\n               + ', pool_type=' + self.pool_type + ')'\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"IMAGENET_DEFAULT_MEAN = (0.485, 0.456, 0.406)\nIMAGENET_DEFAULT_STD = (0.229, 0.224, 0.225)\n\ndef _cfg(url='', **kwargs):\n    return {\n        'url': url, 'num_classes': 1000, 'input_size': (3, 224, 224), 'pool_size': (7, 7),\n        'crop_pct': 0.875, 'interpolation': 'bicubic',\n        'mean': IMAGENET_DEFAULT_MEAN, 'std': IMAGENET_DEFAULT_STD,\n        'first_conv': 'conv_stem', 'classifier': 'classifier',\n        **kwargs\n    }\n\ndef _is_static_pad(kernel_size, stride=1, dilation=1, **_):\n    return stride == 1 and (dilation * (kernel_size - 1)) % 2 == 0\n\n\ndef _get_padding(kernel_size, stride=1, dilation=1, **_):\n    padding = ((stride - 1) + dilation * (kernel_size - 1)) // 2\n    return padding\n\n\ndef _calc_same_pad(i, k, s, d):\n    return max((math.ceil(i / s) - 1) * s + (k - 1) * d + 1 - i, 0)\n\n\ndef _split_channels(num_chan, num_groups):\n    split = [num_chan // num_groups for _ in range(num_groups)]\n    split[0] += num_chan - sum(split)\n    return split\n\nclass Conv2dSame(nn.Conv2d):\n    \"\"\" Tensorflow like 'SAME' convolution wrapper for 2D convolutions\n    \"\"\"\n    def __init__(self, in_channels, out_channels, kernel_size, stride=1,\n                 padding=0, dilation=1, groups=1, bias=True):\n        super(Conv2dSame, self).__init__(\n            in_channels, out_channels, kernel_size, stride, 0, dilation,\n            groups, bias)\n\n    def forward(self, x):\n        ih, iw = x.size()[-2:]\n        kh, kw = self.weight.size()[-2:]\n        pad_h = _calc_same_pad(ih, kh, self.stride[0], self.dilation[0])\n        pad_w = _calc_same_pad(iw, kw, self.stride[1], self.dilation[1])\n        if pad_h > 0 or pad_w > 0:\n            x = F.pad(x, [pad_w//2, pad_w - pad_w//2, pad_h//2, pad_h - pad_h//2])\n        return F.conv2d(x, self.weight, self.bias, self.stride,\n                        self.padding, self.dilation, self.groups)\n\ndef conv2d_pad(in_chs, out_chs, kernel_size, **kwargs):\n    padding = kwargs.pop('padding', '')\n    kwargs.setdefault('bias', False)\n    if isinstance(padding, str):\n        # for any string padding, the padding will be calculated for you, one of three ways\n        padding = padding.lower()\n        if padding == 'same':\n            # TF compatible 'SAME' padding, has a performance and GPU memory allocation impact\n            if _is_static_pad(kernel_size, **kwargs):\n                # static case, no extra overhead\n                padding = _get_padding(kernel_size, **kwargs)\n                return nn.Conv2d(in_chs, out_chs, kernel_size, padding=padding, **kwargs)\n            else:\n                # dynamic padding\n                return Conv2dSame(in_chs, out_chs, kernel_size, **kwargs)\n        elif padding == 'valid':\n            # 'VALID' padding, same as padding=0\n            return nn.Conv2d(in_chs, out_chs, kernel_size, padding=0, **kwargs)\n        else:\n            # Default to PyTorch style 'same'-ish symmetric padding\n            padding = _get_padding(kernel_size, **kwargs)\n            return nn.Conv2d(in_chs, out_chs, kernel_size, padding=padding, **kwargs)\n    else:\n        # padding was specified as a number or pair\n        return nn.Conv2d(in_chs, out_chs, kernel_size, padding=padding, **kwargs)\n\n\ndef select_conv2d(in_chs, out_chs, kernel_size, **kwargs):\n    assert 'groups' not in kwargs  # only use 'depthwise' bool arg\n    if isinstance(kernel_size, list):\n        # We're going to use only lists for defining the MixedConv2d kernel groups,\n        # ints, tuples, other iterables will continue to pass to normal conv and specify h, w.\n        return MixedConv2d(in_chs, out_chs, kernel_size, **kwargs)\n    else:\n        depthwise = kwargs.pop('depthwise', False)\n        groups = out_chs if depthwise else 1\n        return conv2d_pad(in_chs, out_chs, kernel_size, groups=groups, **kwargs)\n\n\ndef _parse_ksize(ss):\n    if ss.isdigit():\n        return int(ss)\n    else:\n        return [int(k) for k in ss.split('.')]\n\ndef _resolve_bn_args(kwargs):\n    bn_args = _BN_ARGS_TF.copy() if kwargs.pop('bn_tf', False) else _BN_ARGS_PT.copy()\n    bn_momentum = kwargs.pop('bn_momentum', None)\n    if bn_momentum is not None:\n        bn_args['momentum'] = bn_momentum\n    bn_eps = kwargs.pop('bn_eps', None)\n    if bn_eps is not None:\n        bn_args['eps'] = bn_eps\n    return bn_args\n    \ndef _round_channels(channels, multiplier=1.0, divisor=8, channel_min=None):\n    \"\"\"Round number of filters based on depth multiplier.\"\"\"\n    if not multiplier:\n        return channels\n\n    channels *= multiplier\n    channel_min = channel_min or divisor\n    new_channels = max(\n        int(channels + divisor / 2) // divisor * divisor,\n        channel_min)\n    # Make sure that round down does not go down by more than 10%.\n    if new_channels < 0.9 * channels:\n        new_channels += divisor\n    return new_channels\n\ndef _decode_arch_def(arch_def, depth_multiplier=1.0):\n    arch_args = []\n    for stack_idx, block_strings in enumerate(arch_def):\n        assert isinstance(block_strings, list)\n        stack_args = []\n        for block_str in block_strings:\n            assert isinstance(block_str, str)\n            stack_args.extend(_decode_block_str(block_str, depth_multiplier))\n        arch_args.append(stack_args)\n    return arch_args\n\ndef _decode_block_str(block_str, depth_multiplier=1.0):\n    \"\"\" Decode block definition string\n    Gets a list of block arg (dicts) through a string notation of arguments.\n    E.g. ir_r2_k3_s2_e1_i32_o16_se0.25_noskip\n    All args can exist in any order with the exception of the leading string which\n    is assumed to indicate the block type.\n    leading string - block type (\n      ir = InvertedResidual, ds = DepthwiseSep, dsa = DeptwhiseSep with pw act, cn = ConvBnAct)\n    r - number of repeat blocks,\n    k - kernel size,\n    s - strides (1-9),\n    e - expansion ratio,\n    c - output channels,\n    se - squeeze/excitation ratio\n    n - activation fn ('re', 'r6', 'hs', or 'sw')\n    Args:\n        block_str: a string representation of block arguments.\n    Returns:\n        A list of block args (dicts)\n    Raises:\n        ValueError: if the string def not properly specified (TODO)\n    \"\"\"\n    assert isinstance(block_str, str)\n    ops = block_str.split('_')\n    block_type = ops[0]  # take the block type off the front\n    ops = ops[1:]\n    options = {}\n    noskip = False\n    for op in ops:\n        # string options being checked on individual basis, combine if they grow\n        if op == 'noskip':\n            noskip = True\n        elif op.startswith('n'):\n            # activation fn\n            key = op[0]\n            v = op[1:]\n            if v == 're':\n                value = F.relu\n            elif v == 'r6':\n                value = F.relu6\n            elif v == 'hs':\n                value = hard_swish\n            elif v == 'sw':\n                value = swish\n            else:\n                continue\n            options[key] = value\n        else:\n            # all numeric options\n            splits = re.split(r'(\\d.*)', op)\n            if len(splits) >= 2:\n                key, value = splits[:2]\n                options[key] = value\n\n    # if act_fn is None, the model default (passed to model init) will be used\n    act_fn = options['n'] if 'n' in options else None\n    exp_kernel_size = _parse_ksize(options['a']) if 'a' in options else 1\n    pw_kernel_size = _parse_ksize(options['p']) if 'p' in options else 1\n\n    num_repeat = int(options['r'])\n    # each type of block has different valid arguments, fill accordingly\n    if block_type == 'ir':\n        block_args = dict(\n            block_type=block_type,\n            dw_kernel_size=_parse_ksize(options['k']),\n            exp_kernel_size=exp_kernel_size,\n            pw_kernel_size=pw_kernel_size,\n            out_chs=int(options['c']),\n            exp_ratio=float(options['e']),\n            se_ratio=float(options['se']) if 'se' in options else None,\n            stride=int(options['s']),\n            act_fn=act_fn,\n            noskip=noskip,\n        )\n    elif block_type == 'ds' or block_type == 'dsa':\n        block_args = dict(\n            block_type=block_type,\n            dw_kernel_size=_parse_ksize(options['k']),\n            pw_kernel_size=pw_kernel_size,\n            out_chs=int(options['c']),\n            se_ratio=float(options['se']) if 'se' in options else None,\n            stride=int(options['s']),\n            act_fn=act_fn,\n            pw_act=block_type == 'dsa',\n            noskip=block_type == 'dsa' or noskip,\n        )\n    elif block_type == 'cn':\n        block_args = dict(\n            block_type=block_type,\n            kernel_size=int(options['k']),\n            out_chs=int(options['c']),\n            stride=int(options['s']),\n            act_fn=act_fn,\n        )\n    else:\n        assert False, 'Unknown block type (%s)' % block_type\n\n    # return a list of block args expanded by num_repeat and\n    # scaled by depth_multiplier\n    num_repeat = int(math.ceil(num_repeat * depth_multiplier))\n    return [deepcopy(block_args) for _ in range(num_repeat)]\n\n\ndefault_cfgs = {\n    'efficientnet_b0': _cfg(\n        url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/efficientnet_b0-d6904d92.pth'),\n    'efficientnet_b1': _cfg(\n        url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/efficientnet_b1-533bc792.pth',\n        input_size=(3, 240, 240), pool_size=(8, 8), crop_pct=0.882),\n    'efficientnet_b2': _cfg(\n        url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/efficientnet_b2-cf78dc4d.pth',\n        input_size=(3, 260, 260), pool_size=(9, 9), crop_pct=0.890),\n    'efficientnet_b3': _cfg(\n        url='', input_size=(3, 300, 300), pool_size=(10, 10), crop_pct=0.904),\n    'efficientnet_b4': _cfg(\n        url='', input_size=(3, 380, 380), pool_size=(12, 12), crop_pct=0.922),\n    'efficientnet_b5': _cfg(\n        url='', input_size=(3, 456, 456), pool_size=(15, 15), crop_pct=0.934),\n    'efficientnet_b6': _cfg(\n        url='', input_size=(3, 528, 528), pool_size=(17, 17), crop_pct=0.942),\n    'efficientnet_b7': _cfg(\n        url='', input_size=(3, 600, 600), pool_size=(19, 19), crop_pct=0.949),\n    'tf_efficientnet_b0': _cfg(\n        url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b0_aa-827b6e33.pth',\n        input_size=(3, 224, 224)),\n    'tf_efficientnet_b1': _cfg(\n        url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b1_aa-ea7a6ee0.pth',\n        input_size=(3, 240, 240), pool_size=(8, 8), crop_pct=0.882),\n    'tf_efficientnet_b2': _cfg(\n        url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b2_aa-60c94f97.pth',\n        input_size=(3, 260, 260), pool_size=(9, 9), crop_pct=0.890),\n    'tf_efficientnet_b3': _cfg(\n        url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b3_aa-84b4657e.pth',\n        input_size=(3, 300, 300), pool_size=(10, 10), crop_pct=0.904),\n    'tf_efficientnet_b4': _cfg(\n        url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b4_aa-818f208c.pth',\n        input_size=(3, 380, 380), pool_size=(12, 12), crop_pct=0.922),\n    'tf_efficientnet_b5': _cfg(\n        url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b5_aa-99018a74.pth',\n        input_size=(3, 456, 456), pool_size=(15, 15), crop_pct=0.934),\n    'tf_efficientnet_b6': _cfg(\n        url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b6_aa-80ba17e4.pth',\n        input_size=(3, 528, 528), pool_size=(17, 17), crop_pct=0.942),\n    'tf_efficientnet_b7': _cfg(\n        url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b7_aa-076e3472.pth',\n        input_size=(3, 600, 600), pool_size=(19, 19), crop_pct=0.949),\n}\n\n\n_DEBUG = False\n\n# Default args for PyTorch BN impl\n_BN_MOMENTUM_PT_DEFAULT = 0.1\n_BN_EPS_PT_DEFAULT = 1e-5\n_BN_ARGS_PT = dict(momentum=_BN_MOMENTUM_PT_DEFAULT, eps=_BN_EPS_PT_DEFAULT)\n\n\n_BN_MOMENTUM_TF_DEFAULT = 1 - 0.99\n_BN_EPS_TF_DEFAULT = 1e-3\n_BN_ARGS_TF = dict(momentum=_BN_MOMENTUM_TF_DEFAULT, eps=_BN_EPS_TF_DEFAULT)\n\ndef _initialize_weight_goog(m):\n    # weight init as per Tensorflow Official impl\n    # https://github.com/tensorflow/tpu/blob/master/models/official/mnasnet/mnasnet_model.py\n    if isinstance(m, nn.Conv2d):\n        n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels  # fan-out\n        m.weight.data.normal_(0, math.sqrt(2.0 / n))\n        if m.bias is not None:\n            m.bias.data.zero_()\n    elif isinstance(m, nn.BatchNorm2d):\n        m.weight.data.fill_(1.0)\n        m.bias.data.zero_()\n    elif isinstance(m, nn.Linear):\n        n = m.weight.size(0)  # fan-out\n        init_range = 1.0 / math.sqrt(n)\n        m.weight.data.uniform_(-init_range, init_range)\n        m.bias.data.zero_()\n\ndef swish(x, inplace=False):\n    if inplace:\n        return x.mul_(x.sigmoid())\n    else:\n        return x * x.sigmoid()\n\n\ndef sigmoid(x, inplace=False):\n    return x.sigmoid_() if inplace else x.sigmoid()\n\nclass InvertedResidual(nn.Module):\n    \"\"\" Inverted residual block w/ optional SE\"\"\"\n\n    def __init__(self, in_chs, out_chs, dw_kernel_size=3,\n                 stride=1, pad_type='', act_fn=F.relu, noskip=False,\n                 exp_ratio=1.0, exp_kernel_size=1, pw_kernel_size=1,\n                 se_ratio=0., se_reduce_mid=False, se_gate_fn=sigmoid,\n                 shuffle_type=None, bn_args=_BN_ARGS_PT, drop_connect_rate=0.):\n        super(InvertedResidual, self).__init__()\n        mid_chs = int(in_chs * exp_ratio)\n        self.has_se = se_ratio is not None and se_ratio > 0.\n        self.has_residual = (in_chs == out_chs and stride == 1) and not noskip\n        self.act_fn = act_fn\n        self.drop_connect_rate = drop_connect_rate\n\n        # Point-wise expansion\n        self.conv_pw = select_conv2d(in_chs, mid_chs, exp_kernel_size, padding=pad_type)\n        self.bn1 = nn.BatchNorm2d(mid_chs, **bn_args)\n\n        self.shuffle_type = shuffle_type\n        if shuffle_type is not None and isinstance(exp_kernel_size, list):\n            self.shuffle = ChannelShuffle(len(exp_kernel_size))\n\n        # Depth-wise convolution\n        self.conv_dw = select_conv2d(\n            mid_chs, mid_chs, dw_kernel_size, stride=stride, padding=pad_type, depthwise=True)\n        self.bn2 = nn.BatchNorm2d(mid_chs, **bn_args)\n\n        # Squeeze-and-excitation\n        if self.has_se:\n            se_base_chs = mid_chs if se_reduce_mid else in_chs\n            self.se = SqueezeExcite(\n                mid_chs, reduce_chs=max(1, int(se_base_chs * se_ratio)), act_fn=act_fn, gate_fn=se_gate_fn)\n\n        # Point-wise linear projection\n        self.conv_pwl = select_conv2d(mid_chs, out_chs, pw_kernel_size, padding=pad_type)\n        self.bn3 = nn.BatchNorm2d(out_chs, **bn_args)\n\n    def forward(self, x):\n        residual = x\n\n        # Point-wise expansion\n        x = self.conv_pw(x)\n        x = self.bn1(x)\n        x = self.act_fn(x, inplace=True)\n\n        # FIXME haven't tried this yet\n        # for channel shuffle when using groups with pointwise convs as per FBNet variants\n        if self.shuffle_type == \"mid\":\n            x = self.shuffle(x)\n\n        # Depth-wise convolution\n        x = self.conv_dw(x)\n        x = self.bn2(x)\n        x = self.act_fn(x, inplace=True)\n\n        # Squeeze-and-excitation\n        if self.has_se:\n            x = self.se(x)\n\n        # Point-wise linear projection\n        x = self.conv_pwl(x)\n        x = self.bn3(x)\n\n        if self.has_residual:\n            if self.drop_connect_rate > 0.:\n                x = drop_connect(x, self.training, self.drop_connect_rate)\n            x += residual\n\n        # NOTE maskrcnn_benchmark building blocks have an SE module defined here for some variants\n\n        return x\n\n\nclass GenEfficientNet(nn.Module):\n    \"\"\" Generic EfficientNet\n    An implementation of efficient network architectures, in many cases mobile optimized networks:\n      * MobileNet-V1\n      * MobileNet-V2\n      * MobileNet-V3\n      * MnasNet A1, B1, and small\n      * FBNet A, B, and C\n      * ChamNet (arch details are murky)\n      * Single-Path NAS Pixel1\n      * EfficientNet B0-B7\n      * MixNet S, M, L\n    \"\"\"\n\n    def __init__(self, block_args, num_classes=1000, in_chans=3, stem_size=32, num_features=1280,\n                 channel_multiplier=1.0, channel_divisor=8, channel_min=None,\n                 pad_type='', act_fn=F.relu, drop_rate=0., drop_connect_rate=0.,\n                 se_gate_fn=sigmoid, se_reduce_mid=False, bn_args=_BN_ARGS_PT,\n                 global_pool='avg', head_conv='default', weight_init='goog'):\n        super(GenEfficientNet, self).__init__()\n        self.num_classes = num_classes\n        self.drop_rate = drop_rate\n        self.act_fn = act_fn\n        self.num_features = num_features\n\n        stem_size = _round_channels(stem_size, channel_multiplier, channel_divisor, channel_min)\n        self.conv_stem = select_conv2d(in_chans, stem_size, 3, stride=2, padding=pad_type)\n        self.bn1 = nn.BatchNorm2d(stem_size, **bn_args)\n        in_chs = stem_size\n\n        builder = _BlockBuilder(\n            channel_multiplier, channel_divisor, channel_min,\n            pad_type, act_fn, se_gate_fn, se_reduce_mid,\n            bn_args, drop_connect_rate, verbose=_DEBUG)\n        self.blocks = nn.Sequential(*builder(in_chs, block_args))\n        in_chs = builder.in_chs\n\n        if not head_conv or head_conv == 'none':\n            self.efficient_head = False\n            self.conv_head = None\n            assert in_chs == self.num_features\n        else:\n            self.efficient_head = head_conv == 'efficient'\n            self.conv_head = select_conv2d(in_chs, self.num_features, 1, padding=pad_type)\n            self.bn2 = None if self.efficient_head else nn.BatchNorm2d(self.num_features, **bn_args)\n\n        self.global_pool = SelectAdaptivePool2d(pool_type=global_pool)\n        self.classifier = nn.Linear(self.num_features * self.global_pool.feat_mult(), self.num_classes)\n\n        for m in self.modules():\n            if weight_init == 'goog':\n                _initialize_weight_goog(m)\n            else:\n                _initialize_weight_default(m)\n\n    def get_classifier(self):\n        return self.classifier\n\n    def reset_classifier(self, num_classes, global_pool='avg'):\n        self.global_pool = SelectAdaptivePool2d(pool_type=global_pool)\n        self.num_classes = num_classes\n        del self.classifier\n        if num_classes:\n            self.classifier = nn.Linear(\n                self.num_features * self.global_pool.feat_mult(), num_classes)\n        else:\n            self.classifier = None\n\n    def forward_features(self, x, pool=True):\n        x = self.conv_stem(x)\n        x = self.bn1(x)\n        x = self.act_fn(x, inplace=True)\n        x = self.blocks(x)\n        if self.efficient_head:\n            # efficient head, currently only mobilenet-v3 performs pool before last 1x1 conv\n            x = self.global_pool(x)  # always need to pool here regardless of flag\n            x = self.conv_head(x)\n            # no BN\n            x = self.act_fn(x, inplace=True)\n            if pool:\n                # expect flattened output if pool is true, otherwise keep dim\n                x = x.view(x.size(0), -1)\n        else:\n            if self.conv_head is not None:\n                x = self.conv_head(x)\n                x = self.bn2(x)\n            x = self.act_fn(x, inplace=True)\n            if pool:\n                x = self.global_pool(x)\n                x = x.view(x.size(0), -1)\n        return x\n\n    def forward(self, x):\n        x = self.forward_features(x)\n        if self.drop_rate > 0.:\n            x = F.dropout(x, p=self.drop_rate, training=self.training)\n        return self.classifier(x)\n    \nclass DepthwiseSeparableConv(nn.Module):\n    \"\"\" DepthwiseSeparable block\n    Used for DS convs in MobileNet-V1 and in the place of IR blocks with an expansion\n    factor of 1.0. This is an alternative to having a IR with optional first pw conv.\n    \"\"\"\n    def __init__(self, in_chs, out_chs, dw_kernel_size=3,\n                 stride=1, pad_type='', act_fn=F.relu, noskip=False,\n                 pw_kernel_size=1, pw_act=False,\n                 se_ratio=0., se_gate_fn=sigmoid,\n                 bn_args=_BN_ARGS_PT, drop_connect_rate=0.):\n        super(DepthwiseSeparableConv, self).__init__()\n        assert stride in [1, 2]\n        self.has_se = se_ratio is not None and se_ratio > 0.\n        self.has_residual = (stride == 1 and in_chs == out_chs) and not noskip\n        self.has_pw_act = pw_act  # activation after point-wise conv\n        self.act_fn = act_fn\n        self.drop_connect_rate = drop_connect_rate\n\n        self.conv_dw = select_conv2d(\n            in_chs, in_chs, dw_kernel_size, stride=stride, padding=pad_type, depthwise=True)\n        self.bn1 = nn.BatchNorm2d(in_chs, **bn_args)\n\n        # Squeeze-and-excitation\n        if self.has_se:\n            self.se = SqueezeExcite(\n                in_chs, reduce_chs=max(1, int(in_chs * se_ratio)), act_fn=act_fn, gate_fn=se_gate_fn)\n\n        self.conv_pw = select_conv2d(in_chs, out_chs, pw_kernel_size, padding=pad_type)\n        self.bn2 = nn.BatchNorm2d(out_chs, **bn_args)\n\n    def forward(self, x):\n        residual = x\n\n        x = self.conv_dw(x)\n        x = self.bn1(x)\n        x = self.act_fn(x, inplace=True)\n\n        if self.has_se:\n            x = self.se(x)\n\n        x = self.conv_pw(x)\n        x = self.bn2(x)\n        if self.has_pw_act:\n            x = self.act_fn(x, inplace=True)\n\n        if self.has_residual:\n            if self.drop_connect_rate > 0.:\n                x = drop_connect(x, self.training, self.drop_connect_rate)\n            x += residual\n        return x\n    \n\n    \nclass SqueezeExcite(nn.Module):\n    def __init__(self, in_chs, reduce_chs=None, act_fn=F.relu, gate_fn=sigmoid):\n        super(SqueezeExcite, self).__init__()\n        self.act_fn = act_fn\n        self.gate_fn = gate_fn\n        reduced_chs = reduce_chs or in_chs\n        self.conv_reduce = nn.Conv2d(in_chs, reduced_chs, 1, bias=True)\n        self.conv_expand = nn.Conv2d(reduced_chs, in_chs, 1, bias=True)\n\n    def forward(self, x):\n        # NOTE adaptiveavgpool can be used here, but seems to cause issues with NVIDIA AMP performance\n        x_se = x.view(x.size(0), x.size(1), -1).mean(-1).view(x.size(0), x.size(1), 1, 1)\n        x_se = self.conv_reduce(x_se)\n        x_se = self.act_fn(x_se, inplace=True)\n        x_se = self.conv_expand(x_se)\n        x = x * self.gate_fn(x_se)\n        return x\n\nclass _BlockBuilder:\n    \"\"\" Build Trunk Blocks\n    This ended up being somewhat of a cross between\n    https://github.com/tensorflow/tpu/blob/master/models/official/mnasnet/mnasnet_models.py\n    and\n    https://github.com/facebookresearch/maskrcnn-benchmark/blob/master/maskrcnn_benchmark/modeling/backbone/fbnet_builder.py\n    \"\"\"\n    def __init__(self, channel_multiplier=1.0, channel_divisor=8, channel_min=None,\n                 pad_type='', act_fn=None, se_gate_fn=sigmoid, se_reduce_mid=False,\n                 bn_args=_BN_ARGS_PT, drop_connect_rate=0., verbose=False):\n        self.channel_multiplier = channel_multiplier\n        self.channel_divisor = channel_divisor\n        self.channel_min = channel_min\n        self.pad_type = pad_type\n        self.act_fn = act_fn\n        self.se_gate_fn = se_gate_fn\n        self.se_reduce_mid = se_reduce_mid\n        self.bn_args = bn_args\n        self.drop_connect_rate = drop_connect_rate\n        self.verbose = verbose\n\n        # updated during build\n        self.in_chs = None\n        self.block_idx = 0\n        self.block_count = 0\n\n    def _round_channels(self, chs):\n        return _round_channels(chs, self.channel_multiplier, self.channel_divisor, self.channel_min)\n\n    def _make_block(self, ba):\n        bt = ba.pop('block_type')\n        ba['in_chs'] = self.in_chs\n        ba['out_chs'] = self._round_channels(ba['out_chs'])\n        ba['bn_args'] = self.bn_args\n        ba['pad_type'] = self.pad_type\n        # block act fn overrides the model default\n        ba['act_fn'] = ba['act_fn'] if ba['act_fn'] is not None else self.act_fn\n        assert ba['act_fn'] is not None\n        if bt == 'ir':\n            ba['drop_connect_rate'] = self.drop_connect_rate * self.block_idx / self.block_count\n            ba['se_gate_fn'] = self.se_gate_fn\n            ba['se_reduce_mid'] = self.se_reduce_mid\n            if self.verbose:\n                logging.info('  InvertedResidual {}, Args: {}'.format(self.block_idx, str(ba)))\n            block = InvertedResidual(**ba)\n        elif bt == 'ds' or bt == 'dsa':\n            ba['drop_connect_rate'] = self.drop_connect_rate * self.block_idx / self.block_count\n            if self.verbose:\n                logging.info('  DepthwiseSeparable {}, Args: {}'.format(self.block_idx, str(ba)))\n            block = DepthwiseSeparableConv(**ba)\n        elif bt == 'cn':\n            if self.verbose:\n                logging.info('  ConvBnAct {}, Args: {}'.format(self.block_idx, str(ba)))\n            block = ConvBnAct(**ba)\n        else:\n            assert False, 'Uknkown block type (%s) while building model.' % bt\n        self.in_chs = ba['out_chs']  # update in_chs for arg of next block\n\n        return block\n\n    def _make_stack(self, stack_args):\n        blocks = []\n        # each stack (stage) contains a list of block arguments\n        for i, ba in enumerate(stack_args):\n            if self.verbose:\n                logging.info(' Block: {}'.format(i))\n            if i >= 1:\n                # only the first block in any stack can have a stride > 1\n                ba['stride'] = 1\n            block = self._make_block(ba)\n            blocks.append(block)\n            self.block_idx += 1  # incr global idx (across all stacks)\n        return nn.Sequential(*blocks)\n\n    def __call__(self, in_chs, block_args):\n        \"\"\" Build the blocks\n        Args:\n            in_chs: Number of input-channels passed to first block\n            block_args: A list of lists, outer list defines stages, inner\n                list contains strings defining block configuration(s)\n        Return:\n             List of block stacks (each stack wrapped in nn.Sequential)\n        \"\"\"\n        if self.verbose:\n            logging.info('Building model trunk with %d stages...' % len(block_args))\n        self.in_chs = in_chs\n        self.block_count = sum([len(x) for x in block_args])\n        self.block_idx = 0\n        blocks = []\n        # outer list of block_args defines the stacks ('stages' by some conventions)\n        for stack_idx, stack in enumerate(block_args):\n            if self.verbose:\n                logging.info('Stack: {}'.format(stack_idx))\n            assert isinstance(stack, list)\n            stack = self._make_stack(stack)\n            blocks.append(stack)\n        return blocks\n\n    \ndef _gen_efficientnet(channel_multiplier=1.0, depth_multiplier=1.0, num_classes=1000, **kwargs):\n    \"\"\"Creates an EfficientNet model.\n    Ref impl: https://github.com/tensorflow/tpu/blob/master/models/official/efficientnet/efficientnet_model.py\n    Paper: https://arxiv.org/abs/1905.11946\n    EfficientNet params\n    name: (channel_multiplier, depth_multiplier, resolution, dropout_rate)\n    'efficientnet-b0': (1.0, 1.0, 224, 0.2),\n    'efficientnet-b1': (1.0, 1.1, 240, 0.2),\n    'efficientnet-b2': (1.1, 1.2, 260, 0.3),\n    'efficientnet-b3': (1.2, 1.4, 300, 0.3),\n    'efficientnet-b4': (1.4, 1.8, 380, 0.4),\n    'efficientnet-b5': (1.6, 2.2, 456, 0.4),\n    'efficientnet-b6': (1.8, 2.6, 528, 0.5),\n    'efficientnet-b7': (2.0, 3.1, 600, 0.5),\n    Args:\n      channel_multiplier: multiplier to number of channels per layer\n      depth_multiplier: multiplier to number of repeats per stage\n    \"\"\"\n    arch_def = [\n        ['ds_r1_k3_s1_e1_c16_se0.25'],\n        ['ir_r2_k3_s2_e6_c24_se0.25'],\n        ['ir_r2_k5_s2_e6_c40_se0.25'],\n        ['ir_r3_k3_s2_e6_c80_se0.25'],\n        ['ir_r3_k5_s1_e6_c112_se0.25'],\n        ['ir_r4_k5_s2_e6_c192_se0.25'],\n        ['ir_r1_k3_s1_e6_c320_se0.25'],\n    ]\n    # NOTE: other models in the family didn't scale the feature count\n    num_features = _round_channels(1280, channel_multiplier, 8, None)\n    model = GenEfficientNet(\n        _decode_arch_def(arch_def, depth_multiplier),\n        num_classes=num_classes,\n        stem_size=32,\n        channel_multiplier=channel_multiplier,\n        num_features=num_features,\n        bn_args=_resolve_bn_args(kwargs),\n        act_fn=swish,\n        **kwargs\n    )\n    return model  \n\n\ndef tf_efficientnet_b0(pretrained=False, num_classes=1000, in_chans=3, **kwargs):\n    \"\"\" EfficientNet-B0. Tensorflow compatible variant  \"\"\"\n    default_cfg = default_cfgs['tf_efficientnet_b0']\n    kwargs['bn_eps'] = _BN_EPS_TF_DEFAULT\n    kwargs['pad_type'] = 'same'\n    model = _gen_efficientnet(\n        channel_multiplier=1.0, depth_multiplier=1.0,\n        num_classes=num_classes, in_chans=in_chans, **kwargs)\n    model.default_cfg = default_cfg\n    if pretrained:\n        load_pretrained(model, default_cfg, num_classes, in_chans)\n    return model\n\n\ndef tf_efficientnet_b1(pretrained=False, num_classes=1000, in_chans=3, **kwargs):\n    \"\"\" EfficientNet-B1. Tensorflow compatible variant  \"\"\"\n    default_cfg = default_cfgs['tf_efficientnet_b1']\n    kwargs['bn_eps'] = _BN_EPS_TF_DEFAULT\n    kwargs['pad_type'] = 'same'\n    model = _gen_efficientnet(\n        channel_multiplier=1.0, depth_multiplier=1.1,\n        num_classes=num_classes, in_chans=in_chans, **kwargs)\n    model.default_cfg = default_cfg\n    if pretrained:\n        load_pretrained(model, default_cfg, num_classes, in_chans)\n    return model\n\n\ndef tf_efficientnet_b2(pretrained=False, num_classes=1000, in_chans=3, **kwargs):\n    \"\"\" EfficientNet-B2. Tensorflow compatible variant  \"\"\"\n    default_cfg = default_cfgs['tf_efficientnet_b2']\n    kwargs['bn_eps'] = _BN_EPS_TF_DEFAULT\n    kwargs['pad_type'] = 'same'\n    model = _gen_efficientnet(\n        channel_multiplier=1.1, depth_multiplier=1.2,\n        num_classes=num_classes, in_chans=in_chans, **kwargs)\n    model.default_cfg = default_cfg\n    if pretrained:\n        load_pretrained(model, default_cfg, num_classes, in_chans)\n    return model\n\n\ndef tf_efficientnet_b3(pretrained=False, num_classes=1000, in_chans=3, **kwargs):\n    \"\"\" EfficientNet-B3. Tensorflow compatible variant \"\"\"\n    default_cfg = default_cfgs['tf_efficientnet_b3']\n    kwargs['bn_eps'] = _BN_EPS_TF_DEFAULT\n    kwargs['pad_type'] = 'same'\n    model = _gen_efficientnet(\n        channel_multiplier=1.2, depth_multiplier=1.4,\n        num_classes=num_classes, in_chans=in_chans, **kwargs)\n    model.default_cfg = default_cfg\n    if pretrained:\n        load_pretrained(model, default_cfg, num_classes, in_chans)\n    return model\n\n\ndef tf_efficientnet_b4(pretrained=False, num_classes=1000, in_chans=3, **kwargs):\n    \"\"\" EfficientNet-B4. Tensorflow compatible variant \"\"\"\n    default_cfg = default_cfgs['tf_efficientnet_b4']\n    kwargs['bn_eps'] = _BN_EPS_TF_DEFAULT\n    kwargs['pad_type'] = 'same'\n    model = _gen_efficientnet(\n        channel_multiplier=1.4, depth_multiplier=1.8,\n        num_classes=num_classes, in_chans=in_chans, **kwargs)\n    model.default_cfg = default_cfg\n    if pretrained:\n        load_pretrained(model, default_cfg, num_classes, in_chans)\n    return model\n\n\ndef tf_efficientnet_b5(pretrained=False, num_classes=1000, in_chans=3, **kwargs):\n    \"\"\" EfficientNet-B5. Tensorflow compatible variant \"\"\"\n    default_cfg = default_cfgs['tf_efficientnet_b5']\n    kwargs['bn_eps'] = _BN_EPS_TF_DEFAULT\n    kwargs['pad_type'] = 'same'\n    model = _gen_efficientnet(\n        channel_multiplier=1.6, depth_multiplier=2.2,\n        num_classes=num_classes, in_chans=in_chans, **kwargs)\n    model.default_cfg = default_cfg\n    if pretrained:\n        load_pretrained(model, default_cfg, num_classes, in_chans)\n    return model\n\n\ndef tf_efficientnet_b6(pretrained=False, num_classes=1000, in_chans=3, **kwargs):\n    \"\"\" EfficientNet-B6. Tensorflow compatible variant \"\"\"\n    # NOTE for train, drop_rate should be 0.5\n    default_cfg = default_cfgs['tf_efficientnet_b6']\n    kwargs['bn_eps'] = _BN_EPS_TF_DEFAULT\n    kwargs['pad_type'] = 'same'\n    model = _gen_efficientnet(\n        channel_multiplier=1.8, depth_multiplier=2.6,\n        num_classes=num_classes, in_chans=in_chans, **kwargs)\n    model.default_cfg = default_cfg\n    if pretrained:\n        load_pretrained(model, default_cfg, num_classes, in_chans)\n    return model\n\n\ndef tf_efficientnet_b7(pretrained=False, num_classes=1000, in_chans=3, **kwargs):\n    \"\"\" EfficientNet-B7. Tensorflow compatible variant \"\"\"\n    # NOTE for train, drop_rate should be 0.5\n    default_cfg = default_cfgs['tf_efficientnet_b7']\n    kwargs['bn_eps'] = _BN_EPS_TF_DEFAULT\n    kwargs['pad_type'] = 'same'\n    model = _gen_efficientnet(\n        channel_multiplier=2.0, depth_multiplier=3.1,\n        num_classes=num_classes, in_chans=in_chans, **kwargs)\n    model.default_cfg = default_cfg\n    if pretrained:\n        load_pretrained(model, default_cfg, num_classes, in_chans)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from fastai import *\nfrom fastai.vision import *\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport os\nimport scipy as sp\nfrom functools import partial\nfrom sklearn import metrics\nfrom collections import Counter\nfrom fastai.callbacks import *\n\nimport PIL\nimport cv2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Set seed for all\ndef seed_everything(seed=77):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_everything()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"LOCAL = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_df():\n    if LOCAL:\n        base_image_dir = os.path.join('.', 'data/')\n    else:\n        base_image_dir = os.path.join('..', 'input/aptos2019-blindness-detection/')\n    \n    train_dir = os.path.join(base_image_dir,'train_images/')\n    \n    df = pd.read_csv(os.path.join(base_image_dir, 'train.csv'))\n    df['path'] = df['id_code'].map(lambda x: os.path.join(train_dir,'{}.png'.format(x)))\n    df = df.drop(columns=['id_code'])\n    df = df.sample(frac=1).reset_index(drop=True) #shuffle dataframe\n    \n    test_df = pd.read_csv(os.path.join(base_image_dir, 'sample_submission.csv'))\n    \n    return df, test_df\n\ndf, test_df = get_df()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import cohen_kappa_score\ndef qk(y_pred, y):\n    return torch.tensor(cohen_kappa_score(torch.round(y_pred), y,\n                                          weights='quadratic'), device='cuda:0')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir models\n!cp '../input/effinet-b5-aptos/Effi0_12e_pretrained.pth' \"models/\"\n!cp '../input/effinet-b5-aptos/Effi1_12e_pretrained.pth' \"models/\"\n!cp '../input/effinet-b5-aptos/Effi2_10e_pretrained.pth' \"models/\"\n!cp '../input/effinet-b5-aptos/Effi3_10e_pretrained.pth' \"models/\"\n!cp '../input/effinet-b5-aptos/best_effi6_224.pth' \"models/\"\n!cp '../input/starter-kernel-for-0-79/models/abcdef.pth' \"models/\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#you can play around with tfms and image sizes\nbs = 64\nsz = 224\ntfms = get_transforms(do_flip=False, max_warp=0, max_rotate=360, xtra_tfms=[flip_lr(p=0.5), zoom(scale=(1.25, 1.35), p=0.6)])\n\ndata5 = (ImageList.from_df(df=df,path='./',cols='path') \n        .split_by_rand_pct(0.2) \n        .label_from_df(cols='diagnosis',label_cls=FloatList) \n        .transform(tfms,size=sz,resize_method=ResizeMethod.SQUISH,padding_mode='zeros')\n        .databunch(bs=bs)\n        .normalize(imagenet_stats)  \n       )\n\nif not LOCAL:\n    data5.add_test(ImageList.from_df(test_df, '../input/aptos2019-blindness-detection/',\n                                      folder='test_images',\n                                      suffix='.png'))\n    \nlearn5 = Learner(data5, \n                tf_efficientnet_b0(pretrained=False, num_classes=1), \n                metrics = [qk], \n                model_dir=\"models\").to_fp16()\nlearn5.load('Effi0_12e_pretrained');\npreds5,y = learn5.get_preds(DatasetType.Test)\ndel learn5, data5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#you can play around with tfms and image sizes\nbs = 64\nsz = 240\ntfms = get_transforms(do_flip=False, max_warp=0, max_rotate=360, xtra_tfms=[flip_lr(p=0.5), zoom(scale=(1.25, 1.35), p=0.6)])\n\ndata6 = (ImageList.from_df(df=df,path='./',cols='path') \n        .split_by_rand_pct(0.2) \n        .label_from_df(cols='diagnosis',label_cls=FloatList) \n        .transform(tfms,size=sz,resize_method=ResizeMethod.SQUISH,padding_mode='zeros')\n        .databunch(bs=bs)\n        .normalize(imagenet_stats)  \n       )\n\nif not LOCAL:\n    data6.add_test(ImageList.from_df(test_df, '../input/aptos2019-blindness-detection/',\n                                      folder='test_images',\n                                      suffix='.png'))\n    \nlearn6 = Learner(data6, \n                tf_efficientnet_b1(pretrained=False, num_classes=1), \n                metrics = [qk], \n                model_dir=\"models\").to_fp16()\nlearn6.load('Effi1_12e_pretrained');\npreds6,y = learn6.get_preds(DatasetType.Test)\ndel learn6, data6","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#you can play around with tfms and image sizes\nbs = 32\nsz = 300\ntfms = get_transforms(do_flip=False, max_warp=0, max_rotate=360, xtra_tfms=[flip_lr(p=0.5), zoom(scale=(1.25, 1.35), p=0.6)])\n\ndata1 = (ImageList.from_df(df=df,path='./',cols='path') \n        .split_by_rand_pct(0.2) \n        .label_from_df(cols='diagnosis',label_cls=FloatList) \n        .transform(tfms,size=sz,resize_method=ResizeMethod.SQUISH,padding_mode='zeros')\n        .databunch(bs=bs)\n        .normalize(imagenet_stats)  \n       )\n\nif not LOCAL:\n    data1.add_test(ImageList.from_df(test_df, '../input/aptos2019-blindness-detection/',\n                                      folder='test_images',\n                                      suffix='.png'))\n    \nlearn1 = Learner(data1, \n                tf_efficientnet_b3(pretrained=False, num_classes=1), \n                metrics = [qk], \n                model_dir=\"models\").to_fp16()\nlearn1.load('Effi3_10e_pretrained');\npreds1,y = learn1.get_preds(DatasetType.Test)\ndel learn1, data1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data2 = (ImageList.from_df(df=df,path='./',cols='path') \n        .split_by_rand_pct(0.2) \n        .label_from_df(cols='diagnosis',label_cls=FloatList) \n        .transform(tfms,size=sz,resize_method=ResizeMethod.SQUISH,padding_mode='zeros')\n        .databunch(bs=bs)\n        .normalize(imagenet_stats)  \n       )\n\nif not LOCAL:\n    data2.add_test(ImageList.from_df(test_df, '../input/aptos2019-blindness-detection/',\n                                      folder='test_images',\n                                      suffix='.png'))\n    \nbs = 32\nsz = 224\ntfms = get_transforms(do_flip=True,flip_vert=True)\nlearn2 = Learner(data2, \n                tf_efficientnet_b2(pretrained=False, num_classes=1), \n                metrics = [qk], \n                model_dir=\"models\").to_fp16()\nlearn2.load('Effi2_10e_pretrained');\n\npreds2,y = learn2.get_preds(DatasetType.Test)\n\ndel learn2, data2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\ngc.collect();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#you can play around with tfms and image sizes\nbs = 32\nsz = 224\ntfms = get_transforms(do_flip=True,flip_vert=True)# max_warp=0, max_rotate=360, xtra_tfms=[zoom(scale=1.35, p=1)])\n\ndata3 = (ImageList.from_df(df=df,path='./',cols='path') \n        .split_by_rand_pct(0.2) \n        .label_from_df(cols='diagnosis',label_cls=FloatList) \n        .transform(tfms,size=sz,resize_method=ResizeMethod.SQUISH,padding_mode='zeros')\n        .databunch(bs=bs)\n        .normalize(imagenet_stats)  \n       )\n\nif not LOCAL:\n    data3.add_test(ImageList.from_df(test_df, '../input/aptos2019-blindness-detection',\n                                      folder='test_images',\n                                      suffix='.png'))\n\nlearn3 = Learner(data3, \n                tf_efficientnet_b6(pretrained=False, num_classes=1), \n                metrics = [qk], \n                model_dir=\"models\").to_fp16()\nlearn3.load('best_effi6_224');\npreds3,y = learn3.get_preds(DatasetType.Test)\ndel learn3, data3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"GlobalParams = collections.namedtuple('GlobalParams', [\n    'batch_norm_momentum', 'batch_norm_epsilon', 'dropout_rate',\n    'num_classes', 'width_coefficient', 'depth_coefficient',\n    'depth_divisor', 'min_depth', 'drop_connect_rate', 'image_size'])\n\n\n# Parameters for an individual model block\nBlockArgs = collections.namedtuple('BlockArgs', [\n    'kernel_size', 'num_repeat', 'input_filters', 'output_filters',\n    'expand_ratio', 'id_skip', 'stride', 'se_ratio'])\n\n\n# Change namedtuple defaults\nGlobalParams.__new__.__defaults__ = (None,) * len(GlobalParams._fields)\nBlockArgs.__new__.__defaults__ = (None,) * len(BlockArgs._fields)\n\n\ndef relu_fn(x):\n    \"\"\" Swish activation function \"\"\"\n    return x * torch.sigmoid(x)\n\n\ndef round_filters(filters, global_params):\n    \"\"\" Calculate and round number of filters based on depth multiplier. \"\"\"\n    multiplier = global_params.width_coefficient\n    if not multiplier:\n        return filters\n    divisor = global_params.depth_divisor\n    min_depth = global_params.min_depth\n    filters *= multiplier\n    min_depth = min_depth or divisor\n    new_filters = max(min_depth, int(filters + divisor / 2) // divisor * divisor)\n    if new_filters < 0.9 * filters:  # prevent rounding by more than 10%\n        new_filters += divisor\n    return int(new_filters)\n\n\ndef round_repeats(repeats, global_params):\n    \"\"\" Round number of filters based on depth multiplier. \"\"\"\n    multiplier = global_params.depth_coefficient\n    if not multiplier:\n        return repeats\n    return int(math.ceil(multiplier * repeats))\n\n\ndef drop_connect(inputs, p, training):\n    \"\"\" Drop connect. \"\"\"\n    if not training: return inputs\n    batch_size = inputs.shape[0]\n    keep_prob = 1 - p\n    random_tensor = keep_prob\n    random_tensor += torch.rand([batch_size, 1, 1, 1], dtype=inputs.dtype, device=inputs.device)\n    binary_tensor = torch.floor(random_tensor)\n    output = inputs / keep_prob * binary_tensor\n    return output\n\n\ndef get_same_padding_conv2d(image_size=None):\n    \"\"\" Chooses static padding if you have specified an image size, and dynamic padding otherwise.\n        Static padding is necessary for ONNX exporting of models. \"\"\"\n    if image_size is None:\n        return Conv2dDynamicSamePadding\n    else:\n        return partial(Conv2dStaticSamePadding, image_size=image_size)\n\nclass Conv2dDynamicSamePadding(nn.Conv2d):\n    \"\"\" 2D Convolutions like TensorFlow, for a dynamic image size \"\"\"\n    def __init__(self, in_channels, out_channels, kernel_size, stride=1, dilation=1, groups=1, bias=True):\n        super().__init__(in_channels, out_channels, kernel_size, stride, 0, dilation, groups, bias)\n        self.stride = self.stride if len(self.stride) == 2 else [self.stride[0]]*2\n\n    def forward(self, x):\n        ih, iw = x.size()[-2:]\n        kh, kw = self.weight.size()[-2:]\n        sh, sw = self.stride\n        oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)\n        pad_h = max((oh - 1) * self.stride[0] + (kh - 1) * self.dilation[0] + 1 - ih, 0)\n        pad_w = max((ow - 1) * self.stride[1] + (kw - 1) * self.dilation[1] + 1 - iw, 0)\n        if pad_h > 0 or pad_w > 0:\n            x = F.pad(x, [pad_w//2, pad_w - pad_w//2, pad_h//2, pad_h - pad_h//2])\n        return F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)\n\n\nclass Conv2dStaticSamePadding(nn.Conv2d):\n    \"\"\" 2D Convolutions like TensorFlow, for a fixed image size\"\"\"\n    def __init__(self, in_channels, out_channels, kernel_size, image_size=None, **kwargs):\n        super().__init__(in_channels, out_channels, kernel_size, **kwargs)\n        self.stride = self.stride if len(self.stride) == 2 else [self.stride[0]] * 2\n\n        # Calculate padding based on image size and save it\n        assert image_size is not None\n        ih, iw = image_size if type(image_size) == list else [image_size, image_size]\n        kh, kw = self.weight.size()[-2:]\n        sh, sw = self.stride\n        oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)\n        pad_h = max((oh - 1) * self.stride[0] + (kh - 1) * self.dilation[0] + 1 - ih, 0)\n        pad_w = max((ow - 1) * self.stride[1] + (kw - 1) * self.dilation[1] + 1 - iw, 0)\n        if pad_h > 0 or pad_w > 0:\n            self.static_padding = nn.ZeroPad2d((pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2))\n        else:\n            self.static_padding = Identity()\n\n    def forward(self, x):\n        x = self.static_padding(x)\n        x = F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)\n        return x\n\n\nclass Identity(nn.Module):\n    def __init__(self,):\n        super(Identity, self).__init__()\n\n    def forward(self, input):\n        return input\n\n\n########################################################################\n############## HELPERS FUNCTIONS FOR LOADING MODEL PARAMS ##############\n########################################################################\n\n\ndef efficientnet_params(model_name):\n    \"\"\" Map EfficientNet model name to parameter coefficients. \"\"\"\n    params_dict = {\n        # Coefficients:   width,depth,res,dropout\n        'efficientnet-b0': (1.0, 1.0, 224, 0.2),\n        'efficientnet-b1': (1.0, 1.1, 240, 0.2),\n        'efficientnet-b2': (1.1, 1.2, 260, 0.3),\n        'efficientnet-b3': (1.2, 1.4, 300, 0.3),\n        'efficientnet-b4': (1.4, 1.8, 380, 0.4),\n        'efficientnet-b5': (1.6, 2.2, 456, 0.4),\n        'efficientnet-b6': (1.8, 2.6, 528, 0.5),\n        'efficientnet-b7': (2.0, 3.1, 600, 0.5),\n    }\n    return params_dict[model_name]\n\n\nclass BlockDecoder(object):\n    \"\"\" Block Decoder for readability, straight from the official TensorFlow repository \"\"\"\n\n    @staticmethod\n    def _decode_block_string(block_string):\n        \"\"\" Gets a block through a string notation of arguments. \"\"\"\n        assert isinstance(block_string, str)\n\n        ops = block_string.split('_')\n        options = {}\n        for op in ops:\n            splits = re.split(r'(\\d.*)', op)\n            if len(splits) >= 2:\n                key, value = splits[:2]\n                options[key] = value\n\n        # Check stride\n        assert (('s' in options and len(options['s']) == 1) or\n                (len(options['s']) == 2 and options['s'][0] == options['s'][1]))\n\n        return BlockArgs(\n            kernel_size=int(options['k']),\n            num_repeat=int(options['r']),\n            input_filters=int(options['i']),\n            output_filters=int(options['o']),\n            expand_ratio=int(options['e']),\n            id_skip=('noskip' not in block_string),\n            se_ratio=float(options['se']) if 'se' in options else None,\n            stride=[int(options['s'][0])])\n\n    @staticmethod\n    def _encode_block_string(block):\n        \"\"\"Encodes a block to a string.\"\"\"\n        args = [\n            'r%d' % block.num_repeat,\n            'k%d' % block.kernel_size,\n            's%d%d' % (block.strides[0], block.strides[1]),\n            'e%s' % block.expand_ratio,\n            'i%d' % block.input_filters,\n            'o%d' % block.output_filters\n        ]\n        if 0 < block.se_ratio <= 1:\n            args.append('se%s' % block.se_ratio)\n        if block.id_skip is False:\n            args.append('noskip')\n        return '_'.join(args)\n\n    @staticmethod\n    def decode(string_list):\n        \"\"\"\n        Decodes a list of string notations to specify blocks inside the network.\n\n        :param string_list: a list of strings, each string is a notation of block\n        :return: a list of BlockArgs namedtuples of block args\n        \"\"\"\n        assert isinstance(string_list, list)\n        blocks_args = []\n        for block_string in string_list:\n            blocks_args.append(BlockDecoder._decode_block_string(block_string))\n        return blocks_args\n\n    @staticmethod\n    def encode(blocks_args):\n        \"\"\"\n        Encodes a list of BlockArgs to a list of strings.\n\n        :param blocks_args: a list of BlockArgs namedtuples of block args\n        :return: a list of strings, each string is a notation of block\n        \"\"\"\n        block_strings = []\n        for block in blocks_args:\n            block_strings.append(BlockDecoder._encode_block_string(block))\n        return block_strings\n\n\ndef efficientnet(width_coefficient=None, depth_coefficient=None, dropout_rate=0.2,\n                 drop_connect_rate=0.2, image_size=None, num_classes=1000):\n    \"\"\" Creates a efficientnet model. \"\"\"\n\n    blocks_args = [\n        'r1_k3_s11_e1_i32_o16_se0.25', 'r2_k3_s22_e6_i16_o24_se0.25',\n        'r2_k5_s22_e6_i24_o40_se0.25', 'r3_k3_s22_e6_i40_o80_se0.25',\n        'r3_k5_s11_e6_i80_o112_se0.25', 'r4_k5_s22_e6_i112_o192_se0.25',\n        'r1_k3_s11_e6_i192_o320_se0.25',\n    ]\n    blocks_args = BlockDecoder.decode(blocks_args)\n\n    global_params = GlobalParams(\n        batch_norm_momentum=0.99,\n        batch_norm_epsilon=1e-3,\n        dropout_rate=dropout_rate,\n        drop_connect_rate=drop_connect_rate,\n        # data_format='channels_last',  # removed, this is always true in PyTorch\n        num_classes=num_classes,\n        width_coefficient=width_coefficient,\n        depth_coefficient=depth_coefficient,\n        depth_divisor=8,\n        min_depth=None,\n        image_size=image_size,\n    )\n\n    return blocks_args, global_params\n\n\ndef get_model_params(model_name, override_params):\n    \"\"\" Get the block args and global params for a given model \"\"\"\n    if model_name.startswith('efficientnet'):\n        w, d, s, p = efficientnet_params(model_name)\n        # note: all models have drop connect rate = 0.2\n        blocks_args, global_params = efficientnet(\n            width_coefficient=w, depth_coefficient=d, dropout_rate=p, image_size=s)\n    else:\n        raise NotImplementedError('model name is not pre-defined: %s' % model_name)\n    if override_params:\n        # ValueError will be raised here if override_params has fields not included in global_params.\n        global_params = global_params._replace(**override_params)\n    return blocks_args, global_params\n\n\nurl_map = {\n    'efficientnet-b0': 'http://storage.googleapis.com/public-models/efficientnet-b0-08094119.pth',\n    'efficientnet-b1': 'http://storage.googleapis.com/public-models/efficientnet-b1-dbc7070a.pth',\n    'efficientnet-b2': 'http://storage.googleapis.com/public-models/efficientnet-b2-27687264.pth',\n    'efficientnet-b3': 'http://storage.googleapis.com/public-models/efficientnet-b3-c8376fa2.pth',\n    'efficientnet-b4': 'http://storage.googleapis.com/public-models/efficientnet-b4-e116e8b3.pth',\n    'efficientnet-b5': 'http://storage.googleapis.com/public-models/efficientnet-b5-586e6cc6.pth',\n}\n\ndef load_pretrained_weights(model, model_name, load_fc=True):\n    \"\"\" Loads pretrained weights, and downloads if loading for the first time. \"\"\"\n    state_dict = model_zoo.load_url(url_map[model_name])\n    if load_fc:\n        model.load_state_dict(state_dict)\n    else:\n        state_dict.pop('_fc.weight')\n        state_dict.pop('_fc.bias')\n        res = model.load_state_dict(state_dict, strict=False)\n        assert str(res.missing_keys) == str(['_fc.weight', '_fc.bias']), 'issue loading pretrained weights'\n    print('Loaded pretrained weights for {}'.format(model_name))\n    \n    \nclass MBConvBlock(nn.Module):\n    \"\"\"\n    Mobile Inverted Residual Bottleneck Block\n\n    Args:\n        block_args (namedtuple): BlockArgs, see above\n        global_params (namedtuple): GlobalParam, see above\n\n    Attributes:\n        has_se (bool): Whether the block contains a Squeeze and Excitation layer.\n    \"\"\"\n\n    def __init__(self, block_args, global_params):\n        super().__init__()\n        self._block_args = block_args\n        self._bn_mom = 1 - global_params.batch_norm_momentum\n        self._bn_eps = global_params.batch_norm_epsilon\n        self.has_se = (self._block_args.se_ratio is not None) and (0 < self._block_args.se_ratio <= 1)\n        self.id_skip = block_args.id_skip  # skip connection and drop connect\n\n        # Get static or dynamic convolution depending on image size\n        Conv2d = get_same_padding_conv2d(image_size=global_params.image_size)\n\n        # Expansion phase\n        inp = self._block_args.input_filters  # number of input channels\n        oup = self._block_args.input_filters * self._block_args.expand_ratio  # number of output channels\n        if self._block_args.expand_ratio != 1:\n            self._expand_conv = Conv2d(in_channels=inp, out_channels=oup, kernel_size=1, bias=False)\n            self._bn0 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)\n\n        # Depthwise convolution phase\n        k = self._block_args.kernel_size\n        s = self._block_args.stride\n        self._depthwise_conv = Conv2d(\n            in_channels=oup, out_channels=oup, groups=oup,  # groups makes it depthwise\n            kernel_size=k, stride=s, bias=False)\n        self._bn1 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)\n\n        # Squeeze and Excitation layer, if desired\n        if self.has_se:\n            num_squeezed_channels = max(1, int(self._block_args.input_filters * self._block_args.se_ratio))\n            self._se_reduce = Conv2d(in_channels=oup, out_channels=num_squeezed_channels, kernel_size=1)\n            self._se_expand = Conv2d(in_channels=num_squeezed_channels, out_channels=oup, kernel_size=1)\n\n        # Output phase\n        final_oup = self._block_args.output_filters\n        self._project_conv = Conv2d(in_channels=oup, out_channels=final_oup, kernel_size=1, bias=False)\n        self._bn2 = nn.BatchNorm2d(num_features=final_oup, momentum=self._bn_mom, eps=self._bn_eps)\n\n    def forward(self, inputs, drop_connect_rate=None):\n        \"\"\"\n        :param inputs: input tensor\n        :param drop_connect_rate: drop connect rate (float, between 0 and 1)\n        :return: output of block\n        \"\"\"\n\n        # Expansion and Depthwise Convolution\n        x = inputs\n        if self._block_args.expand_ratio != 1:\n            x = relu_fn(self._bn0(self._expand_conv(inputs)))\n        x = relu_fn(self._bn1(self._depthwise_conv(x)))\n\n        # Squeeze and Excitation\n        if self.has_se:\n            x_squeezed = F.adaptive_avg_pool2d(x, 1)\n            x_squeezed = self._se_expand(relu_fn(self._se_reduce(x_squeezed)))\n            x = torch.sigmoid(x_squeezed) * x\n\n        x = self._bn2(self._project_conv(x))\n\n        # Skip connection and drop connect\n        input_filters, output_filters = self._block_args.input_filters, self._block_args.output_filters\n        if self.id_skip and self._block_args.stride == 1 and input_filters == output_filters:\n            if drop_connect_rate:\n                x = drop_connect(x, p=drop_connect_rate, training=self.training)\n            x = x + inputs  # skip connection\n        return x\n\n\nclass EfficientNet(nn.Module):\n    \"\"\"\n    An EfficientNet model. Most easily loaded with the .from_name or .from_pretrained methods\n\n    Args:\n        blocks_args (list): A list of BlockArgs to construct blocks\n        global_params (namedtuple): A set of GlobalParams shared between blocks\n\n    Example:\n        model = EfficientNet.from_pretrained('efficientnet-b0')\n\n    \"\"\"\n\n    def __init__(self, blocks_args=None, global_params=None):\n        super().__init__()\n        assert isinstance(blocks_args, list), 'blocks_args should be a list'\n        assert len(blocks_args) > 0, 'block args must be greater than 0'\n        self._global_params = global_params\n        self._blocks_args = blocks_args\n\n        # Get static or dynamic convolution depending on image size\n        Conv2d = get_same_padding_conv2d(image_size=global_params.image_size)\n\n        # Batch norm parameters\n        bn_mom = 1 - self._global_params.batch_norm_momentum\n        bn_eps = self._global_params.batch_norm_epsilon\n\n        # Stem\n        in_channels = 3  # rgb\n        out_channels = round_filters(32, self._global_params)  # number of output channels\n        self._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias=False)\n        self._bn0 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)\n\n        # Build blocks\n        self._blocks = nn.ModuleList([])\n        for block_args in self._blocks_args:\n\n            # Update block input and output filters based on depth multiplier.\n            block_args = block_args._replace(\n                input_filters=round_filters(block_args.input_filters, self._global_params),\n                output_filters=round_filters(block_args.output_filters, self._global_params),\n                num_repeat=round_repeats(block_args.num_repeat, self._global_params)\n            )\n\n            # The first block needs to take care of stride and filter size increase.\n            self._blocks.append(MBConvBlock(block_args, self._global_params))\n            if block_args.num_repeat > 1:\n                block_args = block_args._replace(input_filters=block_args.output_filters, stride=1)\n            for _ in range(block_args.num_repeat - 1):\n                self._blocks.append(MBConvBlock(block_args, self._global_params))\n\n        # Head\n        in_channels = block_args.output_filters  # output of final block\n        out_channels = round_filters(1280, self._global_params)\n        self._conv_head = Conv2d(in_channels, out_channels, kernel_size=1, bias=False)\n        self._bn1 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)\n\n        # Final linear layer\n        self._dropout = self._global_params.dropout_rate\n        self._fc = nn.Linear(out_channels, self._global_params.num_classes)\n\n    def extract_features(self, inputs):\n        \"\"\" Returns output of the final convolution layer \"\"\"\n\n        # Stem\n        x = relu_fn(self._bn0(self._conv_stem(inputs)))\n\n        # Blocks\n        for idx, block in enumerate(self._blocks):\n            drop_connect_rate = self._global_params.drop_connect_rate\n            if drop_connect_rate:\n                drop_connect_rate *= float(idx) / len(self._blocks)\n            x = block(x, drop_connect_rate=drop_connect_rate)\n\n        # Head\n        x = relu_fn(self._bn1(self._conv_head(x)))\n\n        return x\n\n    def forward(self, inputs):\n        \"\"\" Calls extract_features to extract features, applies final linear layer, and returns logits. \"\"\"\n\n        # Convolution layers\n        x = self.extract_features(inputs)\n\n        # Pooling and final linear layer\n        x = F.adaptive_avg_pool2d(x, 1).squeeze(-1).squeeze(-1)\n        if self._dropout:\n            x = F.dropout(x, p=self._dropout, training=self.training)\n        x = self._fc(x)\n        return x\n\n    @classmethod\n    def from_name(cls, model_name, override_params=None):\n        cls._check_model_name_is_valid(model_name)\n        blocks_args, global_params = get_model_params(model_name, override_params)\n        return EfficientNet(blocks_args, global_params)\n\n    @classmethod\n    def from_pretrained(cls, model_name, num_classes=1000):\n        model = EfficientNet.from_name(model_name, override_params={'num_classes': num_classes})\n        return model\n\n    @classmethod\n    def get_image_size(cls, model_name):\n        cls._check_model_name_is_valid(model_name)\n        _, _, res, _ = efficientnet_params(model_name)\n        return res\n\n    @classmethod\n    def _check_model_name_is_valid(cls, model_name, also_need_pretrained_weights=False):\n        \"\"\" Validates model name. None that pretrained weights are only available for\n        the first four models (efficientnet-b{i} for i in 0,1,2,3) at the moment. \"\"\"\n        num_models = 4 if also_need_pretrained_weights else 8\n        valid_models = ['efficientnet_b'+str(i) for i in range(num_models)]\n        if model_name.replace('-','_') not in valid_models:\n            raise ValueError('model_name should be one of: ' + ', '.join(valid_models))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#you can play around with tfms and image sizes\nbs = 64\nsz = 224\ntfms = get_transforms(do_flip=True,flip_vert=True)\n\n\n\ndata4 = (ImageList.from_df(df=df,path='./',cols='path') \n        .split_by_rand_pct(0.2) \n        .label_from_df(cols='diagnosis',label_cls=FloatList) \n        .transform(tfms,size=sz,resize_method=ResizeMethod.SQUISH,padding_mode='zeros') \n        .databunch(bs=bs,num_workers=4) \n        .normalize(imagenet_stats)  \n       )\n\nlearn4 = Learner(data4, \n                EfficientNet.from_pretrained('efficientnet-b5', num_classes=1),\n                metrics = [qk], \n                model_dir=\"models\").to_fp16()\n\nlearn4.data.add_test(ImageList.from_df(test_df,\n                                      '../input/aptos2019-blindness-detection',\n                                      folder='test_images',\n                                      suffix='.png'))\n\nlearn4.load('abcdef');\npreds4,y = learn4.get_preds(DatasetType.Test)\ndel learn4, data4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#https://www.kaggle.com/abhishek/optimizer-for-quadratic-weighted-kappa\nclass OptimizedRounder(object):\n    def __init__(self):\n        self.coef_ = 0\n\n    def _kappa_loss(self, coef, X, y):\n        X_p = np.copy(X)\n        for i, pred in enumerate(X_p):\n            if pred < coef[0]:\n                X_p[i] = 0\n            elif pred >= coef[0] and pred < coef[1]:\n                X_p[i] = 1\n            elif pred >= coef[1] and pred < coef[2]:\n                X_p[i] = 2\n            elif pred >= coef[2] and pred < coef[3]:\n                X_p[i] = 3\n            else:\n                X_p[i] = 4\n\n        ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')\n        return -ll\n\n    def fit(self, X, y):\n        loss_partial = partial(self._kappa_loss, X=X, y=y)\n        initial_coef = [0.5, 1.5, 2.5, 3.5]\n        self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')\n        print(-loss_partial(self.coef_['x']))\n\n    def predict(self, X, coef):\n        X_p = np.copy(X)\n        for i, pred in enumerate(X_p):\n            if pred < coef[0]:\n                X_p[i] = 0\n            elif pred >= coef[0] and pred < coef[1]:\n                X_p[i] = 1\n            elif pred >= coef[1] and pred < coef[2]:\n                X_p[i] = 2\n            elif pred >= coef[2] and pred < coef[3]:\n                X_p[i] = 3\n            else:\n                X_p[i] = 4\n        return X_p\n\n    def coefficients(self):\n        return self.coef_['x']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds6","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred = (preds1+preds2+preds3+preds4+preds5+preds6)/6","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_subm(preds, coefficients=[0.5, 1.5, 2.5, 3.5]):\n    opt = OptimizedRounder()\n#     preds,y = learn.get_preds(DatasetType.Test)\n    tst_pred = opt.predict(preds, coefficients)\n    test_df.diagnosis = tst_pred.astype(int)\n    test_df.to_csv('submission.csv',index=False)\n    print ('done')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nrun_subm(pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def run_subm(learn=learn, coefficients=[0.5, 1.5, 2.5, 3.5]):\n#     opt = OptimizedRounder()\n#     preds,y = learn.get_preds(DatasetType.Test)\n#     tst_pred = opt.predict(preds, coefficients)\n#     test_df.diagnosis = tst_pred.astype(int)\n#     test_df.to_csv('submission.csv',index=False)\n#     print ('done')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# if os.path.exists(directory):\n#     !rm -r '../input/test_preprocessed'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# os.listdir('../input')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}