{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# To have reproducible results\nseed = 5 \nimport numpy as np \nnp.random.seed(seed)\nimport tensorflow as tf\ntf.set_random_seed(seed)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T04:45:48.631366Z","iopub.execute_input":"2023-01-31T04:45:48.631861Z","iopub.status.idle":"2023-01-31T04:45:48.637721Z","shell.execute_reply.started":"2023-01-31T04:45:48.631624Z","shell.execute_reply":"2023-01-31T04:45:48.636668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <a id='1'>Loading and Exploration</a> ","metadata":{}},{"cell_type":"code","source":"import json\nimport math\nimport os\n\nimport cv2\nfrom PIL import Image\n# from keras_efficientnets import *\nfrom keras import layers\nfrom keras.applications.resnet50 import ResNet50, preprocess_input\nfrom keras.applications import DenseNet121\nfrom keras.callbacks import Callback, ModelCheckpoint,EarlyStopping\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Sequential\nfrom keras.optimizers import Adam\nimport matplotlib.pyplot as plt\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import cohen_kappa_score, accuracy_score\nimport scipy\nfrom tqdm import tqdm\nprint(os.listdir('../input'))\n%matplotlib inline\n\nIMG_SIZE=256\nBATCH_SIZE = 16","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2023-01-31T04:45:48.639527Z","iopub.execute_input":"2023-01-31T04:45:48.639981Z","iopub.status.idle":"2023-01-31T04:45:48.656807Z","shell.execute_reply.started":"2023-01-31T04:45:48.639807Z","shell.execute_reply":"2023-01-31T04:45:48.655780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <a id='2'>EfficientNet Utility Codes</a> ","metadata":{}},{"cell_type":"code","source":"\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport numpy as np\nfrom skimage.transform import resize\nimport tensorflow as tf\nimport keras.backend as K\nimport keras.layers as KL\nfrom keras.initializers import Initializer\nfrom keras.utils.generic_utils import get_custom_objects\n\nimport os\nimport re\nimport collections\nimport math\nimport six\nfrom six.moves import xrange  # pylint: disable=redefined-builtin\n\nimport keras.models as KM\nfrom keras.utils import get_file\n\nMEAN_RGB = [0.485 * 255, 0.456 * 255, 0.406 * 255]\nSTDDEV_RGB = [0.229 * 255, 0.224 * 255, 0.225 * 255]\n\nMAP_INTERPOLATION_TO_ORDER = {\n    \"nearest\": 0,\n    \"bilinear\": 1,\n    \"biquadratic\": 2,\n    \"bicubic\": 3,\n}\n\n\ndef center_crop_and_resize(image, image_size, crop_padding=32, interpolation=\"bicubic\"):\n    assert image.ndim in {2, 3}\n    assert interpolation in MAP_INTERPOLATION_TO_ORDER.keys()\n\n    h, w = image.shape[:2]\n\n    padded_center_crop_size = int(\n        (image_size / (image_size + crop_padding)) * min(h, w)\n    )\n    offset_height = ((h - padded_center_crop_size) + 1) // 2\n    offset_width = ((w - padded_center_crop_size) + 1) // 2\n\n    image_crop = image[\n        offset_height : padded_center_crop_size + offset_height,\n        offset_width : padded_center_crop_size + offset_width,\n    ]\n    resized_image = resize(\n        image_crop,\n        (image_size, image_size),\n        order=MAP_INTERPOLATION_TO_ORDER[interpolation],\n        preserve_range=True,\n    )\n\n    return resized_image\n\n\ndef preprocess_input(x):\n    assert x.ndim in (3, 4)\n    assert x.shape[-1] == 3\n\n    x = x - np.array(MEAN_RGB)\n    x = x / np.array(STDDEV_RGB)\n\n    return x\n\nclass EfficientConv2DKernelInitializer(Initializer):\n    def __call__(self, shape, dtype=K.floatx(), **kwargs):\n        kernel_height, kernel_width, _, out_filters = shape\n        fan_out = int(kernel_height * kernel_width * out_filters)\n        return tf.random_normal(\n            shape, mean=0.0, stddev=np.sqrt(2.0 / fan_out), dtype=dtype\n        )\n\n\nclass EfficientDenseKernelInitializer(Initializer):\n    def __call__(self, shape, dtype=K.floatx(), **kwargs):\n        init_range = 1.0 / np.sqrt(shape[1])\n        return tf.random_uniform(shape, -init_range, init_range, dtype=dtype)\n\n\nconv_kernel_initializer = EfficientConv2DKernelInitializer()\ndense_kernel_initializer = EfficientDenseKernelInitializer()\n\n\nget_custom_objects().update(\n    {\n        \"EfficientDenseKernelInitializer\": EfficientDenseKernelInitializer,\n        \"EfficientConv2DKernelInitializer\": EfficientConv2DKernelInitializer,\n    }\n)\n\nclass Swish(KL.Layer):\n    def call(self, inputs):\n        return tf.nn.swish(inputs)\n\n\nclass DropConnect(KL.Layer):\n    def __init__(self, drop_connect_rate=0.0, **kwargs):\n        super().__init__(**kwargs)\n        self.drop_connect_rate = drop_connect_rate\n\n    def call(self, inputs, training=None):\n        def drop_connect():\n            keep_prob = 1.0 - self.drop_connect_rate\n\n            # Compute drop_connect tensor\n            batch_size = tf.shape(inputs)[0]\n            random_tensor = keep_prob\n            random_tensor += tf.random_uniform(\n                [batch_size, 1, 1, 1], dtype=inputs.dtype\n            )\n            binary_tensor = tf.floor(random_tensor)\n            output = tf.div(inputs, keep_prob) * binary_tensor\n            return output\n\n        return K.in_train_phase(drop_connect, inputs, training=training)\n\n    def get_config(self):\n        config = super().get_config()\n        config[\"drop_connect_rate\"] = self.drop_connect_rate\n        return config\n\n\nget_custom_objects().update({\"DropConnect\": DropConnect, \"Swish\": Swish})\n\n\nIMAGENET_WEIGHTS = {\n    \"efficientnet-b0\": {\n        \"name\": \"efficientnet-b0_imagenet_1000.h5\",\n        \"url\": \"https://github.com/qubvel/efficientnet/releases/download/v0.0.1/efficientnet-b0_imagenet_1000.h5\",\n        \"md5\": \"bca04d16b1b8a7c607b1152fe9261af7\",\n    },\n    \"efficientnet-b0-notop\": {\n        \"name\": \"efficientnet-b0_imagenet_1000_notop.h5\",\n        \"url\": \"https://github.com/qubvel/efficientnet/releases/download/v0.0.1/efficientnet-b0_imagenet_1000_notop.h5\",\n        \"md5\": \"45d2f3b6330c2401ef66da3961cad769\",\n    },\n    \"efficientnet-b1\": {\n        \"name\": \"efficientnet-b1_imagenet_1000.h5\",\n        \"url\": \"https://github.com/qubvel/efficientnet/releases/download/v0.0.1/efficientnet-b1_imagenet_1000.h5\",\n        \"md5\": \"bd4a2b82f6f6bada74fc754553c464fc\",\n    },\n    \"efficientnet-b1-notop\": {\n        \"name\": \"efficientnet-b1_imagenet_1000_notop.h5\",\n        \"url\": \"https://github.com/qubvel/efficientnet/releases/download/v0.0.1/efficientnet-b1_imagenet_1000_notop.h5\",\n        \"md5\": \"884aed586c2d8ca8dd15a605ec42f564\",\n    },\n    \"efficientnet-b2\": {\n        \"name\": \"efficientnet-b2_imagenet_1000.h5\",\n        \"url\": \"https://github.com/qubvel/efficientnet/releases/download/v0.0.1/efficientnet-b2_imagenet_1000.h5\",\n        \"md5\": \"45b28b26f15958bac270ab527a376999\",\n    },\n    \"efficientnet-b2-notop\": {\n        \"name\": \"efficientnet-b2_imagenet_1000_notop.h5\",\n        \"url\": \"https://github.com/qubvel/efficientnet/releases/download/v0.0.1/efficientnet-b2_imagenet_1000_notop.h5\",\n        \"md5\": \"42fb9f2d9243d461d62b4555d3a53b7b\",\n    },\n    \"efficientnet-b3\": {\n        \"name\": \"efficientnet-b3_imagenet_1000.h5\",\n        \"url\": \"https://github.com/qubvel/efficientnet/releases/download/v0.0.1/efficientnet-b3_imagenet_1000.h5\",\n        \"md5\": \"decd2c8a23971734f9d3f6b4053bf424\",\n    },\n    \"efficientnet-b3-notop\": {\n        \"name\": \"efficientnet-b3_imagenet_1000_notop.h5\",\n        \"url\": \"https://github.com/qubvel/efficientnet/releases/download/v0.0.1/efficientnet-b3_imagenet_1000_notop.h5\",\n        \"md5\": \"1f7d9a8c2469d2e3d3b97680d45df1e1\",\n    },\n    \"efficientnet-b4\": {\n        \"name\": \"efficientnet-b4_imagenet_1000.h5\",\n        \"url\": \"https://github.com/qubvel/efficientnet/releases/download/v0.0.1/efficientnet-b4_imagenet_1000.h5\",\n        \"md5\": \"01df77157a86609530aeb4f1f9527949\",\n    },\n    \"efficientnet-b4-notop\": {\n        \"name\": \"efficientnet-b4_imagenet_1000_notop.h5\",\n        \"url\": \"https://github.com/qubvel/efficientnet/releases/download/v0.0.1/efficientnet-b4_imagenet_1000_notop.h5\",\n        \"md5\": \"e7c3b780f050f8f49c800f23703f285c\",\n    },\n    \"efficientnet-b5\": {\n        \"name\": \"efficientnet-b5_imagenet_1000.h5\",\n        \"url\": \"https://github.com/qubvel/efficientnet/releases/download/v0.0.1/efficientnet-b5_imagenet_1000.h5\",\n        \"md5\": \"c31311a1a38b5111e14457145fccdf32\",\n    },\n    \"efficientnet-b5-notop\": {\n        \"name\": \"efficientnet-b5_imagenet_1000_notop.h5\",\n        \"url\": \"https://github.com/qubvel/efficientnet/releases/download/v0.0.1/efficientnet-b5_imagenet_1000_notop.h5\",\n        \"md5\": \"a09b36129b41196e0bb659fd84fbdd5f\",\n    },\n}\n\n\nGlobalParams = collections.namedtuple(\n    \"GlobalParams\",\n    [\n        \"batch_norm_momentum\",\n        \"batch_norm_epsilon\",\n        \"dropout_rate\",\n        \"data_format\",\n        \"num_classes\",\n        \"width_coefficient\",\n        \"depth_coefficient\",\n        \"depth_divisor\",\n        \"min_depth\",\n        \"drop_connect_rate\",\n    ],\n)\nGlobalParams.__new__.__defaults__ = (None,) * len(GlobalParams._fields)\n\nBlockArgs = collections.namedtuple(\n    \"BlockArgs\",\n    [\n        \"kernel_size\",\n        \"num_repeat\",\n        \"input_filters\",\n        \"output_filters\",\n        \"expand_ratio\",\n        \"id_skip\",\n        \"strides\",\n        \"se_ratio\",\n    ],\n)\n# defaults will be a public argument for namedtuple in Python 3.7\n# https://docs.python.org/3/library/collections.html#collections.namedtuple\nBlockArgs.__new__.__defaults__ = (None,) * len(BlockArgs._fields)\n\n\ndef efficientnet_params(model_name):\n    \"\"\"Get efficientnet params based on model name.\"\"\"\n    params_dict = {\n        # (width_coefficient, depth_coefficient, resolution, dropout_rate)\n        \"efficientnet-b0\": (1.0, 1.0, 224, 0.2),\n        \"efficientnet-b1\": (1.0, 1.1, 240, 0.2),\n        \"efficientnet-b2\": (1.1, 1.2, 260, 0.3),\n        \"efficientnet-b3\": (1.2, 1.4, 300, 0.3),\n        \"efficientnet-b4\": (1.4, 1.8, 380, 0.4),\n        \"efficientnet-b5\": (1.6, 2.2, 456, 0.4),\n        \"efficientnet-b6\": (1.8, 2.6, 528, 0.5),\n        \"efficientnet-b7\": (2.0, 3.1, 600, 0.5),\n    }\n    return params_dict[model_name]\n\n\nclass BlockDecoder(object):\n    \"\"\"Block Decoder for readability.\"\"\"\n\n    def _decode_block_string(self, block_string):\n        \"\"\"Gets a block through a string notation of arguments.\"\"\"\n        assert isinstance(block_string, str)\n        ops = block_string.split(\"_\")\n        options = {}\n        for op in ops:\n            splits = re.split(r\"(\\d.*)\", op)\n            if len(splits) >= 2:\n                key, value = splits[:2]\n                options[key] = value\n\n        if \"s\" not in options or len(options[\"s\"]) != 2:\n            raise ValueError(\"Strides options should be a pair of integers.\")\n\n        return BlockArgs(\n            kernel_size=int(options[\"k\"]),\n            num_repeat=int(options[\"r\"]),\n            input_filters=int(options[\"i\"]),\n            output_filters=int(options[\"o\"]),\n            expand_ratio=int(options[\"e\"]),\n            id_skip=(\"noskip\" not in block_string),\n            se_ratio=float(options[\"se\"]) if \"se\" in options else None,\n            strides=[int(options[\"s\"][0]), int(options[\"s\"][1])],\n        )\n\n    def _encode_block_string(self, block):\n        \"\"\"Encodes a block to a string.\"\"\"\n        args = [\n            \"r%d\" % block.num_repeat,\n            \"k%d\" % block.kernel_size,\n            \"s%d%d\" % (block.strides[0], block.strides[1]),\n            \"e%s\" % block.expand_ratio,\n            \"i%d\" % block.input_filters,\n            \"o%d\" % block.output_filters,\n        ]\n        if block.se_ratio > 0 and block.se_ratio <= 1:\n            args.append(\"se%s\" % block.se_ratio)\n        if block.id_skip is False:\n            args.append(\"noskip\")\n        return \"_\".join(args)\n\n    def decode(self, string_list):\n        assert isinstance(string_list, list)\n        blocks_args = []\n        for block_string in string_list:\n            blocks_args.append(self._decode_block_string(block_string))\n        return blocks_args\n\n    def encode(self, blocks_args):\n        block_strings = []\n        for block in blocks_args:\n            block_strings.append(self._encode_block_string(block))\n        return block_strings\n\n\ndef efficientnet(\n    width_coefficient=None,\n    depth_coefficient=None,\n    dropout_rate=0.2,\n    drop_connect_rate=0.2,\n):\n    \"\"\"Creates a efficientnet model.\"\"\"\n    blocks_args = [\n        \"r1_k3_s11_e1_i32_o16_se0.25\",\n        \"r2_k3_s22_e6_i16_o24_se0.25\",\n        \"r2_k5_s22_e6_i24_o40_se0.25\",\n        \"r3_k3_s22_e6_i40_o80_se0.25\",\n        \"r3_k5_s11_e6_i80_o112_se0.25\",\n        \"r4_k5_s22_e6_i112_o192_se0.25\",\n        \"r1_k3_s11_e6_i192_o320_se0.25\",\n    ]\n    global_params = GlobalParams(\n        batch_norm_momentum=0.99,\n        batch_norm_epsilon=1e-3,\n        dropout_rate=dropout_rate,\n        drop_connect_rate=drop_connect_rate,\n        data_format=\"channels_last\",\n        num_classes=1000,\n        width_coefficient=width_coefficient,\n        depth_coefficient=depth_coefficient,\n        depth_divisor=8,\n        min_depth=None,\n    )\n    decoder = BlockDecoder()\n    return decoder.decode(blocks_args), global_params\n\n\ndef get_model_params(model_name, override_params=None):\n    \"\"\"Get the block args and global params for a given model.\"\"\"\n    if model_name.startswith(\"efficientnet\"):\n        width_coefficient, depth_coefficient, input_shape, dropout_rate = efficientnet_params(\n            model_name\n        )\n        blocks_args, global_params = efficientnet(\n            width_coefficient, depth_coefficient, dropout_rate\n        )\n    else:\n        raise NotImplementedError(\"model name is not pre-defined: %s\" % model_name)\n\n    if override_params:\n        # ValueError will be raised here if override_params has fields not included\n        # in global_params.\n        global_params = global_params._replace(**override_params)\n\n    # print('global_params= %s', global_params)\n    # print('blocks_args= %s', blocks_args)\n    return blocks_args, global_params, input_shape\n\n\n\n__all__ = [\n    \"EfficientNet\",\n    \"EfficientNetB0\",\n    \"EfficientNetB1\",\n    \"EfficientNetB2\",\n    \"EfficientNetB3\",\n    \"EfficientNetB4\",\n    \"EfficientNetB5\",\n    \"EfficientNetB6\",\n    \"EfficientNetB7\",\n]\n\n\ndef round_filters(filters, global_params):\n    \"\"\"Round number of filters based on depth multiplier.\"\"\"\n    orig_f = filters\n    multiplier = global_params.width_coefficient\n    divisor = global_params.depth_divisor\n    min_depth = global_params.min_depth\n    if not multiplier:\n        return filters\n\n    filters *= multiplier\n    min_depth = min_depth or divisor\n    new_filters = max(min_depth, int(filters + divisor / 2) // divisor * divisor)\n    # Make sure that round down does not go down by more than 10%.\n    if new_filters < 0.9 * filters:\n        new_filters += divisor\n    # print('round_filter input={} output={}'.format(orig_f, new_filters))\n    return int(new_filters)\n\n\ndef round_repeats(repeats, global_params):\n    \"\"\"Round number of filters based on depth multiplier.\"\"\"\n    multiplier = global_params.depth_coefficient\n    if not multiplier:\n        return repeats\n    return int(math.ceil(multiplier * repeats))\n\n\ndef SEBlock(block_args, global_params):\n    num_reduced_filters = max(1, int(block_args.input_filters * block_args.se_ratio))\n    filters = block_args.input_filters * block_args.expand_ratio\n    if global_params.data_format == \"channels_first\":\n        channel_axis = 1\n        spatial_dims = [2, 3]\n    else:\n        channel_axis = -1\n        spatial_dims = [1, 2]\n\n    def block(inputs):\n        x = inputs\n        x = KL.Lambda(lambda a: K.mean(a, axis=spatial_dims, keepdims=True))(x)\n        x = KL.Conv2D(\n            num_reduced_filters,\n            kernel_size=[1, 1],\n            strides=[1, 1],\n            kernel_initializer=conv_kernel_initializer,\n            padding=\"same\",\n            use_bias=True,\n        )(x)\n        x = Swish()(x)\n        # Excite\n        x = KL.Conv2D(\n            filters,\n            kernel_size=[1, 1],\n            strides=[1, 1],\n            kernel_initializer=conv_kernel_initializer,\n            padding=\"same\",\n            use_bias=True,\n        )(x)\n        x = KL.Activation(\"sigmoid\")(x)\n        out = KL.Multiply()([x, inputs])\n        return out\n\n    return block\n\n\ndef MBConvBlock(block_args, global_params, drop_connect_rate=None):\n    batch_norm_momentum = global_params.batch_norm_momentum\n    batch_norm_epsilon = global_params.batch_norm_epsilon\n\n    if global_params.data_format == \"channels_first\":\n        channel_axis = 1\n        spatial_dims = [2, 3]\n    else:\n        channel_axis = -1\n        spatial_dims = [1, 2]\n\n    has_se = (\n        (block_args.se_ratio is not None)\n        and (block_args.se_ratio > 0)\n        and (block_args.se_ratio <= 1)\n    )\n\n    filters = block_args.input_filters * block_args.expand_ratio\n    kernel_size = block_args.kernel_size\n\n    def block(inputs):\n\n        if block_args.expand_ratio != 1:\n            x = KL.Conv2D(\n                filters,\n                kernel_size=[1, 1],\n                strides=[1, 1],\n                kernel_initializer=conv_kernel_initializer,\n                padding=\"same\",\n                use_bias=False,\n            )(inputs)\n            x = KL.BatchNormalization(\n                axis=channel_axis,\n                momentum=batch_norm_momentum,\n                epsilon=batch_norm_epsilon,\n            )(x)\n            x = Swish()(x)\n        else:\n            x = inputs\n\n        x = KL.DepthwiseConv2D(\n            [kernel_size, kernel_size],\n            strides=block_args.strides,\n            depthwise_initializer=conv_kernel_initializer,\n            padding=\"same\",\n            use_bias=False,\n        )(x)\n        x = KL.BatchNormalization(\n            axis=channel_axis, momentum=batch_norm_momentum, epsilon=batch_norm_epsilon\n        )(x)\n        x = Swish()(x)\n\n        if has_se:\n            x = SEBlock(block_args, global_params)(x)\n\n        # output phase\n\n        x = KL.Conv2D(\n            block_args.output_filters,\n            kernel_size=[1, 1],\n            strides=[1, 1],\n            kernel_initializer=conv_kernel_initializer,\n            padding=\"same\",\n            use_bias=False,\n        )(x)\n        x = KL.BatchNormalization(\n            axis=channel_axis, momentum=batch_norm_momentum, epsilon=batch_norm_epsilon\n        )(x)\n\n        if block_args.id_skip:\n            if (\n                all(s == 1 for s in block_args.strides)\n                and block_args.input_filters == block_args.output_filters\n            ):\n                # only apply drop_connect if skip presents.\n                if drop_connect_rate:\n                    x = DropConnect(drop_connect_rate)(x)\n                x = KL.Add()([x, inputs])\n        return x\n\n    return block\n\n\ndef EfficientNet(\n    input_shape, block_args_list, global_params, input_tensor=None, include_top=True, pooling=None\n):\n    batch_norm_momentum = global_params.batch_norm_momentum\n    batch_norm_epsilon = global_params.batch_norm_epsilon\n    if global_params.data_format == \"channels_first\":\n        channel_axis = 1\n    else:\n        channel_axis = -1\n\n    # Stem part\n    if input_tensor is None:\n        inputs = KL.Input(shape=input_shape)\n    else:\n        if not K.is_keras_tensor(input_tensor):\n            inputs = KL.Input(tensor=input_tensor, shape=input_shape)\n        else:\n            inputs = input_tensor\n    x = inputs\n    x = KL.Conv2D(\n        filters=round_filters(32, global_params),\n        kernel_size=[3, 3],\n        strides=[2, 2],\n        kernel_initializer=conv_kernel_initializer,\n        padding=\"same\",\n        use_bias=False,\n    )(x)\n    x = KL.BatchNormalization(\n        axis=channel_axis, momentum=batch_norm_momentum, epsilon=batch_norm_epsilon\n    )(x)\n    x = Swish()(x)\n\n    # Blocks part\n    block_idx = 1\n    n_blocks = sum([block_args.num_repeat for block_args in block_args_list])\n    drop_rate = global_params.drop_connect_rate or 0\n    drop_rate_dx = drop_rate / n_blocks\n\n    for block_args in block_args_list:\n        assert block_args.num_repeat > 0\n        # Update block input and output filters based on depth multiplier.\n        block_args = block_args._replace(\n            input_filters=round_filters(block_args.input_filters, global_params),\n            output_filters=round_filters(block_args.output_filters, global_params),\n            num_repeat=round_repeats(block_args.num_repeat, global_params),\n        )\n\n        # The first block needs to take care of stride and filter size increase.\n        x = MBConvBlock(\n            block_args, global_params, drop_connect_rate=drop_rate_dx * block_idx\n        )(x)\n        block_idx += 1\n\n        if block_args.num_repeat > 1:\n            block_args = block_args._replace(\n                input_filters=block_args.output_filters, strides=[1, 1]\n            )\n\n        for _ in xrange(block_args.num_repeat - 1):\n            x = MBConvBlock(\n                block_args, global_params, drop_connect_rate=drop_rate_dx * block_idx\n            )(x)\n            block_idx += 1\n\n    # Head part\n    x = KL.Conv2D(\n        filters=round_filters(1280, global_params),\n        kernel_size=[1, 1],\n        strides=[1, 1],\n        kernel_initializer=conv_kernel_initializer,\n        padding=\"same\",\n        use_bias=False,\n    )(x)\n    x = KL.BatchNormalization(\n        axis=channel_axis, momentum=batch_norm_momentum, epsilon=batch_norm_epsilon\n    )(x)\n    x = Swish()(x)\n\n    if include_top:\n        x = KL.GlobalAveragePooling2D(data_format=global_params.data_format)(x)\n        if global_params.dropout_rate > 0:\n            x = KL.Dropout(global_params.dropout_rate)(x)\n        x = KL.Dense(\n            global_params.num_classes, kernel_initializer=dense_kernel_initializer\n        )(x)\n        x = KL.Activation(\"softmax\")(x)\n    else:\n        if pooling == \"avg\":\n            x = KL.GlobalAveragePooling2D(data_format=global_params.data_format)(x)\n        elif pooling == \"max\":\n            x = KL.GlobalMaxPooling2D(data_format=global_params.data_format)(x)\n\n    outputs = x\n    model = KM.Model(inputs, outputs)\n\n    return model\n\n\ndef _get_model_by_name(\n    model_name, \n    input_shape=None, \n    input_tensor=None, \n    include_top=True, \n    weights=None, \n    classes=1000, \n    pooling=None\n):\n    if weights not in {None, \"imagenet\"}:\n        raise ValueError('Parameter `weights` should be one of [None, \"imagenet\"]')\n\n    if weights == \"imagenet\" and model_name not in IMAGENET_WEIGHTS:\n        raise ValueError(\n            \"There are not pretrained weights for {} model.\".format(model_name)\n        )\n\n    if weights == \"imagenet\" and include_top and classes != 1000:\n        raise ValueError(\n            \"If using `weights` and `include_top`\" \" `classes` should be 1000\"\n        )\n\n    block_agrs_list, global_params, default_input_shape = get_model_params(\n        model_name, override_params={\"num_classes\": classes}\n    )\n\n    if input_shape is None:\n        input_shape = (default_input_shape, default_input_shape, 3)\n        \n    model = EfficientNet(\n        input_shape,\n        block_agrs_list,\n        global_params,\n        input_tensor=input_tensor,\n        include_top=include_top,\n        pooling=pooling,\n    )\n\n    model.name = model_name\n\n    if weights:\n        if not include_top:\n            weights_name = model_name + \"-notop\"\n        else:\n            weights_name = model_name\n        weights = IMAGENET_WEIGHTS[weights_name]\n        weights_path = get_file(\n            weights[\"name\"],\n            weights[\"url\"],\n            cache_subdir=\"models\",\n            md5_hash=weights[\"md5\"],\n        )\n        model.load_weights(weights_path)\n\n    return model\n\n\ndef EfficientNetB0(\n    include_top=True, input_shape=None, input_tensor=None, weights=None, classes=1000, pooling=None\n):\n    return _get_model_by_name(\n        \"efficientnet-b0\",\n        include_top=include_top,\n        input_shape=input_shape,\n        input_tensor=input_tensor,\n        weights=weights,\n        classes=classes,\n        pooling=pooling,\n    )\n\n\ndef EfficientNetB1(\n    include_top=True, input_shape=None, input_tensor=None, weights=None, classes=1000, pooling=None\n):\n    return _get_model_by_name(\n        \"efficientnet-b1\",\n        include_top=include_top,\n        input_shape=input_shape,\n        input_tensor=input_tensor,\n        weights=weights,\n        classes=classes,\n        pooling=pooling,\n    )\n\n\ndef EfficientNetB2(\n    include_top=True, input_shape=None, input_tensor=None, weights=None, classes=1000, pooling=None\n):\n    return _get_model_by_name(\n        \"efficientnet-b2\",\n        include_top=include_top,\n        input_shape=input_shape,\n        input_tensor=input_tensor,\n        weights=weights,\n        classes=classes,\n        pooling=pooling,\n    )\n\n\ndef EfficientNetB3(\n    include_top=True, input_shape=None, input_tensor=None, weights=None, classes=1000, pooling=None\n):\n    return _get_model_by_name(\n        \"efficientnet-b3\",\n        include_top=include_top,\n        input_shape=input_shape,\n        input_tensor=input_tensor,\n        weights=weights,\n        classes=classes,\n        pooling=pooling,\n    )\n\n\ndef EfficientNetB4(\n    include_top=True, input_shape=None, input_tensor=None, weights=None, classes=1000, pooling=None\n):\n    return _get_model_by_name(\n        \"efficientnet-b4\",\n        include_top=include_top,\n        input_shape=input_shape,\n        input_tensor=input_tensor,\n        weights=weights,\n        classes=classes,\n        pooling=pooling,\n    )\n\n\ndef EfficientNetB5(\n    include_top=True, input_shape=None, input_tensor=None, weights=None, classes=1000, pooling=None\n):\n    return _get_model_by_name(\n        \"efficientnet-b5\",\n        include_top=include_top,\n        input_shape=input_shape,\n        input_tensor=input_tensor,\n        weights=weights,\n        classes=classes,\n        pooling=pooling,\n    )\n\n\ndef EfficientNetB6(\n    include_top=True, input_shape=None, input_tensor=None, weights=None, classes=1000, pooling=None\n):\n    return _get_model_by_name(\n        \"efficientnet-b6\",\n        include_top=include_top,\n        input_shape=input_shape,\n        input_tensor=input_tensor,\n        weights=weights,\n        classes=classes,\n        pooling=pooling,\n    )\n\n\ndef EfficientNetB7(\n    include_top=True, input_shape=None, input_tensor=None, weights=None, classes=1000, pooling=None\n):\n    return _get_model_by_name(\n        \"efficientnet-b7\",\n        include_top=include_top,\n        input_shape=input_shape,\n        input_tensor=input_tensor,\n        weights=weights,\n        classes=classes,\n        pooling=pooling,\n    )\n\n\nEfficientNetB0.__doc__ = _get_model_by_name.__doc__\nEfficientNetB1.__doc__ = _get_model_by_name.__doc__\nEfficientNetB2.__doc__ = _get_model_by_name.__doc__\nEfficientNetB3.__doc__ = _get_model_by_name.__doc__\nEfficientNetB4.__doc__ = _get_model_by_name.__doc__\nEfficientNetB5.__doc__ = _get_model_by_name.__doc__\nEfficientNetB6.__doc__ = _get_model_by_name.__doc__\nEfficientNetB7.__doc__ = _get_model_by_name.__doc__","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-01-31T04:45:48.784906Z","iopub.execute_input":"2023-01-31T04:45:48.785450Z","iopub.status.idle":"2023-01-31T04:45:48.890172Z","shell.execute_reply.started":"2023-01-31T04:45:48.785396Z","shell.execute_reply":"2023-01-31T04:45:48.889312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\ntest_df = pd.read_csv('/kaggle/input/privatetest/test.csv')","metadata":{"execution":{"iopub.status.busy":"2023-01-31T04:45:48.893106Z","iopub.execute_input":"2023-01-31T04:45:48.893538Z","iopub.status.idle":"2023-01-31T04:45:48.921192Z","shell.execute_reply.started":"2023-01-31T04:45:48.893486Z","shell.execute_reply":"2023-01-31T04:45:48.920356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-31T04:45:48.922822Z","iopub.execute_input":"2023-01-31T04:45:48.923265Z","iopub.status.idle":"2023-01-31T04:45:48.933981Z","shell.execute_reply.started":"2023-01-31T04:45:48.923071Z","shell.execute_reply":"2023-01-31T04:45:48.933141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['diagnosis'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-01-31T04:45:48.935962Z","iopub.execute_input":"2023-01-31T04:45:48.936438Z","iopub.status.idle":"2023-01-31T04:45:48.945960Z","shell.execute_reply.started":"2023-01-31T04:45:48.936248Z","shell.execute_reply":"2023-01-31T04:45:48.944947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['diagnosis'].hist()\ntrain_df['diagnosis'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-01-31T04:45:48.949483Z","iopub.execute_input":"2023-01-31T04:45:48.950093Z","iopub.status.idle":"2023-01-31T04:45:49.304913Z","shell.execute_reply.started":"2023-01-31T04:45:48.949752Z","shell.execute_reply":"2023-01-31T04:45:49.303821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Displaying some Sample Images","metadata":{}},{"cell_type":"code","source":"def display_samples(df, columns=4, rows=3):\n    fig=plt.figure(figsize=(5*columns, 4*rows))\n\n    for i in range(columns*rows):\n        image_path = df.loc[i,'id_code']\n        image_id = df.loc[i,'diagnosis']\n        img = cv2.imread(f'../input/aptos2019-blindness-detection/train_images/{image_path}.png')\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        fig.add_subplot(rows, columns, i+1)\n        plt.title(image_id)\n        plt.axis('off')\n        plt.imshow(img)\n    \n    plt.tight_layout()\n\ndisplay_samples(train_df)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-01-31T04:45:49.312561Z","iopub.execute_input":"2023-01-31T04:45:49.318085Z","iopub.status.idle":"2023-01-31T04:45:54.576309Z","shell.execute_reply.started":"2023-01-31T04:45:49.318000Z","shell.execute_reply":"2023-01-31T04:45:54.575590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <a id='3'>Cropping images</a> \n\nWe will resize the images to 256x256, then create a single numpy array to hold the data.","metadata":{}},{"cell_type":"code","source":"def crop_image_from_gray(img,tol=7):\n    if img.ndim ==2:\n        mask = img>tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    elif img.ndim==3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img>tol\n        \n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0): # image is too dark so that we crop out everything,\n            return img # return original image\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n    #         print(img1.shape,img2.shape,img3.shape)\n            img = np.stack([img1,img2,img3],axis=-1)\n    #         print(img.shape)\n        return img\n    \ndef preprocess_image(path, sigmaX=10):\n    image = cv2.imread(path)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    image = crop_image_from_gray(image)\n    image = cv2.resize(image, (IMG_SIZE, IMG_SIZE))\n#     image=cv2.addWeighted ( image,4, cv2.GaussianBlur( image , (0,0) , sigmaX) ,-4 ,128)\n    image=CLAHEgreen(image)\n        \n    return image\n\ndef CLAHEgreen(image):\n    green=image[:, :, 1]\n    clipLimit = 2.0\n    tileGridSize = (8,8)\n    clahe=cv2.createCLAHE(clipLimit = clipLimit, tileGridSize = tileGridSize)\n    cla=clahe.apply(green)\n#     cla=clahe.apply(cla)\n    img=cv2.merge((cla,cla,cla))\n    \n    return img\n\n","metadata":{"execution":{"iopub.status.busy":"2023-01-31T04:45:54.577777Z","iopub.execute_input":"2023-01-31T04:45:54.578279Z","iopub.status.idle":"2023-01-31T04:45:54.596566Z","shell.execute_reply.started":"2023-01-31T04:45:54.578229Z","shell.execute_reply":"2023-01-31T04:45:54.595279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_histograms(df,columns=4, rows=3):\n    ax, fig=plt.subplots(columns*rows,figsize=(5*columns, 4*rows))\n\n    for i in range(columns*rows):\n        image_path = df.loc[i,'id_code']\n        image_id = df.loc[i,'diagnosis']\n        plt.subplot(columns, rows, i+1)\n        img = cv2.imread(f'../input/aptos2019-blindness-detection/train_images/{image_path}.png')\n        plt.hist(img.flatten(),256,[0,256],color='r')\n#         fig.add_subplot(rows, columns, i+1)\n        plt.title(image_id)\n#         plt.axis('off')\n#         plt.imshow(img)\n    \n    plt.tight_layout()    ","metadata":{"execution":{"iopub.status.busy":"2023-01-31T04:45:54.598156Z","iopub.execute_input":"2023-01-31T04:45:54.598631Z","iopub.status.idle":"2023-01-31T04:45:54.613597Z","shell.execute_reply.started":"2023-01-31T04:45:54.598398Z","shell.execute_reply":"2023-01-31T04:45:54.612458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_histograms(train_df)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T04:45:54.615250Z","iopub.execute_input":"2023-01-31T04:45:54.615895Z","iopub.status.idle":"2023-01-31T04:46:10.427409Z","shell.execute_reply.started":"2023-01-31T04:45:54.615480Z","shell.execute_reply":"2023-01-31T04:46:10.426540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_histograms_preprocess(df,columns=4, rows=3):\n    ax, fig=plt.subplots(columns*rows,figsize=(5*columns, 4*rows))\n\n    for i in range(columns*rows):\n        image_path = df.loc[i,'id_code']\n        image_id = df.loc[i,'diagnosis']\n        plt.subplot(columns, rows, i+1)\n        img = preprocess_image(f'../input/aptos2019-blindness-detection/train_images/{image_path}.png')\n        plt.hist(img.flatten(),256,[0,256],color='r')\n#         fig.add_subplot(rows, columns, i+1)\n        plt.title(image_id)\n#         plt.axis('off')\n#         plt.imshow(img)\n    \n    plt.tight_layout()    ","metadata":{"execution":{"iopub.status.busy":"2023-01-31T04:46:10.428849Z","iopub.execute_input":"2023-01-31T04:46:10.429304Z","iopub.status.idle":"2023-01-31T04:46:10.438433Z","shell.execute_reply.started":"2023-01-31T04:46:10.429248Z","shell.execute_reply":"2023-01-31T04:46:10.437712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_histograms_preprocess(train_df)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T04:46:10.440187Z","iopub.execute_input":"2023-01-31T04:46:10.440778Z","iopub.status.idle":"2023-01-31T04:46:26.525796Z","shell.execute_reply.started":"2023-01-31T04:46:10.440723Z","shell.execute_reply":"2023-01-31T04:46:26.524704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display_samples_gaussian(df, columns=4, rows=3):\n    fig=plt.figure(figsize=(5*columns, 4*rows))\n\n    for i in range(columns*rows):\n        image_path = df.loc[i,'id_code']\n        image_id = df.loc[i,'diagnosis']\n        img = preprocess_image(f'../input/aptos2019-blindness-detection/train_images/{image_path}.png')\n        \n        fig.add_subplot(rows, columns, i+1)\n        plt.title(image_id)\n        plt.axis('off')\n        plt.imshow(img)\n    \n    plt.tight_layout()\n\ndisplay_samples_gaussian(train_df)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T04:46:26.527288Z","iopub.execute_input":"2023-01-31T04:46:26.527821Z","iopub.status.idle":"2023-01-31T04:46:30.798382Z","shell.execute_reply.started":"2023-01-31T04:46:26.527754Z","shell.execute_reply":"2023-01-31T04:46:30.797446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N = train_df.shape[0]\nx_train = np.empty((N, IMG_SIZE, IMG_SIZE, 3), dtype=np.uint8)\n\nfor i, image_id in enumerate(tqdm(train_df['id_code'])):\n    x_train[i, :, :, :] = preprocess_image(\n        f'../input/aptos2019-blindness-detection/train_images/{image_id}.png'\n    )","metadata":{"execution":{"iopub.status.busy":"2023-01-31T04:46:30.799617Z","iopub.execute_input":"2023-01-31T04:46:30.800029Z","iopub.status.idle":"2023-01-31T04:56:41.079901Z","shell.execute_reply.started":"2023-01-31T04:46:30.799982Z","shell.execute_reply":"2023-01-31T04:56:41.078780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = pd.get_dummies(train_df['diagnosis']).values\n\n\nprint(y_train.shape)\n","metadata":{"execution":{"iopub.status.busy":"2023-01-31T04:56:41.081256Z","iopub.execute_input":"2023-01-31T04:56:41.081803Z","iopub.status.idle":"2023-01-31T04:56:41.089291Z","shell.execute_reply.started":"2023-01-31T04:56:41.081743Z","shell.execute_reply":"2023-01-31T04:56:41.088354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating multilabels\n\nInstead of predicting a single label, we will change our target to be a multilabel problem; i.e., if the target is a certain class, then it encompasses all the classes before it. E.g. encoding a class 4 retinopathy would usually be `[0, 0, 0, 1]`, but in our case we will predict `[1, 1, 1, 1]`. For more details, please check out [Lex's kernel](https://www.kaggle.com/lextoumbourou/blindness-detection-resnet34-ordinal-targets).","metadata":{}},{"cell_type":"code","source":"y_train_multi = np.empty(y_train.shape, dtype=y_train.dtype)\ny_train_multi[:, 4] = y_train[:, 4]\n\nfor i in range(3, -1, -1):\n    y_train_multi[:, i] = np.logical_or(y_train[:, i], y_train_multi[:, i+1])\n\nprint(\"Original y_train:\", y_train.sum(axis=0))\nprint(\"Multilabel version:\", y_train_multi.sum(axis=0))","metadata":{"execution":{"iopub.status.busy":"2023-01-31T04:56:41.090550Z","iopub.execute_input":"2023-01-31T04:56:41.091072Z","iopub.status.idle":"2023-01-31T04:56:41.104014Z","shell.execute_reply.started":"2023-01-31T04:56:41.091018Z","shell.execute_reply":"2023-01-31T04:56:41.102842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now we can split it into a training and validation set.","metadata":{}},{"cell_type":"code","source":"x_train, x_val, y_train, y_val = train_test_split(\n    x_train, y_train_multi, \n    test_size=0.15, \n    random_state=2019\n)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T04:56:41.105463Z","iopub.execute_input":"2023-01-31T04:56:41.106147Z","iopub.status.idle":"2023-01-31T04:56:41.757659Z","shell.execute_reply.started":"2023-01-31T04:56:41.105964Z","shell.execute_reply":"2023-01-31T04:56:41.756607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_val=x_val/255","metadata":{"execution":{"iopub.status.busy":"2023-01-31T04:56:41.759109Z","iopub.execute_input":"2023-01-31T04:56:41.759412Z","iopub.status.idle":"2023-01-31T04:56:42.406738Z","shell.execute_reply.started":"2023-01-31T04:56:41.759364Z","shell.execute_reply":"2023-01-31T04:56:42.405950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['diagnosis']=train_df['diagnosis'].astype(str)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T04:56:42.410648Z","iopub.execute_input":"2023-01-31T04:56:42.410937Z","iopub.status.idle":"2023-01-31T04:56:42.418323Z","shell.execute_reply.started":"2023-01-31T04:56:42.410886Z","shell.execute_reply":"2023-01-31T04:56:42.417282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n\ndatagen =  ImageDataGenerator(\n        zoom_range=0.6,  # set range for random zoom, changed from 0.15 to 0.3, now changed from 0.3 to 0.45, from 0.45 to 0.6\n        # set mode for filling points outside the input boundaries\n        fill_mode='constant',\n        cval=0.,  # value used for fill_mode = \"constant\"\n        horizontal_flip=True,  # randomly flip images\n        vertical_flip=True,# randomly flip images\n        rotation_range=360,\n        width_shift_range=0.1,\n        height_shift_range=0.1,\n        rescale=1./255\n    )","metadata":{"execution":{"iopub.status.busy":"2023-01-31T04:56:42.419960Z","iopub.execute_input":"2023-01-31T04:56:42.420531Z","iopub.status.idle":"2023-01-31T04:56:42.431141Z","shell.execute_reply.started":"2023-01-31T04:56:42.420476Z","shell.execute_reply":"2023-01-31T04:56:42.430201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_generator = datagen.flow(x_train, y_train, batch_size=BATCH_SIZE, seed=2019)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T04:56:42.432873Z","iopub.execute_input":"2023-01-31T04:56:42.433351Z","iopub.status.idle":"2023-01-31T04:56:44.414300Z","shell.execute_reply.started":"2023-01-31T04:56:42.433182Z","shell.execute_reply":"2023-01-31T04:56:44.413413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <a id='5'>Quadratic Weighted Kappa</a> \n\nQuadratic Weighted Kappa (QWK, the greek letter $\\kappa$), also known as Cohen's Kappa, is the official evaluation metric. For our kernel, we will use a custom callback to monitor the score, and plot it at the end.\n\n### What is Cohen Kappa?\n\nAccording to the [wikipedia article](https://en.wikipedia.org/wiki/Cohen%27s_kappa), we have\n> The definition of $\\kappa$ is:\n> $$\\kappa \\equiv \\frac{p_o - p_e}{1 - p_e}$$\n> where $p_o$ is the relative observed agreement among raters (identical to accuracy), and $p_e$ is the hypothetical probability of chance agreement, using the observed data to calculate the probabilities of each observer randomly seeing each category.\n\n### How is it computed?\n\nLet's take the example of a binary classification problem. Say we have:","metadata":{}},{"cell_type":"code","source":"true_labels = np.array([1, 0, 1, 1, 0, 1])\npred_labels = np.array([1, 0, 0, 0, 0, 1])","metadata":{"execution":{"iopub.status.busy":"2023-01-31T04:56:44.415652Z","iopub.execute_input":"2023-01-31T04:56:44.415947Z","iopub.status.idle":"2023-01-31T04:56:44.421154Z","shell.execute_reply.started":"2023-01-31T04:56:44.415890Z","shell.execute_reply":"2023-01-31T04:56:44.420288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can construct the following table:\n\n| true | pred | agreement      |\n|------|------|----------------|\n| 1    | 1    | true positive  |\n| 0    | 0    | true negative  |\n| 1    | 0    | false negative |\n| 1    | 0    | false negative |\n| 0    | 0    | true negative  |\n| 1    | 1    | true positive  |\n\n\nThen the \"observed proportionate agreement\" is calculated exactly the same way as accuracy:\n\n$$\np_o = acc = \\frac{tp + tn}{all} = {2 + 2}{6} = 0.66\n$$\n\nThis can be confirmed using scikit-learn:","metadata":{}},{"cell_type":"code","source":"accuracy_score(true_labels, pred_labels)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T04:56:44.422657Z","iopub.execute_input":"2023-01-31T04:56:44.423178Z","iopub.status.idle":"2023-01-31T04:56:44.433008Z","shell.execute_reply.started":"2023-01-31T04:56:44.423126Z","shell.execute_reply":"2023-01-31T04:56:44.432016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Additionally, we also need to compute `p_e`:\n\n$$p_{yes} = \\frac{tp + fp}{all} \\frac{tp + fn}{all} = \\frac{2}{6} \\frac{4}{6} = 0.222$$\n\n$$p_{no} = \\frac{fn + tn}{all} \\frac{fp + tn}{all} = \\frac{4}{6} \\frac{2}{6} = 0.222$$\n\n$$p_{e} = p_{yes} + p_{no} = 0.222 + 0.222 = 0.444$$\n\nFinally,\n\n$$\n\\kappa = \\frac{p_o - p_e}{1-p_e} = \\frac{0.666 - 0.444}{1 - 0.444} = 0.4\n$$\n\nLet's verify with scikit-learn:","metadata":{}},{"cell_type":"code","source":"cohen_kappa_score(true_labels, pred_labels)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T04:56:44.434480Z","iopub.execute_input":"2023-01-31T04:56:44.435013Z","iopub.status.idle":"2023-01-31T04:56:44.442615Z","shell.execute_reply.started":"2023-01-31T04:56:44.434962Z","shell.execute_reply":"2023-01-31T04:56:44.441660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### What is the weighted kappa?\n\nThe wikipedia page offer a very concise explanation: \n> The weighted kappa allows disagreements to be weighted differently and is especially useful when **codes are ordered**. Three matrices are involved, the matrix of observed scores, the matrix of expected scores based on chance agreement, and the weight matrix. Weight matrix cells located on the diagonal (upper-left to bottom-right) represent agreement and thus contain zeros. Off-diagonal cells contain weights indicating the seriousness of that disagreement.\n\nSimply put, if two scores disagree, then the penalty will depend on how far they are apart. That means that our score will be higher if (a) the real value is 4 but the model predicts a 3, and the score will be lower if (b) the model instead predicts a 0. This metric makes sense for this competition, since the labels 0-4 indicates how severe the illness is. Intuitively, a model that predicts a severe retinopathy (3) when it is in reality a proliferative retinopathy (4) is probably better than a model that predicts a mild retinopathy (1).","metadata":{}},{"cell_type":"markdown","source":"### Creating keras callback for QWK","metadata":{}},{"cell_type":"code","source":"class Metrics(Callback):\n    def on_train_begin(self, logs={}):\n        self.val_kappas = []\n\n    def on_epoch_end(self, epoch, logs={}):\n\n        X_val, y_val = self.validation_data[:2]\n        y_val = y_val.sum(axis=1) - 1\n        \n        y_pred = self.model.predict(X_val) > 0.5\n        y_pred = y_pred.astype(int).sum(axis=1) - 1\n        \n        _val_kappa = cohen_kappa_score(\n            y_val,\n            y_pred, \n            weights='quadratic'\n        )\n\n        self.val_kappas.append(_val_kappa)\n\n        print(f\"val_kappa: {_val_kappa:.4f}\")\n        \n        if _val_kappa == max(self.val_kappas):\n            print(\"Validation Kappa has improved. Saving model.\")\n            model.save_weights('model.h5')\n            model_json = model.to_json()\n            with open('model.json', \"w\") as json_file:\n                json_file.write(model_json)\n            json_file.close()\n\n        return","metadata":{"execution":{"iopub.status.busy":"2023-01-31T04:56:44.444123Z","iopub.execute_input":"2023-01-31T04:56:44.444652Z","iopub.status.idle":"2023-01-31T04:56:44.455184Z","shell.execute_reply.started":"2023-01-31T04:56:44.444586Z","shell.execute_reply":"2023-01-31T04:56:44.454219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <a id='6'>Model : EfficientNetB5 </a> ","metadata":{}},{"cell_type":"code","source":"efficient = EfficientNetB5(\n    weights=None,\n    include_top=False,\n    input_shape=(IMG_SIZE,IMG_SIZE,3)\n)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T04:56:44.456590Z","iopub.execute_input":"2023-01-31T04:56:44.457164Z","iopub.status.idle":"2023-01-31T04:57:00.107949Z","shell.execute_reply.started":"2023-01-31T04:56:44.457111Z","shell.execute_reply":"2023-01-31T04:57:00.107060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"efficient.load_weights('../input/efficientnet-keras-weights-b0b5/efficientnet-b5_imagenet_1000_notop.h5')","metadata":{"execution":{"iopub.status.busy":"2023-01-31T04:57:00.109259Z","iopub.execute_input":"2023-01-31T04:57:00.109550Z","iopub.status.idle":"2023-01-31T04:57:16.158726Z","shell.execute_reply.started":"2023-01-31T04:57:00.109501Z","shell.execute_reply":"2023-01-31T04:57:16.157783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_model():\n    model = Sequential()\n    model.add(efficient)\n    model.add(layers.GlobalAveragePooling2D())\n    model.add(layers.Dropout(0.5))\n    model.add(layers.Dense(5, activation='sigmoid'))\n    \n    \n    model.compile(\n        loss='binary_crossentropy',\n        optimizer=Adam(lr=0.00005),\n        metrics=['accuracy']\n    )\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2023-01-31T04:57:16.160272Z","iopub.execute_input":"2023-01-31T04:57:16.160740Z","iopub.status.idle":"2023-01-31T04:57:16.173747Z","shell.execute_reply.started":"2023-01-31T04:57:16.160687Z","shell.execute_reply":"2023-01-31T04:57:16.173034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = build_model()\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-01-31T04:57:16.175294Z","iopub.execute_input":"2023-01-31T04:57:16.176345Z","iopub.status.idle":"2023-01-31T04:57:28.338597Z","shell.execute_reply.started":"2023-01-31T04:57:16.176287Z","shell.execute_reply":"2023-01-31T04:57:28.337784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <a id='7'>Training and Evaluation</a> ","metadata":{}},{"cell_type":"code","source":"kappa_metrics = Metrics()\nest=EarlyStopping(monitor='val_loss',patience=5, min_delta=0.005)\ncall_backs=[est,kappa_metrics]\n\nhistory = model.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / BATCH_SIZE,\n    validation_data=(x_val,y_val),\n     epochs=15,\n    callbacks=call_backs)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T04:57:28.340043Z","iopub.execute_input":"2023-01-31T04:57:28.340328Z","iopub.status.idle":"2023-01-31T05:29:56.455067Z","shell.execute_reply.started":"2023-01-31T04:57:28.340279Z","shell.execute_reply":"2023-01-31T05:29:56.448647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.load_weights('model.h5')\n\ntest_pred=[]\n\nfor i,img_code in enumerate(tqdm(test_df['id_code'])):\n    img=preprocess_image(f'/kaggle/input/privatetest/test/{img_code}.png')\n    pred=model.predict(img[np.newaxis]/255)>0.5\n    pred=pred.astype(int).sum(axis=1) - 1\n    test_pred.append(pred)\n","metadata":{"execution":{"iopub.status.busy":"2023-01-31T05:29:56.460265Z","iopub.execute_input":"2023-01-31T05:29:56.464684Z","iopub.status.idle":"2023-01-31T05:30:08.513682Z","shell.execute_reply.started":"2023-01-31T05:29:56.460518Z","shell.execute_reply":"2023-01-31T05:30:08.512779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred=np.concatenate(test_pred,axis=0)\ntest_pred=pd.Series(test_pred)\ntest_df['diagnosis'] = test_pred\ntest_df.to_csv('submission_new.csv',index=False)\nprint(test_df.head())","metadata":{"execution":{"iopub.status.busy":"2023-01-31T05:30:08.514898Z","iopub.execute_input":"2023-01-31T05:30:08.515197Z","iopub.status.idle":"2023-01-31T05:30:08.691729Z","shell.execute_reply.started":"2023-01-31T05:30:08.515147Z","shell.execute_reply":"2023-01-31T05:30:08.690960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dist = (test_df.diagnosis.value_counts()/len(test_df))*100\nprint('Prediction distribution:')\nprint(dist)\ntest_df.diagnosis.hist()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-31T05:30:08.695679Z","iopub.execute_input":"2023-01-31T05:30:08.696130Z","iopub.status.idle":"2023-01-31T05:30:08.980369Z","shell.execute_reply.started":"2023-01-31T05:30:08.695935Z","shell.execute_reply":"2023-01-31T05:30:08.979183Z"},"trusted":true},"execution_count":null,"outputs":[]}]}