{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Thanks to https://www.kaggle.com/xhlulu/ranzcr-efficientnet-tpu-training","metadata":{}},{"cell_type":"code","source":"!pip install efficientnet -q","metadata":{"execution":{"iopub.status.busy":"2021-08-03T00:50:18.962799Z","iopub.execute_input":"2021-08-03T00:50:18.963399Z","iopub.status.idle":"2021-08-03T00:50:28.457802Z","shell.execute_reply.started":"2021-08-03T00:50:18.963312Z","shell.execute_reply":"2021-08-03T00:50:28.456482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!git clone https://github.com/google/automl.git\nimport sys\nsys.path.insert(0,'/kaggle/working/automl/efficientnetv2')","metadata":{"execution":{"iopub.status.busy":"2021-08-03T00:50:28.459725Z","iopub.execute_input":"2021-08-03T00:50:28.460048Z","iopub.status.idle":"2021-08-03T00:50:31.6709Z","shell.execute_reply.started":"2021-08-03T00:50:28.460013Z","shell.execute_reply":"2021-08-03T00:50:31.669773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\nimport efficientnet.tfkeras as efn\nimport numpy as np\nimport pandas as pd\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.model_selection import train_test_split\nimport tensorflow as tf\nfrom sklearn.model_selection import GroupKFold\nimport math\nfrom kaggle_datasets import KaggleDatasets\nimport tensorflow as tf\nimport effnetv2_model","metadata":{"execution":{"iopub.status.busy":"2021-08-03T00:50:31.672603Z","iopub.execute_input":"2021-08-03T00:50:31.67291Z","iopub.status.idle":"2021-08-03T00:50:39.335603Z","shell.execute_reply.started":"2021-08-03T00:50:31.672878Z","shell.execute_reply":"2021-08-03T00:50:39.334701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# units","metadata":{}},{"cell_type":"code","source":"def choice(p, image1, image2):\n    rnd = random_float()\n    image = tf.where(rnd <= p, image1, image2)\n    return image\n\ndef random_float(minval=0.0, maxval=1.0):\n    rnd = tf.random.uniform(\n        [], minval=minval, maxval=maxval, dtype=tf.float32)\n    return rnd","metadata":{"execution":{"iopub.status.busy":"2021-08-03T00:50:39.337518Z","iopub.execute_input":"2021-08-03T00:50:39.337916Z","iopub.status.idle":"2021-08-03T00:50:39.344748Z","shell.execute_reply.started":"2021-08-03T00:50:39.337874Z","shell.execute_reply":"2021-08-03T00:50:39.343333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mirror_boundary(v, max_v):\n    # v % (max_v*2.0-2.0) ==> v % (512*2-2) ==> [0..1022]\n    # [0..1022] - (max_v-1.0) ==> [0..1022] - 511 ==> [-511..511]\n    # -1.0 * abs([-511..511]) ==> [-511..0]\n    # [-511..0] + max_v - 1.0 ==> [-511..0] + 511 ==> [0..511]\n    mirror_v = -1.0 * tf.math.abs(\n        v % (max_v*2.0-2.0) - (max_v-1.0)) + max_v-1.0\n    return mirror_v\n\ndef clip_boundary(v, max_v):\n    clip_v = tf.clip_by_value(v, 0.0, max_v-1.0)\n    return clip_v\n\ndef interpolate_bilinear(image, map_x, map_y):\n    def _gather(image, map_x, map_y):\n        map_stack = tf.stack([map_x, map_y]) # [ 2, height, width ]\n        map_indices = tf.transpose(\n            map_stack, perm=[1, 2, 0])       # [ height, width, 2 ]\n        map_indices = tf.cast(map_indices, dtype=tf.int32)\n        gather_image = tf.gather_nd(image, map_indices)\n        return gather_image\n    \n    ll = _gather(image, tf.math.floor(map_x), tf.math.floor(map_y))\n    lr = _gather(image, tf.math.ceil(map_x), tf.math.floor(map_y))\n    ul = _gather(image, tf.math.floor(map_x), tf.math.ceil(map_y))\n    ur = _gather(image, tf.math.ceil(map_x), tf.math.ceil(map_y))\n    \n    fraction_x = tf.expand_dims(map_x % 1.0, axis=-1) # [h, w, 1]\n    int_l = (lr - ll) * fraction_x + ll\n    int_u = (ur - ul) * fraction_x + ul\n    \n    fraction_y = tf.expand_dims(map_y % 1.0, axis=-1) # [h, w, 1]\n    interpolate_image = (int_u - int_l) * fraction_y + int_l\n    return interpolate_image\n\ndef remap(image, height, width, map_x, map_y, mode):\n    assert \\\n        mode in ('mirror', 'constant'), \\\n        \"mode is neither 'mirror' nor 'constant'\"\n\n    height_f = tf.cast(height, dtype=tf.float32)\n    width_f = tf.cast(width, dtype=tf.float32)\n    map_x = tf.reshape(map_x, shape=[height, width])\n    map_y = tf.reshape(map_y, shape=[height, width])\n    if mode == 'mirror':\n        b_map_x = mirror_boundary(map_x, width_f)\n        b_map_y = mirror_boundary(map_y, height_f)\n    else:\n        b_map_x = clip_boundary(map_x, width_f)\n        b_map_y = clip_boundary(map_y, height_f)\n        \n    image_remap = interpolate_bilinear(image, b_map_x, b_map_y)\n    \n    if mode == 'constant':\n        map_stack = tf.stack([map_x, map_y])\n        map_indices = tf.transpose(map_stack, perm=[1, 2, 0])\n        x_ge_0 = (0.0 <= map_indices[ : , : , 0])    # [h, w]\n        x_lt_w = (map_indices[ : , : , 0] < width_f)\n        y_ge_0 = (0.0 <= map_indices[ : , : , 1])\n        y_lt_h = (map_indices[ : , : , 1] < height_f)\n        inside_boundary = tf.math.reduce_all(\n            tf.stack([x_ge_0, x_lt_w, y_ge_0, y_lt_h]), axis=0) # [h, w]\n        inside_boundary = inside_boundary[ : , : , tf.newaxis]  # [h, w, 1]\n        image_remap = tf.where(inside_boundary, image_remap, 0.0)\n\n    return image_remap","metadata":{"execution":{"iopub.status.busy":"2021-08-03T00:50:39.346553Z","iopub.execute_input":"2021-08-03T00:50:39.346952Z","iopub.status.idle":"2021-08-03T00:50:39.36874Z","shell.execute_reply.started":"2021-08-03T00:50:39.34691Z","shell.execute_reply":"2021-08-03T00:50:39.367325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Optical Distortion","metadata":{}},{"cell_type":"code","source":"def initUndistortRectifyMap(height, width, k, dx, dy):\n    height = tf.cast(height, dtype=tf.float32)\n    width = tf.cast(width, dtype=tf.float32)\n    \n    f_x = width\n    f_y = height\n    c_x = width * 0.5 + dx\n    c_y = height * 0.5 + dy\n    \n    f_dash_x = f_x\n    c_dash_x = (width - 1.0) * 0.5\n    f_dash_y = f_y\n    c_dash_y = (height - 1.0) * 0.5\n\n    h_rng = tf.range(height, dtype=tf.float32)\n    w_rng = tf.range(width, dtype=tf.float32)\n    v, u = tf.meshgrid(h_rng, w_rng)\n    \n    x = (u - c_dash_x) / f_dash_x\n    y = (v - c_dash_y) / f_dash_y\n    x_dash = x\n    y_dash = y\n    \n    r_2 = x_dash * x_dash + y_dash * y_dash\n    r_4 = r_2 * r_2\n    x_dash_dash = x_dash * (1 + k*r_2 + k*r_4)\n    y_dash_dash = y_dash * (1 + k*r_2 + k*r_4)\n\n    map_x = x_dash_dash * f_x + c_x\n    map_y = y_dash_dash * f_y + c_y\n    return map_x, map_y\n\n\ndef OpticalDistortion(distort_limit, shift_limit, p=1.0):\n    def _do_optical_distortion(image):\n        k = random_float(-distort_limit, distort_limit)\n        dx = random_float(-shift_limit, shift_limit)\n        dy = random_float(-shift_limit, shift_limit)\n        image_shape = tf.shape(image)\n        height = image_shape[0]\n        width = image_shape[1]\n        map_x, map_y = initUndistortRectifyMap(\n            height, width, k, dx, dy)\n        aug_image = remap(\n            image, height, width, map_x, map_y, mode='mirror')\n        return choice(p, aug_image, image)\n    return _do_optical_distortion\n\noptical_distortion = OpticalDistortion(\n    distort_limit=1.0, shift_limit=0.05, p=0.75)","metadata":{"execution":{"iopub.status.busy":"2021-08-03T00:50:39.370391Z","iopub.execute_input":"2021-08-03T00:50:39.370835Z","iopub.status.idle":"2021-08-03T00:50:39.387229Z","shell.execute_reply.started":"2021-08-03T00:50:39.37079Z","shell.execute_reply":"2021-08-03T00:50:39.385898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# HorizontalFlip","metadata":{}},{"cell_type":"code","source":"def HorizontalFlip(p):\n    def _do_horizontal_flip(image):\n        aug_image = tf.image.flip_left_right(image)\n        return choice(p, aug_image, image)\n    return _do_horizontal_flip\n\nhorizonflip = HorizontalFlip(0.65)","metadata":{"execution":{"iopub.status.busy":"2021-08-03T00:50:39.388708Z","iopub.execute_input":"2021-08-03T00:50:39.389101Z","iopub.status.idle":"2021-08-03T00:50:39.414821Z","shell.execute_reply.started":"2021-08-03T00:50:39.389054Z","shell.execute_reply":"2021-08-03T00:50:39.413469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# updown_Flip","metadata":{}},{"cell_type":"code","source":"def Updown_Flip(p):\n    def _do_up_down_flip(image):\n        aug_image = tf.image.random_flip_up_down(image)\n        return choice(p, aug_image, image)\n    return _do_up_down_flip\n\nupdown_flip = Updown_Flip(0.65)","metadata":{"execution":{"iopub.status.busy":"2021-08-03T00:50:39.417686Z","iopub.execute_input":"2021-08-03T00:50:39.418056Z","iopub.status.idle":"2021-08-03T00:50:39.427682Z","shell.execute_reply.started":"2021-08-03T00:50:39.418022Z","shell.execute_reply":"2021-08-03T00:50:39.426598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# RandomContrast","metadata":{}},{"cell_type":"code","source":"def RandomContrast(lower, upper, p):\n    def _do_random_contrast(image):\n        aug_image = tf.image.random_contrast(image, lower, upper)\n        return choice(p, aug_image, image)\n    return _do_random_contrast\n\nrandom_contrast = RandomContrast(lower=0.2, upper=0.8, p=0.75)","metadata":{"execution":{"iopub.status.busy":"2021-08-03T00:50:39.429861Z","iopub.execute_input":"2021-08-03T00:50:39.430186Z","iopub.status.idle":"2021-08-03T00:50:39.440355Z","shell.execute_reply.started":"2021-08-03T00:50:39.430157Z","shell.execute_reply":"2021-08-03T00:50:39.43911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# HeuSaturationValue","metadata":{}},{"cell_type":"code","source":"def HueSaturationValue(\n        hue_shift_limit, sat_shift_limit, val_shift_limit, p):\n    def _do_hue_saturation_value(image):\n        hsv_image = tf.image.rgb_to_hsv(image)\n        hue_shift = random_float(-hue_shift_limit, hue_shift_limit)\n        sat_shift = random_float(-sat_shift_limit, sat_shift_limit)\n        val_shift = random_float(-val_shift_limit, val_shift_limit)\n\n        hue_values = (hsv_image[ ... , :1 ] + hue_shift) % 1.0\n        sat_values = tf.clip_by_value(\n            hsv_image[ ... , 1:2 ] + sat_shift, 0.0, 1.0)\n        val_values = tf.clip_by_value(\n            hsv_image[ ... , 2: ] + val_shift, 0.0, 1.0)\n        hsv_image = tf.concat(\n            [hue_values, sat_values, val_values], axis=-1)\n        aug_image = tf.image.hsv_to_rgb(hsv_image)\n        return choice(p, aug_image, image)\n    return _do_hue_saturation_value\n\nhue_saturation_value = HueSaturationValue(\n    hue_shift_limit=0.2, sat_shift_limit=0.3,\n    val_shift_limit=0.2, p=0.75)","metadata":{"execution":{"iopub.status.busy":"2021-08-03T00:50:39.442115Z","iopub.execute_input":"2021-08-03T00:50:39.442566Z","iopub.status.idle":"2021-08-03T00:50:39.458024Z","shell.execute_reply.started":"2021-08-03T00:50:39.44252Z","shell.execute_reply":"2021-08-03T00:50:39.456728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ShiftScaleRotate","metadata":{}},{"cell_type":"code","source":"def affine_transform(height, width, tx, ty, z, theta):\n    cx = (width - 1.0) * 0.5\n    cy = (height - 1.0) * 0.5\n    \n    center_shift_mat = tf.convert_to_tensor([\n        [1.0, 0.0, -cx],\n        [0.0, 1.0, -cy],\n        [0.0, 0.0, 1.0]], dtype=tf.float32)\n    trans_mat = center_shift_mat\n    \n    rot_rad = -2.0 * math.pi * theta / 360.0\n    roration_mat = tf.convert_to_tensor([\n        [tf.math.cos(rot_rad), tf.math.sin(rot_rad), 0.0],\n        [-tf.math.sin(rot_rad), tf.math.cos(rot_rad), 0.0],\n        [0.0, 0.0, 1.0]], dtype=tf.float32)\n    trans_mat = tf.linalg.matmul(roration_mat, trans_mat)\n    \n    shift_mat = tf.convert_to_tensor([\n        [1.0, 0.0, cx - tx],\n        [0.0, 1.0, cy - ty],\n        [0.0, 0.0, 1.0]], dtype=tf.float32)\n    trans_mat = tf.linalg.matmul(shift_mat, trans_mat)\n\n    zoom_mat = tf.convert_to_tensor([\n        [1.0 / z, 0.0, 0.0],\n        [0.0, 1.0 / z, 0.0],\n        [0.0, 0.0, 1.0]], dtype=tf.float32)\n    trans_mat = tf.linalg.matmul(zoom_mat, trans_mat)\n    \n    h_rng = tf.range(height, dtype=tf.float32)\n    w_rng = tf.range(width, dtype=tf.float32)\n    y, x = tf.meshgrid(h_rng, w_rng)\n    x = tf.reshape(x, [-1])\n    y = tf.reshape(y, [-1])\n    ones = tf.ones_like(x)\n    coord_mat = tf.stack([x, y, ones])\n    \n    res_mat = tf.linalg.matmul(trans_mat, coord_mat)\n    map_x = res_mat[0]\n    map_y = res_mat[1]\n    return map_x, map_y","metadata":{"execution":{"iopub.status.busy":"2021-08-03T00:50:39.459532Z","iopub.execute_input":"2021-08-03T00:50:39.459893Z","iopub.status.idle":"2021-08-03T00:50:39.475434Z","shell.execute_reply.started":"2021-08-03T00:50:39.459862Z","shell.execute_reply":"2021-08-03T00:50:39.47416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ShiftScaleRotate(\n        shift_limit, scale_limit, rotate_limit, p):\n    def _do_shift_scale_rotate(image):\n        image_shape = tf.shape(image)\n        height_i = image_shape[0]\n        width_i = image_shape[1]\n        height_f = tf.cast(height_i, dtype=tf.float32)\n        width_f = tf.cast(width_i, dtype=tf.float32)\n        tx = width_f * random_float(-shift_limit, shift_limit)\n        ty = height_f * random_float(-shift_limit, shift_limit)\n        z = random_float(1.0 - scale_limit, 1.0 + scale_limit)\n        theta = random_float(-rotate_limit, rotate_limit)\n\n        map_x, map_y = affine_transform(\n            height_f, width_f, tx, ty, z, theta)\n        aug_image = remap(\n            image, height_i, width_i, map_x, map_y, mode='constant')\n        return choice(p, aug_image, image)\n    return _do_shift_scale_rotate\n\nshift_scale_rotate = ShiftScaleRotate(\n    shift_limit=0.2, scale_limit=0.3, rotate_limit=30, p=0.75)","metadata":{"execution":{"iopub.status.busy":"2021-08-03T00:50:39.476949Z","iopub.execute_input":"2021-08-03T00:50:39.477264Z","iopub.status.idle":"2021-08-03T00:50:39.494015Z","shell.execute_reply.started":"2021-08-03T00:50:39.477232Z","shell.execute_reply":"2021-08-03T00:50:39.492802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# cut out","metadata":{}},{"cell_type":"code","source":"IMSIZE = (224, 240, 260, 300, 380, 456, 528, 600, 768)\nimage_size = 600\nIMS = 7\n\ndef randints(shape, minval, maxval):\n    # maxval+1 to include maxval for the result.\n    # generated range is [minval, maxval) (maxval is not included)\n    return tf.random.uniform(\n        shape=shape, minval=minval, maxval=maxval+1, dtype=tf.int32)\n\ndef make_range_masks(size, starts, ends):\n    indice = tf.range(size, dtype=tf.int32)\n    start_masks = (\n        starts[ : , tf.newaxis] <= indice[  tf.newaxis, : ])\n    end_masks = (\n        indice[ tf.newaxis, : ] <= ends[ : , tf.newaxis])\n    range_masks = start_masks & end_masks\n    return range_masks\n\ndef make_region_mask(tops, lefts, bottoms, rights):\n    row_masks = make_range_masks(image_size, tops, bottoms)\n    col_masks = make_range_masks(image_size, lefts, rights)\n    region_masks = \\\n        row_masks[ : , : , tf.newaxis ] & \\\n        col_masks[ : , tf.newaxis, : ]\n    region_mask = tf.math.reduce_any(region_masks, axis=0)\n    region_mask = region_mask[ : , : , tf.newaxis]\n    return region_mask\n\ndef Cutout(num_cuts, mask_factor, p):\n    def _do_cutout(image):\n        image_shape = tf.shape(image)\n        height_i = image_shape[0]\n        width_i = image_shape[1]\n        height_f = tf.cast(height_i, dtype=tf.float32)\n        width_f = tf.cast(width_i, dtype=tf.float32)\n        cut_h = tf.cast(height_f * mask_factor, dtype=tf.int32)\n        cut_w = tf.cast(width_f * mask_factor, dtype=tf.int32)\n\n        y_centers = randints([num_cuts], 0, image_size - 1)\n        x_centers = randints([num_cuts], 0, image_size - 1)\n        tops = tf.math.maximum(y_centers - cut_h//2, 0)\n        lefts = tf.math.maximum(x_centers - cut_w//2, 0)\n        bottoms = tf.math.minimum(tops + cut_h, height_i - 1)\n        rights = tf.math.minimum(lefts + cut_w, width_i - 1)\n\n        cut_region = make_region_mask(tops, lefts, bottoms, rights)\n        mask_value = tf.constant(0.0, dtype=tf.float32)\n        aug_image = tf.where(cut_region, mask_value, image)\n        return choice(p, aug_image, image)\n    return _do_cutout\n\n\ncut_out = Cutout(num_cuts=1, mask_factor=0.4, p=0.75)","metadata":{"execution":{"iopub.status.busy":"2021-08-03T00:50:39.495829Z","iopub.execute_input":"2021-08-03T00:50:39.496195Z","iopub.status.idle":"2021-08-03T00:50:39.514353Z","shell.execute_reply.started":"2021-08-03T00:50:39.496154Z","shell.execute_reply":"2021-08-03T00:50:39.513245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### MIX UP","metadata":{}},{"cell_type":"code","source":"AUG_BATCH = 128\ndef mixup(image, label, PROBABILITY = 0.3):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with mixup applied\n    DIM = image_size\n    CLASSES = 4\n    \n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        # print(j)\n        # DO MIXUP WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.float32)\n        # CHOOSE RANDOM\n        k = tf.cast( tf.random.uniform([],0,AUG_BATCH),tf.int32)\n        a = tf.random.uniform([],0,1)*P # this is beta dist with alpha=1.0\n        # MAKE MIXUP IMAGE\n        img1 = image[j,]\n        img2 = image[k,]\n        imgs.append((1-a)*img1 + a*img2)\n        # MAKE CUTMIX LABEL\n        if len(label.shape)==1:\n            lab1 = tf.one_hot(label[j],CLASSES)\n            lab2 = tf.one_hot(label[k],CLASSES)\n        else:\n            lab1 = label[j,]\n            lab2 = label[k,]\n            \n        lab1 = tf.cast(lab1,dtype=tf.float32)\n        lab2 = tf.cast(lab2,dtype=tf.float32)\n        labs.append((1-a)*lab1 + a*lab2)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image2,label2","metadata":{"execution":{"iopub.status.busy":"2021-08-03T00:50:39.516042Z","iopub.execute_input":"2021-08-03T00:50:39.516671Z","iopub.status.idle":"2021-08-03T00:50:39.531296Z","shell.execute_reply.started":"2021-08-03T00:50:39.516631Z","shell.execute_reply":"2021-08-03T00:50:39.530017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import albumentations\n\n\ndef auto_select_accelerator():\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        print(\"Running on TPU:\", tpu.master())\n    except ValueError:\n        strategy = tf.distribute.get_strategy()\n    print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n    \n    return strategy\n\n\ndef build_decoder(with_labels=True, target_size=(256, 256), ext='jpg'):\n    def decode(path):\n        file_bytes = tf.io.read_file(path)\n        if ext == 'png':\n            img = tf.image.decode_png(file_bytes, channels=3)\n        elif ext in ['jpg', 'jpeg']:\n            img = tf.image.decode_jpeg(file_bytes, channels=3)\n        else:\n            raise ValueError(\"Image extension not supported\")\n        img = tf.cast(img, tf.float32) / 255.0\n        img = tf.image.resize(img, target_size)\n\n        return img\n    \n    def decode_with_labels(path, label):\n        return decode(path), label\n    \n    return decode_with_labels if with_labels else decode\n\n\ndef build_augmenter(with_labels=True):\n    def augment(img):\n        img = horizonflip(img)\n        img = updown_flip(img)\n        img = random_contrast(img)\n        img = optical_distortion(img)\n        img = hue_saturation_value(img)\n        img = shift_scale_rotate(img)\n        # img = cut_out(img)\n\n        return img\n    \n    def augment_with_labels(img, label):\n        return augment(img), label\n    \n    return augment_with_labels if with_labels else augment\n\n\ndef build_dataset(paths, labels=None, bsize=128, cache=True,\n                  decode_fn=None, augment_fn=None,\n                  augment=True, repeat=True, shuffle=1024, \n                  cache_dir=\"\"):\n    if cache_dir != \"\" and cache is True:\n        os.makedirs(cache_dir, exist_ok=True)\n    \n    if decode_fn is None:\n        decode_fn = build_decoder(labels is not None)\n    \n    if augment_fn is None:\n        augment_fn = build_augmenter(labels is not None)\n    \n    AUTO = tf.data.experimental.AUTOTUNE\n    slices = paths if labels is None else (paths, labels)\n    dset = tf.data.Dataset.from_tensor_slices(slices)\n    dset = dset.map(decode_fn, num_parallel_calls=AUTO)\n    dset = dset.cache(cache_dir) if cache else dset\n    dset = dset.map(augment_fn, num_parallel_calls=AUTO) if augment else dset\n    dset = dset.repeat() if repeat else dset\n    dset = dset.shuffle(shuffle) if shuffle else dset\n    dset = dset.batch(bsize).prefetch(AUTO)\n    # dset = dset.map(mixup)\n    \n    \n    return dset","metadata":{"execution":{"iopub.status.busy":"2021-08-03T00:50:39.532871Z","iopub.execute_input":"2021-08-03T00:50:39.533348Z","iopub.status.idle":"2021-08-03T00:50:40.895506Z","shell.execute_reply.started":"2021-08-03T00:50:39.533306Z","shell.execute_reply":"2021-08-03T00:50:40.894372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"COMPETITION_NAME = \"siimcovid19-512-img-png-600-study-png\"\nstrategy = auto_select_accelerator()\nBATCH_SIZE =  strategy.num_replicas_in_sync * 16\nGCS_DS_PATH = KaggleDatasets().get_gcs_path(COMPETITION_NAME)","metadata":{"execution":{"iopub.status.busy":"2021-08-03T00:50:40.899122Z","iopub.execute_input":"2021-08-03T00:50:40.899483Z","iopub.status.idle":"2021-08-03T00:50:47.327052Z","shell.execute_reply.started":"2021-08-03T00:50:40.899434Z","shell.execute_reply":"2021-08-03T00:50:47.325953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"load_dir = f\"/kaggle/input/{COMPETITION_NAME}/\"\ndf = pd.read_csv('../input/siim-covid19-detection/train_study_level.csv')\nlabel_cols = df.columns[1:5]\n","metadata":{"execution":{"iopub.status.busy":"2021-08-03T00:50:47.328595Z","iopub.execute_input":"2021-08-03T00:50:47.328913Z","iopub.status.idle":"2021-08-03T00:50:47.358751Z","shell.execute_reply.started":"2021-08-03T00:50:47.328883Z","shell.execute_reply":"2021-08-03T00:50:47.357493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gkf  = GroupKFold(n_splits = 5)\ndf['fold'] = -1\nfor fold, (train_idx, val_idx) in enumerate(gkf.split(df, groups = df.id.tolist())):\n    df.loc[val_idx, 'fold'] = fold","metadata":{"execution":{"iopub.status.busy":"2021-08-03T00:50:47.360097Z","iopub.execute_input":"2021-08-03T00:50:47.360427Z","iopub.status.idle":"2021-08-03T00:50:47.414131Z","shell.execute_reply.started":"2021-08-03T00:50:47.360396Z","shell.execute_reply":"2021-08-03T00:50:47.413047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# lr","metadata":{}},{"cell_type":"code","source":"def lrfn(epoch):\n    LR_START = 0.00001\n    LR_MAX = 0.00005 * strategy.num_replicas_in_sync\n    LR_MIN = 0.000005\n    LR_RAMPUP_EPOCHS = 7\n    LR_SUSTAIN_EPOCHS = 0\n    LR_EXP_DECAY = .8\n    \n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n\nlr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=1)","metadata":{"execution":{"iopub.status.busy":"2021-08-03T00:50:47.415632Z","iopub.execute_input":"2021-08-03T00:50:47.415928Z","iopub.status.idle":"2021-08-03T00:50:47.423357Z","shell.execute_reply.started":"2021-08-03T00:50:47.415899Z","shell.execute_reply":"2021-08-03T00:50:47.422397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EFFICIENTNETv2","metadata":{}},{"cell_type":"code","source":"for i in range(5):\n    \n    valid_paths = GCS_DS_PATH + '/study/' + df[df['fold'] == i]['id'] + '.png' #\"/train/\"\n    train_paths = GCS_DS_PATH + '/study/' + df[df['fold'] != i]['id'] + '.png' #\"/train/\" \n    valid_labels = df[df['fold'] == i][label_cols].values\n    train_labels = df[df['fold'] != i][label_cols].values\n\n    IMSIZE = (224, 240, 260, 300, 380, 456, 528, 600, 768)\n    IMS = 7\n\n    decoder = build_decoder(with_labels=True, target_size=(IMSIZE[IMS], IMSIZE[IMS]), ext='png')\n    test_decoder = build_decoder(with_labels=False, target_size=(IMSIZE[IMS], IMSIZE[IMS]),ext='png')\n\n    train_dataset = build_dataset(\n        train_paths, train_labels, bsize=BATCH_SIZE, decode_fn=decoder\n    )\n    # train_dataset = train_dataset.map(mixup)\n    valid_dataset = build_dataset(\n        valid_paths, valid_labels, bsize=BATCH_SIZE, decode_fn=decoder,\n        repeat=False, shuffle=False, augment=False\n    )\n\n    try:\n        n_labels = train_labels.shape[1]\n    except:\n        n_labels = 1\n        \n    strategy = auto_select_accelerator()\n    with strategy.scope():\n        model = tf.keras.Sequential([\n            efn.EfficientNetB7(\n                input_shape=(IMSIZE[IMS], IMSIZE[IMS], 3),\n                weights='imagenet',\n                include_top=False),\n            tf.keras.layers.GlobalAveragePooling2D(),\n            tf.keras.layers.Dense(n_labels, activation='softmax')\n        ])\n        model.compile(\n            optimizer=tf.keras.optimizers.Adam(),\n            loss=tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.005),\n            metrics=[tf.keras.metrics.AUC(multi_label=True)])\n\n        model.summary()\n\n\n    steps_per_epoch = train_paths.shape[0] // BATCH_SIZE\n    checkpoint = tf.keras.callbacks.ModelCheckpoint(\n        f'model{i}.h5', save_best_only=True, monitor='val_loss', mode='min')\n    lr_reducer = tf.keras.callbacks.ReduceLROnPlateau(\n        monitor=\"val_loss\", patience=3, min_lr=1e-6, mode='min')\n    early_stop = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=3) \n\n    history = model.fit(\n        train_dataset, \n        epochs=20,\n        verbose=1,\n        callbacks=[checkpoint, lr_reducer],\n        steps_per_epoch=steps_per_epoch,\n        validation_data=valid_dataset)\n    \n\n    hist_df = pd.DataFrame(history.history)\n    hist_df.to_csv(f'history{i}.csv')","metadata":{"execution":{"iopub.status.busy":"2021-08-03T00:50:47.424997Z","iopub.execute_input":"2021-08-03T00:50:47.425733Z","iopub.status.idle":"2021-08-03T00:56:33.230269Z","shell.execute_reply.started":"2021-08-03T00:50:47.425685Z","shell.execute_reply":"2021-08-03T00:56:33.226699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}