{"cells":[{"metadata":{},"cell_type":"markdown","source":"Original author: Rakka Alhazimi"},{"metadata":{},"cell_type":"markdown","source":"To do:\n1. Read tfrec file\n2. Image Augmentation\n3. Build Neural Network\n4. Train Model\n5. Create Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -U efficientnet","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"94e77131-fd75-4e82-aad9-0b9ddbd50e7b","_cell_guid":"9b1f1a35-bb65-477a-9df3-193829b49c88","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom keras import backend as K\nfrom efficientnet import keras as efn\n\nfrom kaggle_datasets import KaggleDatasets\n\nimport re, math","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.__version__","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Detect TPU"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is set. \n    # On Kaggle this is always the case.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  \n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    print(\"tpu\")\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # default distribution strategy in Tensorflow. \n    # Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nREPLICAS = strategy.num_replicas_in_sync\n\nprint(\"REPLICAS: \", REPLICAS)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Initial Parameter"},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 16 * REPLICAS\nEPOCHS = 20\n\nIMAGE_SIZE = (512, 512)\n\nTRAIN_IMG_NUM = 12753\nVAL_IMG_NUM = 3712\nTEST_IMG_NUM = 7382\n\nSTEPS_PER_EPOCHS = TRAIN_IMG_NUM // BATCH_SIZE\nSTEPS_PER_EPOCHS_FULL = (TRAIN_IMG_NUM + VAL_IMG_NUM) // BATCH_SIZE\n\nAUTO = tf.data.experimental.AUTOTUNE\n\nAugParams = {\n    'd1' : 100,\n    'd2': 160,\n    'rotate' : 45,\n    'ratio' : 0.5\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Read TFRecord File"},{"metadata":{},"cell_type":"markdown","source":"You can learn more about TFRecord from [here](https://www.tensorflow.org/tutorials/load_data/tfrecord#tfrecords_format_details)"},{"metadata":{},"cell_type":"markdown","source":"## Get Filenames"},{"metadata":{"trusted":true},"cell_type":"code","source":"# With TPU enable, we can only read filenames through Google Cloud Storage\nGCS_DATA_PATH = KaggleDatasets().get_gcs_path()\n\n\ndef get_filenames(path):\n    return tf.io.gfile.glob(GCS_DATA_PATH + path)\n\ntrain_filenames = get_filenames(\"/tfrecords-jpeg-{0}x{1}/train/*.tfrec\".format(*IMAGE_SIZE))\nval_filenames = get_filenames(\"/tfrecords-jpeg-{0}x{1}/val/*.tfrec\".format(*IMAGE_SIZE))\ntest_filenames = get_filenames(\"/tfrecords-jpeg-{0}x{1}/test/*.tfrec\".format(*IMAGE_SIZE))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load Dataset\nTo open tfrec file, we need to pass the filenames into **tf.data.TFRecordDataset** class"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create dataset from multiple filenames.\ndef load_dataset(filenames, ordered=True):\n    \n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=-1)\n    ignore_order = tf.data.Options()\n    \n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, more speed\n    \n    return dataset.with_options(ignore_order)\n\n\ntrain_records = load_dataset(train_filenames, False)\nval_records = load_dataset(val_filenames, False)\ntest_records = load_dataset(test_filenames, True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Identify tfrecord features\nTake one sample from training dataset and parse with tf.train.Example()"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"# for raw_record in train_records.take(1):\n#     example = tf.train.Example()\n#     example.ParseFromString(raw_record.numpy())\n#     print(example)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Read raw bytes string\nContents inside tfrec file are bytes format, we need to parse it back into specified features."},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(raw):\n    \"\"\"Decode parsed bytes string into jpeg format\"\"\"\n    \n    decoded = tf.io.decode_jpeg(raw) \n    image = tf.cast(decoded, tf.float32) / 255. # normalize to 0..1 value\n    image = tf.reshape(image, [*IMAGE_SIZE, 3]) # Size require for TPU\n    \n    return image ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from functools import partial\n\n# Write all known features here, in format \"feature\": \"type\"\nconfig = tf.io.FixedLenFeature\n\nfeature_train = {\"class\": config([], tf.int64),\n                 \"id\"   : config([], tf.string),\n                 \"image\": config([], tf.string),}\n\nfeature_test = {\"id\"   : config([], tf.string),\n                \"image\": config([], tf.string),}\n\n\ndef read_tfrecord(example_single, features):\n    \"\"\"Parse raw bytes string from tfrec\"\"\"\n    \n    parsed = tf.io.parse_single_example(example_single, features)\n    \n    idm = parsed.get(\"id\")\n    label = parsed.get(\"class\")\n    image = decode_image(parsed.get(\"image\"))\n    \n    if not features.get(\"class\"):                # Test data didn't have class/label\n        return image, idm                        # Return image and id\n    \n    return image, label                          # Return image and label\n\n\n# Use functools.partial to set up default args for specific data\n\n# Default arg for train, val data\nparse_train = partial(read_tfrecord, features=feature_train) \n\n# Default arg for test data\nparse_test = partial(read_tfrecord, features=feature_test)  ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* tf.io.FixedLenFeature : a class to configure the incoming feature\n* first arg             : shape [] means single element\n* second arg            : tf.int64 means dtype\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Clean dataset, ready to be trained or modified first\ntrain_dataset = train_records.map(parse_train, num_parallel_calls=AUTO)\nval_dataset = val_records.map(parse_train, num_parallel_calls=AUTO)\ntest_dataset = test_records.map(parse_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Plot Random Images"},{"metadata":{},"cell_type":"markdown","source":"## Create function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_random_image(rows, cols, dataset, shuffle=5000):    \n    index = 1\n    plt.figure(figsize=(3 * cols, rows * 3))\n    for image, label in dataset.shuffle(shuffle).take(rows * cols):\n        plt.subplot(rows, cols, index)\n        plt.imshow(image.numpy())\n        plt.title(label.numpy())\n        plt.axis(\"off\")\n        index += 1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot_random_image(2, 5, train_dataset)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Validation Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot_random_image(2, 5, val_dataset)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Test Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot_random_image(2, 5, test_dataset)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Image Augmentation"},{"metadata":{},"cell_type":"markdown","source":"## Rotation, Shift, Zoom, Shear"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear = math.pi * shear / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape( tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3] )\n        \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape( tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3] )    \n    \n    # ZOOM MATRIX\n    zoom_matrix = tf.reshape( tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3] )\n    \n    # SHIFT MATRIX\n    shift_matrix = tf.reshape( tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3] )\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rot_shift_zoom_shear(image, DIM = IMAGE_SIZE[0]):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n\n    XDIM = DIM % 2\n    \n    rot = 15. * tf.random.normal([1],dtype='float32')\n    shr = 5. * tf.random.normal([1],dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n    w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n    h_shift = 16. * tf.random.normal([1],dtype='float32') \n    w_shift = 16. * tf.random.normal([1],dtype='float32') \n  \n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image,tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Grid Mask"},{"metadata":{"trusted":true},"cell_type":"code","source":"def transform(image, inv_mat, image_shape):\n\n    h, w, c = image_shape\n    cx, cy = w//2, h//2\n\n    new_xs = tf.repeat( tf.range(-cx, cx, 1), h)\n    new_ys = tf.tile( tf.range(-cy, cy, 1), [w])\n    new_zs = tf.ones([h*w], dtype=tf.int32)\n\n    old_coords = tf.matmul(inv_mat, tf.cast(tf.stack([new_xs, new_ys, new_zs]), tf.float32))\n    old_coords_x, old_coords_y = tf.round(old_coords[0, :] + w//2), tf.round(old_coords[1, :] + h//2)\n\n    clip_mask_x = tf.logical_or(old_coords_x<0, old_coords_x>w-1)\n    clip_mask_y = tf.logical_or(old_coords_y<0, old_coords_y>h-1)\n    clip_mask = tf.logical_or(clip_mask_x, clip_mask_y)\n\n    old_coords_x = tf.boolean_mask(old_coords_x, tf.logical_not(clip_mask))\n    old_coords_y = tf.boolean_mask(old_coords_y, tf.logical_not(clip_mask))\n    new_coords_x = tf.boolean_mask(new_xs+cx, tf.logical_not(clip_mask))\n    new_coords_y = tf.boolean_mask(new_ys+cy, tf.logical_not(clip_mask))\n\n    old_coords = tf.cast(tf.stack([old_coords_y, old_coords_x]), tf.int32)\n    new_coords = tf.cast(tf.stack([new_coords_y, new_coords_x]), tf.int64)\n    rotated_image_values = tf.gather_nd(image, tf.transpose(old_coords))\n    rotated_image_channel = list()\n    for i in range(c):\n        vals = rotated_image_values[:,i]\n        sparse_channel = tf.SparseTensor(tf.transpose(new_coords), vals, [h, w])\n        rotated_image_channel.append(tf.sparse.to_dense(sparse_channel, default_value=0, validate_indices=False))\n\n    return tf.transpose(tf.stack(rotated_image_channel), [1,2,0])\n\ndef random_rotate(image, angle, image_shape):\n\n    def get_rotation_mat_inv(angle):\n          #transform to radian\n        angle = math.pi * angle / 180\n\n        cos_val = tf.math.cos(angle)\n        sin_val = tf.math.sin(angle)\n        one = tf.constant([1], tf.float32)\n        zero = tf.constant([0], tf.float32)\n\n        rot_mat_inv = tf.concat([cos_val, sin_val, zero,\n                                     -sin_val, cos_val, zero,\n                                     zero, zero, one], axis=0)\n        rot_mat_inv = tf.reshape(rot_mat_inv, [3,3])\n\n        return rot_mat_inv\n    angle = float(angle) * tf.random.normal([1],dtype='float32')\n    rot_mat_inv = get_rotation_mat_inv(angle)\n    return transform(image, rot_mat_inv, image_shape)\n\n\ndef GridMask(image_height, image_width, d1, d2, rotate_angle=1, ratio=0.5):\n\n    h, w = image_height, image_width\n    hh = int(np.ceil(np.sqrt(h*h+w*w)))\n    hh = hh+1 if hh%2==1 else hh\n    d = tf.random.uniform(shape=[], minval=d1, maxval=d2, dtype=tf.int32)\n    l = tf.cast(tf.cast(d,tf.float32)*ratio+0.5, tf.int32)\n\n    st_h = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)\n    st_w = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)\n\n    y_ranges = tf.range(-1 * d + st_h, -1 * d + st_h + l)\n    x_ranges = tf.range(-1 * d + st_w, -1 * d + st_w + l)\n\n    for i in range(0, hh//d+1):\n        s1 = i * d + st_h\n        s2 = i * d + st_w\n        y_ranges = tf.concat([y_ranges, tf.range(s1,s1+l)], axis=0)\n        x_ranges = tf.concat([x_ranges, tf.range(s2,s2+l)], axis=0)\n\n    x_clip_mask = tf.logical_or(x_ranges <0 , x_ranges > hh-1)\n    y_clip_mask = tf.logical_or(y_ranges <0 , y_ranges > hh-1)\n    clip_mask = tf.logical_or(x_clip_mask, y_clip_mask)\n\n    x_ranges = tf.boolean_mask(x_ranges, tf.logical_not(clip_mask))\n    y_ranges = tf.boolean_mask(y_ranges, tf.logical_not(clip_mask))\n\n    hh_ranges = tf.tile(tf.range(0,hh), [tf.cast(tf.reduce_sum(tf.ones_like(x_ranges)), tf.int32)])\n    x_ranges = tf.repeat(x_ranges, hh)\n    y_ranges = tf.repeat(y_ranges, hh)\n\n    y_hh_indices = tf.transpose(tf.stack([y_ranges, hh_ranges]))\n    x_hh_indices = tf.transpose(tf.stack([hh_ranges, x_ranges]))\n\n    y_mask_sparse = tf.SparseTensor(tf.cast(y_hh_indices, tf.int64),  tf.zeros_like(y_ranges), [hh, hh])\n    y_mask = tf.sparse.to_dense(y_mask_sparse, 1, False)\n\n    x_mask_sparse = tf.SparseTensor(tf.cast(x_hh_indices, tf.int64), tf.zeros_like(x_ranges), [hh, hh])\n    x_mask = tf.sparse.to_dense(x_mask_sparse, 1, False)\n\n    mask = tf.expand_dims( tf.clip_by_value(x_mask + y_mask, 0, 1), axis=-1)\n\n    mask = random_rotate(mask, rotate_angle, [hh, hh, 1])\n    mask = tf.image.crop_to_bounding_box(mask, (hh-h)//2, (hh-w)//2, image_height, image_width)\n\n    return mask\n\ndef apply_grid_mask(image, image_shape):\n    mask = GridMask(image_shape[0],\n                    image_shape[1],\n                    AugParams['d1'],\n                    AugParams['d2'],\n                    AugParams['rotate'],\n                    AugParams['ratio'])\n    \n    if image_shape[-1] == 3:\n        mask = tf.concat([mask, mask, mask], axis=-1)\n\n    return image * tf.cast(mask, tf.float32)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Random Blackout"},{"metadata":{"trusted":true},"cell_type":"code","source":"def random_blockout(img, sl=0.1, sh=0.2, rl=0.4):\n\n    h, w, c = IMAGE_SIZE[0], IMAGE_SIZE[1], 3\n    origin_area = tf.cast(h*w, tf.float32)\n\n    e_size_l = tf.cast(tf.round(tf.sqrt(origin_area * sl * rl)), tf.int32)\n    e_size_h = tf.cast(tf.round(tf.sqrt(origin_area * sh / rl)), tf.int32)\n\n    e_height_h = tf.minimum(e_size_h, h)\n    e_width_h = tf.minimum(e_size_h, w)\n\n    erase_height = tf.random.uniform(shape=[], minval=e_size_l, maxval=e_height_h, dtype=tf.int32)\n    erase_width = tf.random.uniform(shape=[], minval=e_size_l, maxval=e_width_h, dtype=tf.int32)\n\n    erase_area = tf.zeros(shape=[erase_height, erase_width, c])\n    erase_area = tf.cast(erase_area, tf.uint8)\n\n    pad_h = h - erase_height\n    pad_top = tf.random.uniform(shape=[], minval=0, maxval=pad_h, dtype=tf.int32)\n    pad_bottom = pad_h - pad_top\n\n    pad_w = w - erase_width\n    pad_left = tf.random.uniform(shape=[], minval=0, maxval=pad_w, dtype=tf.int32)\n    pad_right = pad_w - pad_left\n\n    erase_mask = tf.pad([erase_area], [[0,0],[pad_top, pad_bottom], [pad_left, pad_right], [0,0]], constant_values=1)\n    erase_mask = tf.squeeze(erase_mask, axis=0)\n    erased_img = tf.multiply(tf.cast(img,tf.float32), tf.cast(erase_mask, tf.float32))\n\n    return tf.cast(erased_img, img.dtype)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def augmentation(image, label):\n    \n    prob = tf.random.uniform(shape=[], minval=0.0, maxval=1.0)\n    \n    if prob <= 0.25: image = apply_grid_mask(image, (*IMAGE_SIZE,3))\n    elif prob <= 0.50: image = rot_shift_zoom_shear(image)\n    elif prob <= 0.75: image = random_blockout(image)\n    else: pass\n    \n    return tf.cast(image, tf.float32), label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Show augmented images\nrows, cols = 4, 6\nplt.figure(figsize=(2.9 * cols, rows * 2.9))\n\nfor row, element in zip(range(rows), train_dataset):\n    one_element = tf.data.Dataset.from_tensors(element).repeat()\n    for col, (image, _) in zip(range(cols), one_element.map(augmentation, ).as_numpy_iterator()):\n        plt.subplot(rows, cols, row * cols + col + 1)\n        plt.axis('off')\n        plt.imshow(image)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Learning Rate Scheduler"},{"metadata":{"trusted":true},"cell_type":"code","source":"lr_start = 5e-4\nlr_min = 5e-6\nepoch_decay = 4\n\ndef scheduler(epoch, lr):\n    result = max(lr_min, lr * tf.math.exp(-0.2))\n    return tf.constant(result)\n\n# Plot learning rate scheduler\nlr_show = []\nprevious = lr_start\nfor i in range(EPOCHS):\n    current = scheduler(i, previous)\n    lr_show.append(current)\n    previous = current\n\nlr_show = list(map(lambda x: x.numpy(), lr_show))\n\nplt.plot(range(EPOCHS), lr_show);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Image Classification with Pretrained CNN"},{"metadata":{},"cell_type":"markdown","source":"## Train model with only training data"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_input = (\n    train_dataset\n        .repeat()\n        .map(augmentation, num_parallel_calls=AUTO)\n        .shuffle(2048)\n        .batch(BATCH_SIZE)\n        .prefetch(AUTO)\n)\n\nval_input = (\n    val_dataset\n        .batch(BATCH_SIZE)\n        .cache()\n)\n\ntest_input = (\n    test_dataset.batch(BATCH_SIZE)\n    .prefetch(AUTO)\n                 \n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Params for Pretrained CNN\nparams = {\n    \"include_top\":False, \n    \"input_shape\":[*IMAGE_SIZE, 3], \n    \"classes\":104,\n    \"pooling\":\"avg\"}\n\ndef create_dnet():\n    model = keras.applications.DenseNet201(**params)\n    compiler = re.compile(r\"conv5_block\")\n\n    for layer in model.layers:\n        if not compiler.search(layer.name):\n            layer.trainable = False # transfer learning\n\n    return model\n\ndef create_efn():\n    model = efn.EfficientNetB6(**params)\n    compiler = re.compile(r\"block7\")\n\n    for layer in model.layers:\n        if not compiler.search(layer.name):\n            layer.trainable = False # transfer learning\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Merge two pretrained CNN"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Build neural network using pretrained CNN\nloss = \"sparse_categorical_crossentropy\"\nmetric = \"sparse_categorical_accuracy\"\n\nwith strategy.scope():\n\n    efficient = create_efn()\n    dense_net = create_dnet()\n\n    input_image = keras.Input(shape=[*IMAGE_SIZE, 3])\n    efficient = efficient(input_image)\n    dense_net = dense_net(input_image)\n\n    dropout_1 = keras.layers.Dropout(0.3)(efficient)\n    dropout_2 = keras.layers.Dropout(0.3)(dense_net)\n    \n    concat = keras.layers.concatenate([dropout_1, dropout_2])\n    \n    dense_2 = keras.layers.Dense(104, activation=\"softmax\")(concat)\n\n    merged_model = keras.Model(input_image, dense_2)\n\n    merged_model.compile(optimizer=keras.optimizers.Adam(lr_start), \n                         loss=loss,\n                         metrics=[metric])\n    \n    full_model = merged_model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"keras.utils.plot_model(merged_model, \"merged.png\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Setup Callbacks\nearly_stop = keras.callbacks.EarlyStopping(monitor=metric, patience=5)\n\ncheck_point = keras.callbacks.ModelCheckpoint(\n                                    filepath=\"best_model.h5\",\n                                    monitor=\"val_sparse_categorical_accuracy\",\n                                    save_best_only=True)\n\nlearning_schedule = keras.callbacks.LearningRateScheduler(scheduler)\n\n\ncallbacks_list = [early_stop, check_point, learning_schedule]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Fit with training data, validate with validation data\n\n# history = merged_model.fit(train_input,\n#                             epochs=EPOCHS,\n#                             steps_per_epoch=STEPS_PER_EPOCHS,\n#                             callbacks=callbacks_list,\n#                             validation_data=val_input,)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_acc = history.history[\"sparse_categorical_accuracy\"] \n# val_acc = history.history[\"val_sparse_categorical_accuracy\"]\n# epochs = list(range(EPOCHS))\n\n# plt.figure()\n# plt.plot(epochs, train_acc, label=\"training acc\")\n# plt.plot(epochs, val_acc, label=\"validaiton acc\")\n# plt.legend();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Utilize Both Train and Validation Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":" full_input = (\n    train_dataset.concatenate(val_dataset)\n        .repeat()\n        .map(augmentation, num_parallel_calls=AUTO)\n        .shuffle(2048)\n        .batch(BATCH_SIZE)\n        .prefetch(AUTO)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Change model check point to \"sparse_accuracy\"\ncheck_point.monitor = metric\n\nhistory = full_model.fit(full_input,\n                         epochs=EPOCHS,\n                         steps_per_epoch=STEPS_PER_EPOCHS_FULL,\n                         callbacks=callbacks_list,\n        )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_model = keras.models.load_model(\"./best_model.h5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_weights = best_model.get_weights()\n\n# use merged or full model, the weight will be same anyway\nmerged_model.set_weights(best_weights)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"merged_model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Test Time Augmentation (TTA)"},{"metadata":{"trusted":true},"cell_type":"code","source":"def aug_gridmask(image):\n    image = apply_grid_mask(image, (*IMAGE_SIZE, 3))\n    \n    return tf.cast(image, tf.float32)\n\ndef aug_rot(image):\n    image = rot_shift_zoom_shear(image)\n    \n    return tf.cast(image, tf.float32)\n\ndef aug_blockout(image):\n    image = random_blockout(image)\n    \n    return tf.cast(image, tf.float32)\n\ndef aug_none(image):\n    return image\n\naug_list = [aug_gridmask, aug_rot, aug_blockout, aug_none]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict_tta(model, test_image, aug_list=aug_list):\n    probs = []\n    \n    for index, aug in enumerate(aug_list):\n        copy_image = (\n            test_image.unbatch()\n            .map(aug)\n            .batch(BATCH_SIZE))\n        \n        probs.append(model.predict(copy_image))\n    \n    return np.mean(probs, axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_image = test_input.map(lambda image, _id: image, num_parallel_calls=AUTO)\ntest_id = test_input.map(lambda image, _id: _id).unbatch().batch(TEST_IMG_NUM)\nimage_id = next(iter(test_id)).numpy().astype(\"U\")\n\n# probabilities = merged_model.predict(test_image)\nprobabilities = predict_tta(merged_model, test_image)\npredictions = np.argmax(probabilities, axis=-1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame({\"id\": image_id, \"label\": predictions})\nsubmission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}