{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q efficientnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport argparse\nimport pandas as pd\nimport numpy as np\nimport argparse\nimport tensorflow as tf\n\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.losses import categorical_crossentropy\nfrom tensorflow.keras.metrics import categorical_accuracy, top_k_categorical_accuracy\nfrom kaggle_datasets import KaggleDatasets\nfrom tensorflow.keras.callbacks import LearningRateScheduler\n\nfrom tensorflow.keras import layers as L\n\n\nimport efficientnet.tfkeras as efn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def normalize(image):\n    image -= tf.constant([0.485 * 255, 0.456 * 255, 0.406 * 255])\n    image /= tf.constant([0.229 * 255, 0.224 * 255, 0.225 * 255])\n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model(input_size, backbone='efficientnet-b0', weights='imagenet', tta=False):\n    print(f'Using backbone {backbone} and weights {weights}')\n    x = L.Input(shape=input_size, name='imgs', dtype='float32')\n    y = normalize(x)\n    \n    if backbone.startswith('efficientnet'):\n        model_fn = getattr(efn, f'EfficientNetB{backbone[-1]}')\n        \n    y = model_fn(input_shape=input_size, weights=weights, include_top=False)(y)\n    y = L.GlobalAveragePooling2D()(y)\n    y = L.Dropout(0.05)(y)\n    # 1292 out 1295 are present at bottom layer\n    y = L.BatchNormalization()(y)\n    y = L.Dense(1292, activation='softmax')(y)\n    model = tf.keras.Model(x, y)\n    \n    if tta:\n        assert False, 'This doesn not make sense'\n        x_flip = tf.reverse(x, [2])  # 'NHWC'\n        y_tta = tf.add(model(x), model(x_flip)) / 2.0\n        tta_model = tf.keras.Model(x, y_tta)\n        return model, tta_model\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Read mixup paper\ndef mixup(img_batch, label_batch, bs):\n    weight = tf.random.uniform([bs])\n    x_weight = tf.reshape(weight, [bs,1,1,1])\n    y_weight = tf.reshape(weight, [bs,1])\n    index = tf.random.shuffle(tf.range(bs, dtype=tf.int32))\n    x1, x2 = img_batch, tf.gather(img_batch, index)\n    img_batch = x1* x_weight + x2*(1. - x_weight)\n    y1, y2 = label_batch, tf.gather(label_batch, index)\n    label_batch = y1* y_weight + y2*(1. - y_weight)\n    return img_batch, label_batch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_strategy():\n  # Detect hardware, return appropriate distribution strategy\n  try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection\n    print('Running on TPU ', tpu.cluster_spec().as_dict()['worker'])\n  except ValueError:\n    tpu = None\n\n  if tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\n  else:\n    strategy = tf.distribute.get_strategy()\n\n  print('REPLICAS: ', strategy.num_replicas_in_sync)\n  return strategy\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def one_hot(image, label):\n  label = tf.one_hot(label, 1292)\n  return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_tfrecords(example, input_size):\n  features = {\n      'img': tf.io.FixedLenFeature([], tf.string),\n      'image_id': tf.io.FixedLenFeature([], tf.int64),\n      'grapheme_root': tf.io.FixedLenFeature([], tf.int64),\n      'vowel_diacritic': tf.io.FixedLenFeature([], tf.int64),\n      'consonant_diacritic': tf.io.FixedLenFeature([], tf.int64),\n      'unique_tuple': tf.io.FixedLenFeature([], tf.int64),\n  }\n  example = tf.io.parse_single_example(example, features)\n  img = tf.image.decode_image(example['img'])\n  img = tf.reshape(img, input_size + (1, ))\n  img = tf.cast(img, tf.float32)\n  # grayscale -> RGB\n  img = tf.repeat(img, 3, -1)\n\n  # image_id = tf.cast(example['image_id'], tf.int32)\n  # grapheme_root = tf.cast(example['grapheme_root'], tf.int32)\n  # vowel_diacritic = tf.cast(example['vowel_diacritic'], tf.int32)\n  # consonant_diacritic = tf.cast(example['consonant_diacritic'], tf.int32)\n  unique_tuple = tf.cast(example['unique_tuple'], tf.int32)\n  return img, unique_tuple","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import the necessary packages\nimport matplotlib.pyplot as plt\nimport numpy as np\nclass LearningRateDecay:\n\tdef plot(self, epochs, title=\"Learning Rate Schedule\"):\n\t\t# compute the set of learning rates for each corresponding\n\t\t# epoch\n\t\tlrs = [self(i) for i in epochs]\n\t\t# the learning rate schedule\n\t\tplt.style.use(\"ggplot\")\n\t\tplt.figure()\n\t\tplt.plot(epochs, lrs)\n\t\tplt.title(title)\n\t\tplt.xlabel(\"Epoch #\")\n\t\tplt.ylabel(\"Learning Rate\")\n\nclass StepDecay(LearningRateDecay):\n\tdef __init__(self, initAlpha=0.01, factor=0.25, dropEvery=10):\n\t\t# store the base initial learning rate, drop factor, and\n\t\t# epochs to drop every\n\t\tself.initAlpha = initAlpha\n\t\tself.factor = factor\n\t\tself.dropEvery = dropEvery\n\tdef __call__(self, epoch):\n\t\t# compute the learning rate for the current epoch\n\t\texp = np.floor((1 + epoch) / self.dropEvery)\n\t\talpha = self.initAlpha * (self.factor ** exp)\n\t\t# return the learning rate\n\t\treturn float(alpha)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def main():\n    parser = argparse.ArgumentParser()\n    parser.add_argument('--model_id', type=int, default=0)\n    parser.add_argument('--seed', type=int, default=123)\n    parser.add_argument('--lr', type=float, default=1e-4)\n    parser.add_argument('--input_size', type=str, default='160,256')\n    parser.add_argument('--batch_size', type=int, default=256)\n    parser.add_argument('--epochs', type=int, default=35)\n    parser.add_argument('--backbone', type=str, default='efficientnet-b4')\n    parser.add_argument('--weights', type=str, default='imagenet')\n    args, _ = parser.parse_known_args()\n\n    args.input_size = tuple(int(x) for x in args.input_size.split(','))\n    np.random.seed(args.seed)\n    tf.random.set_seed(args.seed)\n\n  # build the model\n    strategy = get_strategy()\n    with strategy.scope():\n        model = get_model(input_size=args.input_size + (3, ), backbone=args.backbone,\n        weights=args.weights)\n        schedule = StepDecay(initAlpha=1e-4, factor=0.25, dropEvery=15)\n        model.compile(optimizer='adam',\n                loss='categorical_crossentropy',\n                metrics=[categorical_accuracy, top_k_categorical_accuracy])\n\n    callbacks = [LearningRateScheduler(schedule)]\n    AUTO = tf.data.experimental.AUTOTUNE #\n    ignore_order = tf.data.Options() #\n    ignore_order.experimental_deterministic = False #\n    \n    ds_path = KaggleDatasets().get_gcs_path('bengali-tfrecords-v010')\n    train_fns = tf.io.gfile.glob(os.path.join(ds_path, 'records/train*.tfrec'))\n    train_ds = tf.data.TFRecordDataset(train_fns, num_parallel_reads=AUTO)\n#     train_ds = train_ds.with_optional(ignore_order)\n    train_ds = train_ds.map(lambda e: read_tfrecords(e, args.input_size), num_parallel_calls=AUTO)\n    train_ds = train_ds.repeat().batch(args.batch_size)\n    train_ds = train_ds.map(one_hot)\n    train_ds = train_ds.map(lambda a, b: mixup(a, b, args.batch_size), num_parallel_calls=AUTO)\n    \n    val_fns = tf.io.gfile.glob(os.path.join(ds_path, 'records/val*.tfrec'))\n    val_ds = tf.data.TFRecordDataset(val_fns, num_parallel_reads=AUTO)\n#     val_ds = val_ds.with_optional(ignore_order)\n    val_ds = val_ds.map(lambda e: read_tfrecords(e, args.input_size), num_parallel_calls=AUTO)\n    val_ds = val_ds.batch(args.batch_size)\n    val_ds = val_ds.map(one_hot)\n\n  # train\n    num_train_samples = sum(int(fn.split('_')[2]) for fn in train_fns)\n     # num_val_samples = sum(int(fn.split('_')[2]) for fn in val_fns)\n    steps_per_epoch = num_train_samples // args.batch_size\n    print(f'Training on {num_train_samples} samples. Each epochs requires {steps_per_epoch} steps')\n    h = model.fit(train_ds, steps_per_epoch=steps_per_epoch, epochs=args.epochs, verbose=1,\n          validation_data=val_ds,callbacks=callbacks)\n    print(h)\n    weight_fn = 'model-%04d.h5' % args.model_id\n    model.save_weights(weight_fn)\n    model.save('Enet-final.h5')\n    print(f'Saved weights to: {weight_fn}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nmain()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}