{"cells":[{"metadata":{},"cell_type":"markdown","source":"source\n\nhttps://www.kaggle.com/fuyixing/starter-keras-tuner-efficientnet-tpu"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q tensorflow==2.3.0 # Use 2.3.0 for built-in EfficientNet\n!pip install -q git+https://github.com/keras-team/keras-tuner@master # Use github head for newly added TPU support\n!pip install -q cloud-tpu-client # Needed for sync TPU version\n\n!pip install -U tensorflow-gcs-config==2.3.0 # Needed for using private dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import random, re, math\nimport numpy as np, pandas as pd\nimport matplotlib.pyplot as plt\nimport tensorflow as tf, tensorflow.keras.backend as K\nfrom kaggle_datasets import KaggleDatasets\n\nprint('Tensorflow version ' + tf.__version__)\nimport kerastuner as kt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    # Sync TPU version\n    from cloud_tpu_client import Client\n    c = Client()\n    c.configure_tpu_version(tf.__version__, restart_type='ifNeeded')\n    \n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n    \n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Configuration\nIMAGE_SIZE = [256, 256]\nEPOCHS_SEARCH = 5\nEPOCHS_FINAL = 5\n# SEED = 123\nBATCH_SIZE = 32 * strategy.num_replicas_in_sync","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.data.experimental import AUTOTUNE\nbase_path = KaggleDatasets().get_gcs_path('gld-v2-256')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport functools\n\n\ndef create_dataset(file_pattern, allowed_labels, augmentation: bool = False, num_classes=None):\n    # Select only dataset within a list of allowed labels\n    if not num_classes:\n        raise ValueError('num_classses must be set.')\n\n    ignore_order = tf.data.Options()\n    ignore_order.experimental_deterministic = False\n    filenames = tf.io.gfile.glob(file_pattern)\n    filenames = filenames\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTOTUNE).shuffle(1000)\n\n    # Create a description of the features.\n    feature_description = {\n        'image/height': tf.io.FixedLenFeature([], tf.int64, default_value=0),\n        'image/width': tf.io.FixedLenFeature([], tf.int64, default_value=0),\n        'image/channels': tf.io.FixedLenFeature([], tf.int64, default_value=0),\n        'image/format': tf.io.FixedLenFeature([], tf.string, default_value=''),\n        'image/id': tf.io.FixedLenFeature([], tf.string, default_value=''),\n        'image/filename': tf.io.FixedLenFeature([], tf.string, default_value=''),\n        'image/encoded': tf.io.FixedLenFeature([], tf.string, default_value=''),\n        'image/class/label': tf.io.FixedLenFeature([], tf.int64, default_value=0),\n    }\n\n    parse_func = functools.partial(\n        _parse_example,\n        name_to_features=feature_description,\n        augmentation=augmentation\n    )\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(parse_func, num_parallel_calls=AUTOTUNE)\n\n    def label_predicate(x, y):\n        return tf.greater(tf.reduce_sum(tf.cast(tf.equal(allowed_labels, y), tf.float32)), 0.)\n\n    def relabel(x, y):\n        y = tf.reduce_min(tf.where(tf.equal(allowed_labels, y)))\n        return x, tf.one_hot(y, num_classes)\n\n    dataset = dataset.filter(label_predicate)\n    dataset = dataset.map(relabel, num_parallel_calls=AUTOTUNE)\n    dataset = dataset.repeat()\n\n    dataset = dataset.batch(BATCH_SIZE, drop_remainder=True)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset\n\ndef _parse_example(example, name_to_features, augmentation):\n    parsed_example = tf.io.parse_single_example(example, name_to_features)\n\n    image = parsed_example['image/encoded']\n    image = tf.io.decode_jpeg(image)\n    image = tf.cast(image, tf.float32)\n    image = tf.image.resize(image, IMAGE_SIZE)\n    image.set_shape([*IMAGE_SIZE, 3])\n\n    label = tf.cast(parsed_example['image/class/label'], tf.int64)\n    return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# original labelling\ntraining_csv_path = os.path.join(base_path, \"train.csv\")\ntrain_csv = pd.read_csv(str(training_csv_path))\n\n# original labelling\nclean_training_csv_path = os.path.join(base_path, \"train_clean.csv\")\nclean_train_csv = pd.read_csv(str(clean_training_csv_path))\n###\norig_unique_landmark_ids = clean_train_csv[\"landmark_id\"].tolist()\nprint('max label:', max(orig_unique_landmark_ids))\n###","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"landmark_ids_occurance = [len(x.split(\" \")) for x in clean_train_csv[\"images\"]]\n# The labelling used in tfrecord is compressed, corresponding to 0 based id of clean_csv\ncompressed_landmark_ids_to_occurance = list(enumerate(landmark_ids_occurance))\n\n#unique_landmark_ids = [x[0] for x in unique_landmark_ids_to_occurance]\n\nallowed_labels = [x[0] for x in compressed_landmark_ids_to_occurance if x[1] >= 25]\nallowed_labels = tf.convert_to_tensor(allowed_labels, dtype=tf.int64)\n\nnum_samples = sum([x for x in landmark_ids_occurance if x >= 25])\nNUM_CLASSES = len([x for x in landmark_ids_occurance if x >= 25])\n\n\n# unique_landmark_ids_occurance = tf.convert_to_tensor(unique_landmark_ids_occurance)\nprint(num_samples)\nsteps_per_epoch = int(num_samples / BATCH_SIZE)\n_num_samples = steps_per_epoch * BATCH_SIZE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_tf_records = os.path.join(base_path, 'train*128')\n# val_tf_records = os.path.join(base_path, 'val*128')\nall_tf_records = os.path.join(base_path, '*128')\n\n# ds_train = create_dataset(train_tf_records,\n#                           allowed_labels,\n#                           num_classes = NUM_CLASSES)\n\n# ds_val = create_dataset(val_tf_records,\n#                         allowed_labels,\n#                         num_classes = NUM_CLASSES)\n\nds_all = create_dataset(all_tf_records,\n                        allowed_labels,\n                        num_classes = NUM_CLASSES)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for img, lbl in ds_train.shuffle(10).take(1):\n#     plt.imshow(tf.cast(img[0], tf.int32))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from kerastuner.applications.efficientnet import HyperEfficientNet\n# class MyHyperEfficientNet(HyperEfficientNet):\n#     def _compile(self, model, hp):\n        \n#         for l in model.layers:\n#             # For efficientnet implementation we use, layers in the\n#             # Feature extraction part of model all have 'block' in name.\n#             if 'block' in l.name:\n#                 l.trainable = False\n                \n#         super(MyHyperEfficientNet, self)._compile(model, hp)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Define HyperModel using built-in application\n# from kerastuner.applications.efficientnet import HyperEfficientNet\n# hm = HyperEfficientNet(input_shape=[*IMAGE_SIZE, 3] , classes=NUM_CLASSES)\n\n# # Optional: Restrict default hyperparameters.\n# # To take effect, pass this `hp` instance when constructing tuner as `hyperparameters=hp`\n# from kerastuner.engine.hyperparameters import HyperParameters\n# hp = HyperParameters()\n# hp.Choice('version', ['B0', 'B1', 'B2', 'B3']) #restrict choice of EfficientNet version from B0-B7 to B0-B4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define Oracle\n# oracle = kt.tuners.randomsearch.RandomSearchOracle(\n#     objective='val_accuracy',\n#     max_trials=5,\n#     hyperparameters=hp,\n# )\n\n# # Initiate Tuner\n# tuner = kt.engine.tuner.Tuner(\n#     hypermodel=hm,\n#     oracle=oracle,\n#     distribution_strategy=strategy, # This strategy's scope is used for building each model during the search.\n#     directory='landmark',\n#     project_name='randomsearch_efficientnet',\n# )\n# tuner.search_space_summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_split = 0.2\nnum_val_samples = int(num_samples * val_split)\nnum_train_samples = int(num_samples * (1 - val_split))\n\nnum_train_batches = num_train_samples // BATCH_SIZE\nnum_val_batches = num_val_samples // BATCH_SIZE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# tuner.search(ds_train,\n#              epochs=EPOCHS_SEARCH,\n#              validation_data=ds_val,\n#              steps_per_epoch=num_train_batches,\n#              validation_steps=num_val_batches,\n#              verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# tuner.results_summary()\n# model = tuner.get_best_models()[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"LR_START = 0.00001\nLR_MAX = 0.0001 * strategy.num_replicas_in_sync\nLR_MIN = 0.00001\nLR_RAMPUP_EPOCHS = 15\nLR_SUSTAIN_EPOCHS = 3\nLR_EXP_DECAY = .8\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n    \nlr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.applications import EfficientNetB7\n\nwith strategy.scope():\n    input = tf.keras.layers.Input(shape = (*IMAGE_SIZE,3))\n    \n    # Create and Compile Model and show Summary\n    effnet_model = EfficientNetB7(weights = \"imagenet\", include_top = False, input_tensor = input, pooling = 'avg', classes = None)\n    \n    X = tf.keras.layers.Dropout(0.25)(effnet_model.output)\n    X = tf.keras.layers.Dense(1024, activation = 'relu')(X)\n    X = tf.keras.layers.BatchNormalization()(X)\n    X = tf.keras.layers.Dropout(0.25)(X)\n    preds = tf.keras.layers.Dense(NUM_CLASSES, activation = 'softmax')(X)\n    \n    # Create Final Model\n    model = tf.keras.Model(inputs = effnet_model.input, outputs = preds)\n\n    # UnFreeze all layers\n    for layer in model.layers:\n        layer.trainable = True\n        \n    opt = tf.keras.optimizers.Adam(lr=1e-4, decay=1e-4 / EPOCHS_FINAL)\n\n    model.compile(\n        optimizer=opt,\n        loss = 'categorical_crossentropy',\n        metrics=['categorical_accuracy']\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from tensorflow.keras.applications import EfficientNetB2\n\n\n# with strategy.scope():\n#     model = tf.keras.Sequential([\n#             EfficientNetB2(weights=\"imagenet\", include_top=False, input_shape=(*IMAGE_SIZE, 3)),\n\n#             tf.keras.layers.GlobalAveragePooling2D(),\n#             tf.keras.layers.Flatten(name=\"flatten\"),\n#             tf.keras.layers.Dense(256, activation=\"relu\"),\n#             tf.keras.layers.Dropout(0.5),\n#             tf.keras.layers.Dense(NUM_CLASSES, activation='softmax')\n#     ])\n    \n#     opt = tf.keras.optimizers.Adam(lr=1e-4, decay=1e-4 / EPOCHS_FINAL)\n\n#     model.compile(\n#         optimizer=opt,\n#         loss = 'categorical_crossentropy',\n#         metrics=['categorical_accuracy']\n#     )\n# #     model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# K.clear_session()\n# model = tf.keras.models.load_model('../input/landmark-tpu/model.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train the best model with all data\nmodel.fit(ds_all,\n          epochs=EPOCHS_FINAL,\n#           batch_size=BATCH_SIZE,\n          steps_per_epoch=num_train_batches + num_val_batches,\n#           callbacks=[tf.keras.callbacks.ReduceLROnPlateau(),lr_callback],\n          callbacks=[lr_callback],\n          verbose=2\n               )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.chdir(\"/kaggle/working/\")\nmodel.save(\"model.h5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}