{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook uses parts of code from Martin Goerner's notebook [\nGetting started with 100+ flowers on TPU](https://www.kaggle.com/mgornergoogle/getting-started-with-100-flowers-on-tpu).","metadata":{}},{"cell_type":"code","source":"!pip install -q efficientnet","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport re, math\nfrom sklearn.model_selection import train_test_split\nfrom kaggle_datasets import KaggleDatasets\nimport tensorflow as tf\nimport tensorflow.keras.layers as L\nimport tensorflow_addons as tfa\nimport efficientnet.tfkeras as efn","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\n# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect() # TPU detection\n    strategy = tf.distribute.TPUStrategy(tpu)\nexcept ValueError:\n    strategy = tf.distribute.MirroredStrategy() # for GPU or multi-GPU machines\n\nprint(\"Number of accelerators: \", strategy.num_replicas_in_sync)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path('herb2021-256')\nprint(GCS_DS_PATH)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    N_CLASSES = 64500\n    IMAGE_SIZE = [256, 256]\n    EPOCHS = 2\n    BATCH_SIZE = 16 * strategy.num_replicas_in_sync","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)  # image format uint8 [0,255]\n    image = tf.cast(image, tf.float32) / 255.0  # convert image to floats in [0, 1] range\n    image = tf.reshape(image, [*CFG.IMAGE_SIZE, 3])\n    return image\n\ndef get_idx(image, idnum):\n    idnum = tf.strings.split(idnum, sep='/')[6]\n    idnum = tf.strings.regex_replace(idnum, \".jpg\", \"\")\n    idnum = tf.strings.to_number(idnum, out_type=tf.int64)\n    return image, idnum\n\ndef onehot(image,label):\n    return image,tf.one_hot(label, CFG.N_CLASSES)\n    \ndef data_augment(image, label):\n    image = tf.image.random_flip_left_right(image)\n    return image, label\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'image_idx': tf.io.FixedLenFeature([], tf.string)\n    }\n\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum=example['image_idx']\n    return image, idnum\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'image_idx': tf.io.FixedLenFeature([], tf.string),\n        'label': tf.io.FixedLenFeature([], tf.int64),\n    }\n\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = example['label']\n    return image, label\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # disregarding data order. Order does not matter since we will be shuffling the data anyway.\n\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    return dataset\n\ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES)\n    dataset = dataset.map(onehot, num_parallel_calls=AUTO)\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(CFG.BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_validation_dataset():\n    dataset = load_dataset(VALIDATION_FILENAMES)\n    dataset = dataset.map(onehot, num_parallel_calls=AUTO)\n    dataset = dataset.batch(CFG.BATCH_SIZE)\n    #dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAINING_FILENAMES = tf.io.gfile.glob(GCS_DS_PATH + '/*.tfrec')\nTRAINING_FILENAMES, VALIDATION_FILENAMES = train_test_split(TRAINING_FILENAMES, test_size=0.15, \n                                                            shuffle=True, random_state=42)\nNUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // (2*CFG.BATCH_SIZE)\nprint('Dataset: {} training images, {} validation images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model():\n    base_model = efn.EfficientNetB2(weights='imagenet', \n                                    include_top=False, \n                                    pooling='avg',\n                                    input_shape=(*CFG.IMAGE_SIZE, 3))\n    model = tf.keras.Sequential([\n        base_model,\n        L.Dense(CFG.N_CLASSES, activation='softmax')\n    ])\n    \n    model.compile(optimizer='adam',\n                  loss='categorical_crossentropy',\n                  metrics=[tfa.metrics.F1Score(CFG.N_CLASSES, average='macro')])\n    \n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    model = get_model()\n    \nmodel.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr_callback = tf.keras.callbacks.ReduceLROnPlateau(patience=2, min_delta=0.001,\n                                                  monitor='val_loss', mode='min')\nes_callback = tf.keras.callbacks.EarlyStopping(patience=5, min_delta=0.001, \n                                               monitor='val_loss', mode='min',\n                                               restore_best_weights=True)\nchk_callback = tf.keras.callbacks.ModelCheckpoint('best.h5', monitor='val_loss', \n                                                  save_best_only=True,\n                                                  save_weights_only=True, \n                                                  mode='min')\nhistory = model.fit(\n    get_training_dataset(), \n    steps_per_epoch=STEPS_PER_EPOCH,\n    epochs=CFG.EPOCHS,\n    validation_data=get_validation_dataset(),\n    callbacks=[lr_callback, chk_callback, es_callback],\n    verbose=2)\n    \nmodel.save_weights(\"last.h5\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}