{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport re\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nfrom tensorflow.keras.activations import softmax, relu\nfrom tensorflow.keras.layers import Dense, BatchNormalization, Activation\nfrom tensorflow.keras.losses import categorical_crossentropy\nfrom tensorflow.keras.metrics import categorical_accuracy\nfrom tensorflow.keras.models import Model, Sequential\nfrom tensorflow.keras.optimizers import SGD, Adam\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.python.keras.layers import Input, Conv2D, MaxPool2D, Dropout, \\\n    GlobalAveragePooling2D\nfrom tensorflow.python.keras.layers import RandomRotation, RandomFlip, Add, GlobalAvgPool2D\nfrom tensorflow.keras.losses import categorical_crossentropy, sparse_categorical_crossentropy\nfrom tensorflow.data import Dataset, TFRecordDataset\nfrom kaggle_datasets import KaggleDatasets\n\nprint(\"Using TensorFlow version %s\" % tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print(f'Running on TPU {tpu.master()}')\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset_base_folder = KaggleDatasets().get_gcs_path(\"cassava-leaf-disease-classification\")\ndataset_train_folder = f\"{dataset_base_folder}/train\"\ndataset_val_folder = f\"{dataset_base_folder}/val\"\ndestination_classes = [str(i) for i in range(5)]\noriginal_train_data_folder = f\"{dataset_base_folder}/train_images\"\ncsv_file = f\"{dataset_base_folder}/train.csv\"\noriginal_train_tfrecs_folder = f'{dataset_base_folder}/train_tfrecords'\n\ncache_train = f'{dataset_base_folder}/dataset_cache/train'\ncache_test = f'{dataset_base_folder}/dataset_cache/val'\n\n\ndef compute_class_images_count(base_folder: str, class_name: str):\n    return sum((1 for _ in os.listdir(f'{base_folder}/{class_name}')))\n\n\ndef compute_all_classes_images_count(base_folder: str):\n    return sum((compute_class_images_count(base_folder, c) for c in destination_classes))\n\n\ndef compute_train_images_count():\n    return compute_all_classes_images_count(dataset_train_folder)\n\n\ndef compute_val_images_count():\n    return compute_all_classes_images_count(dataset_val_folder)\n\ndef count_data_items_tfrecs(filenames):\n    \"\"\"Count number of images in TFRecord files\"\"\"\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\ndef compute_train_val_size(base_folder: str, split: float) -> (int, int):\n    \"\"\"Return the size of train and validation dataset according to split\"\"\"\n    filenames = tf.io.gfile.glob(f'{base_folder}/ld_train*.tfrec')\n    dataset_size = count_data_items_tfrecs(filenames)\n    return (dataset_size * (1 - split), dataset_size * split)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#  **PARAMETERS**"},{"metadata":{"trusted":true},"cell_type":"code","source":"LR = 1e-4\nBATCH_SIZE_BASE = 32\nBATCH_SIZE = BATCH_SIZE_BASE * REPLICAS\nMOMENTUM = 0.8\nDROP_RATE = 0.2\n# TRAIN_SIZE = compute_train_images_count()\n# VAL_SIZE = compute_val_images_count()\nSPLIT = 0.2\nTRAIN_SIZE, VAL_SIZE = compute_train_val_size(original_train_tfrecs_folder, SPLIT)\nEPOCHS = 100\nTARGET_SIZE = (512, 512)\nOG_SIZE = (800, 600)\nNB_CLASSES = 5\nCHANNELS = 3\nSEED = 420\nNB_MODELS = 5\n\nLOCAL_LOGS_FOLDER = f\"./\"\nMODEL_ID = len([m for m in os.listdir(LOCAL_LOGS_FOLDER) if str(m).isnumeric()])\nLOCAL_LOGS_PATH = f'{LOCAL_LOGS_FOLDER}/{MODEL_ID}'\nAUTO = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **PREPROCESSING**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_dataset_iterator(base_folder: str, size: int, cache_folder: str):\n    \"\"\"Create Dataset Iterator from directory and cached on disk\"\"\"\n\n    def inner_func():\n        return ImageDataGenerator(rescale=1.0 / 255).flow_from_directory(base_folder,\n                                                                         target_size=TARGET_SIZE,\n                                                                         batch_size=1)\n\n    return (Dataset.from_generator(inner_func,\n                                   output_types=(tf.float32, tf.float32),\n                                   output_shapes=(\n                                       (1, *TARGET_SIZE, 3),\n                                       (1, len(destination_classes))\n                                   )\n                                   )\n            .take(size)\n            .unbatch()\n            .batch(BATCH_SIZE)\n            .cache(f'{cache_folder}/cache')\n            .repeat()\n            .prefetch(tf.data.experimental.AUTOTUNE)\n            .as_numpy_iterator()\n            )\n\n\ndef resize_input(x):\n    \"\"\"Resize an image to the desired size\"\"\"\n    x = tf.image.resize(x, [*TARGET_SIZE])\n    x = tf.reshape(x, [*TARGET_SIZE, CHANNELS])\n    return x\n\n\ndef decode_image(image_data) -> tf.image:\n    \"\"\"Decode Image of format String Feature to tf.utf8 then to float32\"\"\"\n    image = tf.image.decode_jpeg(image_data, channels=CHANNELS)\n    image = (tf.cast(image, tf.float32) / 255.0)\n    image = tf.image.resize(image, [*OG_SIZE])\n    image = tf.reshape(image, [*OG_SIZE, CHANNELS])\n    return image\n\n\ndef read_tfrecord(example, labeled=True):\n    \"\"\"Read a TFRecord(str, int64) and transform it into TFRecord(float32, int32)\"\"\"\n    if labeled:\n        TFREC_FORMAT = {\n            'image': tf.io.FixedLenFeature([], tf.string),\n            'target': tf.io.FixedLenFeature([], tf.int64),\n        }\n    else:\n        TFREC_FORMAT = {\n            'image': tf.io.FixedLenFeature([], tf.string),\n            'image_name': tf.io.FixedLenFeature([], tf.string),\n        }\n    example = tf.io.parse_single_example(example, TFREC_FORMAT)\n    image = decode_image(example['image'])\n\n    if labeled:\n        print(example['target'])\n        label_or_name = tf.cast(example['target'], tf.int32)\n        print(label_or_name)\n        # label_or_name.\n    else:\n        label_or_name = example['image_name']\n    return image, label_or_name\n\n\ndef load_dataset(filenames) -> Dataset:\n    \"\"\"Load tfrecord files and transform them into a tf.Dataset\"\"\"\n    ignore_order = tf.data.Options()\n    ignore_order.experimental_deterministic = False\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(lambda x: read_tfrecord(x), num_parallel_calls=AUTO)\n    return dataset\n\n\n# DATA AUGMENTATION\ndef random_apply_data_aug(dataset: Dataset) -> Dataset:\n    \"\"\"Randomly apply data transformation filters on dataset\"\"\"\n\n    def flip(x: tf.Tensor) -> tf.Tensor:\n        x = tf.image.random_flip_left_right(x)\n        x = tf.image.random_flip_up_down(x)\n        return x\n\n    def color(x: tf.Tensor) -> tf.Tensor:\n        x = tf.image.random_hue(x, 0.3, seed=SEED)\n        x = tf.image.random_saturation(x, 0.6, 1.6)\n        x = tf.image.random_brightness(x, 0.05)\n        x = tf.image.random_contrast(x, 0.7, 1.3)\n        return x\n\n    def rotate(x: tf.Tensor) -> tf.Tensor:\n        return tf.image.rot90(x, tf.random.uniform(shape=[], minval=0, maxval=4, dtype=tf.int32))\n\n    def zoom(x: tf.Tensor) -> tf.Tensor:\n        return tf.image.random_crop(x, [int(TARGET_SIZE[0] * 0.8), int(TARGET_SIZE[1] * 0.8), 3])\n\n    funcs_data_aug = [flip, color, rotate, zoom]\n    for f in funcs_data_aug:\n        dataset = dataset.map(\n            lambda x, y: tf.cond(\n                tf.random.uniform([], 0, 1) > 0.5,\n                lambda: (f(x), y),\n                lambda: (x, y)),\n            num_parallel_calls=AUTO)\n    return dataset\n\n\ndef create_dataset_tfrec_input(base_folder: str) -> (TFRecordDataset, TFRecordDataset):\n    filenames = tf.io.gfile.glob(f'{base_folder}/ld_train*.tfrec')\n    dataset = load_dataset(filenames)\n    dataset = dataset.shuffle(buffer_size=2048)\n    return dataset\n\n\ndef split_dataset(dataset: Dataset,\n                  augment=True,\n                  validation_only=False,\n                  k_fold=0,\n                  index=None):\n    \"\"\"\n    Args:\n        dataset: dataset to be splitted\n        augment: apply augmentation on train dataset\n        validation_only: return only the validation dataset\n        k_fold: If > 0, split dataset into k folds of the same size.\n        index: must be specified by a positive integer when k_fold > 0 indicating the index of the\n        fold that will be used as validation dataset, all the other folds will become train dataset.\n\n    Return: a train dataset and a validation dataset splitted from dataset by a pre-determined \"split\" factor\n    \"\"\"\n    if k_fold == 0:\n        dataset.shuffle(1000)\n        validation_dataset = dataset.take(VAL_SIZE)\n        validation_dataset = validation_dataset.map(lambda x, y: (resize_input(x), y))\n        validation_dataset = validation_dataset.batch(BATCH_SIZE).repeat().prefetch(AUTO)\n\n        if not validation_only:\n            train_dataset = dataset.skip(VAL_SIZE)\n            if augment:\n                train_dataset = random_apply_data_aug(train_dataset)\n            train_dataset = train_dataset.map(lambda x, y: (resize_input(x), y))\n            train_dataset = train_dataset.batch(BATCH_SIZE).repeat().prefetch(AUTO)\n\n            return train_dataset, validation_dataset\n        else:\n            return validation_dataset\n    elif k_fold > 0:\n        if index is None or index < 0 or index >= k_fold:\n            exit(\"index of validation fold not specified or not valid\")\n        else:\n            val_start = VAL_SIZE * index\n            val_end = val_start + VAL_SIZE\n            validation_dataset = dataset.skip(val_start)\n            validation_dataset = validation_dataset.take(VAL_SIZE)\n            validation_dataset = validation_dataset.map(lambda x, y: (resize_input(x), y))\n            validation_dataset = validation_dataset.batch(BATCH_SIZE).repeat().prefetch(AUTO)\n\n            if not validation_only:\n                train_dataset_1 = dataset.take(val_start)\n                train_dataset_2 = dataset.skip(val_end)\n                train_dataset = train_dataset_1.concatenate(train_dataset_2)\n                if augment:\n                    train_dataset = random_apply_data_aug(train_dataset)\n                train_dataset = train_dataset.map(lambda x, y: (resize_input(x), y))\n                train_dataset = train_dataset.batch(BATCH_SIZE).repeat().prefetch(AUTO)\n\n                return train_dataset, validation_dataset\n            else:\n                return validation_dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = create_dataset_tfrec_input(original_train_tfrecs_folder)\ndataset_train, dataset_test = split_dataset(dataset, augment=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **DEFINE MODELS**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_base_model(add_custom_layers_func) -> Model:\n    m = Sequential()\n\n    add_custom_layers_func(m)\n\n    m.add(Dense(NB_CLASSES, activation=softmax))\n\n    return m\n\n\ndef add_resNet50(m: Sequential):\n    inputs = Input(shape=(*TARGET_SIZE, 3))\n    base_model = tf.keras.applications.ResNet50(include_top=False, weights='imagenet', input_shape=(*TARGET_SIZE, 3))\n    base_model.trainable = False\n    base_model = base_model(inputs, training=False)\n    gap2d = GlobalAveragePooling2D(name='avg_pool')(base_model)\n    bn = BatchNormalization(name='top_bn')(gap2d)\n    dense = Dense(1000, activation=relu, name='fc1000')(bn)\n#     dropout = Dropout(DROP_RATE, name='top_drop')(dense)\n\n    model = Model(inputs, dense)\n    m.add(model)\n    \n    \ndef make_resNet50():\n    inputs = Input(shape=(*TARGET_SIZE, 3))\n    base_model = tf.keras.applications.ResNet50V2(include_top=False, weights='imagenet', input_shape=(*TARGET_SIZE, 3))\n    base_model.trainable = False\n    base_model = base_model(inputs, training=False)\n    gap2d = GlobalAveragePooling2D(name='avg_pool')(base_model)\n    bn = BatchNormalization(name='top_bn')(gap2d)\n#     dense = Dense(1000, activation=relu, name='fc1000')(bn)\n    pred = Dense(NB_CLASSES, activation=softmax)(bn)\n#     dropout = Dropout(DROP_RATE, name='top_drop')(dense)\n\n    model = Model(inputs, pred)\n    return model\n\n\ndef get_callbacks():\n    callbacks = []\n\n    save_callback = tf.keras.callbacks.ModelCheckpoint(\n        filepath=f'ResNet50_best.h5',\n        monitor='val_categorical_accuracy',\n        mode='max',\n        save_best_only=True)\n    callbacks.append(save_callback)\n\n    reduce_LR = tf.keras.callbacks.ReduceLROnPlateau(\n        monitor='val_acc', factor=0.2, patience=5,\n        mode='max', min_delta=0.0005, min_lr=0.000001\n    )\n    callbacks.append(reduce_LR)\n\n    earlystopper = tf.keras.callbacks.EarlyStopping(\n        monitor='val_categorical_accuracy',\n        min_delta=0.005,\n        patience=10,\n        mode='max',\n        restore_best_weights=True\n    )\n    callbacks.append(earlystopper)\n    return callbacks\n\n\ndef train_models(m: Model, dataset_train_it, dataset_val_it, lr=None, epochs=None):\n    lr = lr if LR is None else lr\n    epochs = EPOCHS if epochs is None else epochs\n    # m.compile(\n    #     optimizer=SGD(momentum=MOMENTUM, lr=LR),\n    #     loss=categorical_crossentropy,\n    #     metrics=[categorical_accuracy]\n    # )\n    m.compile(\n        optimizer=SGD(momentum=MOMENTUM, lr=lr),\n        loss=sparse_categorical_crossentropy,\n        metrics=[categorical_accuracy]\n    )\n    m.summary()\n\n    history = m.fit(\n        dataset_train_it,\n        validation_data=dataset_val_it,\n        steps_per_epoch=TRAIN_SIZE // BATCH_SIZE,\n        validation_steps=VAL_SIZE // BATCH_SIZE,\n        epochs=EPOCHS,\n        callbacks=get_callbacks()\n    )\n    return history\n\ndef train_pretrained_model(m: Model, dataset_train_it, dataset_val_it, lr, epochs):\n    history_pre_finetuning = train_models(m, dataset_train_it, dataset_val_it, lr=lr[0], epochs=epochs[0])\n    m.trainable = True\n    history_post_finetuning = train_models(m, dataset_train_it, dataset_val_it, lr=lr[1], epochs=epochs[1])\n    history = [history_pre_finetuning, history_post_finetuning]\n    return history","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **CREATE MODEL**"},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    model = make_resNet50()\n#     model = create_base_model(add_resNet50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"lr = [0.0005, 0.00001]\nepochs = [5, 5]\nhistory = train_pretrained_model(model, dataset_train, dataset_test, lr, epochs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    inputs = Input(shape=(*TARGET_SIZE, 3))\n    \n    norm_layer = tf.keras.layers.experimental.preprocessing.Normalization()\n    mean = np.array([127.5] * 3)\n    var = mean ** 2\n    # Scale inputs to [-1, +1]\n    x = norm_layer(inputs)\n    norm_layer.set_weights([mean, var])\n    \n    base_model = tf.keras.applications.ResNet50V2(include_top=False, weights='imagenet', input_shape=(*TARGET_SIZE, 3))\n    base_model.trainable = False\n    \n    x = base_model(x, training=False)\n    x = GlobalAveragePooling2D()(x)\n    x = Dropout(0.2)(x)  # Regularize with dropout\n    outputs = Dense(NB_CLASSES, activation=softmax)(x)\n    model = Model(inputs, outputs)\n    \n    model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(\n    optimizer=Adam(),\n    loss=sparse_categorical_crossentropy,\n    metrics=[categorical_accuracy],\n)\n\nepochs = 20\nmodel.fit(dataset_train, epochs=epochs, validation_data=dataset_test,\n         steps_per_epoch=TRAIN_SIZE // BATCH_SIZE,\n        validation_steps=VAL_SIZE // BATCH_SIZE,)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base_model.trainable = True\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(nrows=5, ncols=2, figsize=(16, 12))\nhistory_df = pd.DataFrame(history.history)\nhistory_df[['loss', 'val_loss']].plot(ax=axes[i,0])\nhistory_df[['accuracy', 'val_accuracy']].plot(ax=axes[i,1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"models = []\nwith strategy.scope():\n    for i in range(n_models):\n        models.append(create_base_model(add_resNet50))\n        \nhistories = []\nfor i, model in enumerate(models):\n    print(F\"Training model: {i}\")\n    train_dataset, validation_dataset = n_fold_dataset(augment = True, index=i)\n    history = train_pretrained_model(model, train_dataset, validation_dataset)\n    histories.append(history)"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"for i in range(n_models):\n    validation_dataset = n_fold_dataset(augment = False, train=False, index=i)\n    for model in models:\n        model.evaluate(validation_dataset)"},{"metadata":{},"cell_type":"markdown","source":"fig, axes = plt.subplots(nrows=5, ncols=2, figsize=(16, 12))\n\nfor i, history in enumerate(histories):\n    history_df = pd.DataFrame(history.history)\n    history_df[['loss', 'val_loss']].plot(ax=axes[i,0])\n    history_df[['accuracy', 'val_accuracy']].plot(ax=axes[i,1])"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}