{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Flower Classification using CNN Models\n\nWe recovered Phil Cullington's notebook using TPU to make Flower Classification.  \nValue of initial accuracy : ~0.35","metadata":{}},{"cell_type":"markdown","source":"# Getting libraries","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom kaggle_datasets import KaggleDatasets\nimport numpy as np\nimport math\nimport matplotlib.pyplot as plt\nimport re\nprint(\"Tensorflow version \" + tf.__version__)\n!pip install efficientnet\nimport efficientnet.tfkeras as efficientnet","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:32:16.571625Z","iopub.execute_input":"2022-11-09T22:32:16.572232Z","iopub.status.idle":"2022-11-09T22:32:27.947791Z","shell.execute_reply.started":"2022-11-09T22:32:16.572183Z","shell.execute_reply":"2022-11-09T22:32:27.945544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Detect my accelerator","metadata":{}},{"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = (\n        tf.distribute.cluster_resolver.TPUClusterResolver()\n    )  # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n    print(\"Running on TPU \", tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = (\n        tf.distribute.get_strategy()\n    )  # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:32:27.952014Z","iopub.execute_input":"2022-11-09T22:32:27.952492Z","iopub.status.idle":"2022-11-09T22:32:29.903048Z","shell.execute_reply.started":"2022-11-09T22:32:27.952374Z","shell.execute_reply":"2022-11-09T22:32:29.900475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data path\n### Look on work folders","metadata":{}},{"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path(\n    \"tpu-getting-started\"\n)  # you can list the bucket with \"!gsutil ls $GCS_DS_PATH\"\n# GCS_DS_PATH\n!gsutil ls $GCS_DS_PATH","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:32:29.904856Z","iopub.status.idle":"2022-11-09T22:32:29.905695Z","shell.execute_reply.started":"2022-11-09T22:32:29.905353Z","shell.execute_reply":"2022-11-09T22:32:29.905392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Set parameters","metadata":{}},{"cell_type":"code","source":"size = 331\nIMAGE_SIZE = [size, size]  # at 512x512, a GPU will run out of memory. Use the TPU\nEPOCHS = 15\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync # Use this line for TPU\n#BATCH_SIZE = 8 * strategy.num_replicas_in_sync  # Use this line for GPU\nAUTO = tf.data.AUTOTUNE","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:32:29.907407Z","iopub.status.idle":"2022-11-09T22:32:29.908262Z","shell.execute_reply.started":"2022-11-09T22:32:29.907890Z","shell.execute_reply":"2022-11-09T22:32:29.907923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GCS_PATH_EXT = KaggleDatasets().get_gcs_path('tf-flower-photo-tfrec')\n\nIMAGENET_FILES = tf.io.gfile.glob(GCS_PATH_EXT + f'/imagenet_no_test/tfrecords-jpeg-{size}x{size}/*.tfrec'.format(size=size))\nINATURELIST_FILES = tf.io.gfile.glob(GCS_PATH_EXT + f'/inaturalist_no_test/tfrecords-jpeg-{size}x{size}/*.tfrec'.format(size=size))\nOPENIMAGE_FILES = tf.io.gfile.glob(GCS_PATH_EXT + f'/openimage_no_test/tfrecords-jpeg-{size}x{size}/*.tfrec'.format(size=size))\nOXFORD_FILES = tf.io.gfile.glob(GCS_PATH_EXT + f'/oxford_102_no_test/tfrecords-jpeg-{size}x{size}/*.tfrec'.format(size=size))\nTENSORFLOW_FILES = tf.io.gfile.glob(GCS_PATH_EXT + f'/tf_flowers_no_test/tfrecords-jpeg-{size}x{size}/*.tfrec'.format(size=size))","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:32:29.910279Z","iopub.status.idle":"2022-11-09T22:32:29.911131Z","shell.execute_reply.started":"2022-11-09T22:32:29.910777Z","shell.execute_reply":"2022-11-09T22:32:29.910828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define classes name","metadata":{}},{"cell_type":"code","source":"CLASSES = [\n    \"pink primrose\",\n    \"hard-leaved pocket orchid\",\n    \"canterbury bells\",\n    \"sweet pea\",\n    \"wild geranium\",\n    \"tiger lily\",\n    \"moon orchid\",\n    \"bird of paradise\",\n    \"monkshood\",\n    \"globe thistle\",  # 00 - 09\n    \"snapdragon\",\n    \"colt's foot\",\n    \"king protea\",\n    \"spear thistle\",\n    \"yellow iris\",\n    \"globe-flower\",\n    \"purple coneflower\",\n    \"peruvian lily\",\n    \"balloon flower\",\n    \"giant white arum lily\",  # 10 - 19\n    \"fire lily\",\n    \"pincushion flower\",\n    \"fritillary\",\n    \"red ginger\",\n    \"grape hyacinth\",\n    \"corn poppy\",\n    \"prince of wales feathers\",\n    \"stemless gentian\",\n    \"artichoke\",\n    \"sweet william\",  # 20 - 29\n    \"carnation\",\n    \"garden phlox\",\n    \"love in the mist\",\n    \"cosmos\",\n    \"alpine sea holly\",\n    \"ruby-lipped cattleya\",\n    \"cape flower\",\n    \"great masterwort\",\n    \"siam tulip\",\n    \"lenten rose\",  # 30 - 39\n    \"barberton daisy\",\n    \"daffodil\",\n    \"sword lily\",\n    \"poinsettia\",\n    \"bolero deep blue\",\n    \"wallflower\",\n    \"marigold\",\n    \"buttercup\",\n    \"daisy\",\n    \"common dandelion\",  # 40 - 49\n    \"petunia\",\n    \"wild pansy\",\n    \"primula\",\n    \"sunflower\",\n    \"lilac hibiscus\",\n    \"bishop of llandaff\",\n    \"gaura\",\n    \"geranium\",\n    \"orange dahlia\",\n    \"pink-yellow dahlia\",  # 50 - 59\n    \"cautleya spicata\",\n    \"japanese anemone\",\n    \"black-eyed susan\",\n    \"silverbush\",\n    \"californian poppy\",\n    \"osteospermum\",\n    \"spring crocus\",\n    \"iris\",\n    \"windflower\",\n    \"tree poppy\",  # 60 - 69\n    \"gazania\",\n    \"azalea\",\n    \"water lily\",\n    \"rose\",\n    \"thorn apple\",\n    \"morning glory\",\n    \"passion flower\",\n    \"lotus\",\n    \"toad lily\",\n    \"anthurium\",  # 70 - 79\n    \"frangipani\",\n    \"clematis\",\n    \"hibiscus\",\n    \"columbine\",\n    \"desert-rose\",\n    \"tree mallow\",\n    \"magnolia\",\n    \"cyclamen \",\n    \"watercress\",\n    \"canna lily\",  # 80 - 89\n    \"hippeastrum \",\n    \"bee balm\",\n    \"pink quill\",\n    \"foxglove\",\n    \"bougainvillea\",\n    \"camellia\",\n    \"mallow\",\n    \"mexican petunia\",\n    \"bromelia\",\n    \"blanket flower\",  # 90 - 99\n    \"trumpet creeper\",\n    \"blackberry lily\",\n    \"common tulip\",\n    \"wild rose\",\n]","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-11-09T22:32:29.913091Z","iopub.status.idle":"2022-11-09T22:32:29.913933Z","shell.execute_reply.started":"2022-11-09T22:32:29.913541Z","shell.execute_reply":"2022-11-09T22:32:29.913572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"VAL_FILENAMES = tf.io.gfile.glob(GCS_DS_PATH + f\"/tfrecords-jpeg-{size}x{size}/val/*.tfrec\")\nTEST_FILESNAMES = tf.io.gfile.glob(GCS_DS_PATH + f\"/tfrecords-jpeg-{size}x{size}/test/*.tfrec\")","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:32:29.915260Z","iopub.status.idle":"2022-11-09T22:32:29.915679Z","shell.execute_reply.started":"2022-11-09T22:32:29.915461Z","shell.execute_reply":"2022-11-09T22:32:29.915502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAINING_FILENAMES = tf.io.gfile.glob(GCS_DS_PATH + f\"/tfrecords-jpeg-{size}x{size}/train/*.tfrec\") + TENSORFLOW_FILES + OXFORD_FILES + OPENIMAGE_FILES + INATURELIST_FILES + IMAGENET_FILES \n#TRAINING_FILENAMES = tf.io.gfile.glob(GCS_DS_PATH + f\"/tfrecords-jpeg-{size}x{size}/train/*.tfrec\") + TENSORFLOW_FILES + OXFORD_FILES\n# TRAINING_FILENAMES = tf.io.gfile.glob(GCS_DS_PATH + f\"/tfrecords-jpeg-{size}x{size}/train/*.tfrec\") + VAL_FILENAMES+ TENSORFLOW_FILES + OXFORD_FILES","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:32:29.917503Z","iopub.status.idle":"2022-11-09T22:32:29.917912Z","shell.execute_reply.started":"2022-11-09T22:32:29.917697Z","shell.execute_reply":"2022-11-09T22:32:29.917716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" # Functions space","metadata":{}},{"cell_type":"code","source":"### Decoding image\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = (\n        tf.cast(image, tf.float32) / 255.0\n    )  # convert image to floats in [0, 1] range\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])  # explicit size needed for TPU\n    return image\n\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),  # tf.string means bytestring\n        \"class\": tf.io.FixedLenFeature([], tf.int64),  # shape [] means single element\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example[\"image\"])\n    label = tf.cast(example[\"class\"], tf.int32)\n    return image, label  # returns a dataset of (image, label) pairs\n\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),  # tf.string means bytestring\n        \"id\": tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n        # class is missing, this competitions's challenge is to predict flower classes for the test dataset\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example[\"image\"])\n    idnum = example[\"id\"]\n    return image, idnum  # returns a dataset of image(s)\n\n\n### Data augmentation function\n#### Horizontal and vertical mirorring\n#### Random cropping\n#### Random rotation\n#### Saturation, brightness and contrast can be modified randomly too\n\n\ndef data_augment(x, y):\n    x = tf.image.random_flip_left_right(x)\n    #x = tf.image.random_flip_up_down(x)\n    #x = tf.image.random_crop(x, size = [size,size,3])\n    #p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    # if p_rotate > .8:\n    #    x = tf.image.rot90(x, k=3)\n    # elif p_rotate > .6:\n    #    x = tf.image.rot90(x, k=2)\n    # elif p_rotate > .4:\n    #    x = tf.image.rot90(x, k=1)\n    #x = tf.image.random_saturation(x,0.5,2)\n    #x = tf.image.random_brightness(x, 2)\n    #x = tf.image.random_contrast(x,0.8,1.2)\n    return x, y\n\n\n### Loading data\ndef load_dataset(filenames, labeled=True, ordered=False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # disregarding data order. Order does not matter since we will be shuffling the data anyway.\n\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False  # disable order, increase speed\n\n    dataset = tf.data.TFRecordDataset(\n        filenames\n    )  # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(\n        ignore_order\n    )  # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    return dataset\n\n\n### Getting training, validation and test dataset\ndef get_training_dataset():\n    dataset = load_dataset(\n        TRAINING_FILENAMES,\n        labeled=True,\n    )\n    dataset = dataset.repeat()  # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    return dataset\n\n\ndef get_validation_dataset(ordered = False):\n    dataset = load_dataset(\n        tf.io.gfile.glob(GCS_DS_PATH + f\"/tfrecords-jpeg-{size}x{size}/val/*.tfrec\"),\n        labeled=True,\n        ordered=ordered,\n    )\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    return dataset\n\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(\n        tf.io.gfile.glob(GCS_DS_PATH + f\"/tfrecords-jpeg-{size}x{size}/test/*.tfrec\"),\n        labeled=False,\n        ordered=ordered,\n    )\n    dataset = dataset.batch(BATCH_SIZE)\n    return dataset\n\n\ntraining_dataset = get_training_dataset()\nvalidation_dataset = get_validation_dataset()\n\n\ndef batch_to_numpy_images_and_labels(data):\n    images, labels = data\n    numpy_images = images.numpy()\n    numpy_labels = labels.numpy()\n    if numpy_labels.dtype == object:  # binary string in this case,\n        # these are image ID strings\n        numpy_labels = [None for _ in enumerate(numpy_images)]\n    # If no labels, only image IDs, return None for labels (this is\n    # the case for test data)\n    return numpy_images, numpy_labels\n\n\ndef title_from_label_and_target(label, correct_label):\n    if correct_label is None:\n        return CLASSES[label], True\n    correct = label == correct_label\n    return (\n        \"{} [{}{}{}]\".format(\n            CLASSES[label],\n            \"OK\" if correct else \"NO\",\n            \"\\u2192\" if not correct else \"\",\n            CLASSES[correct_label] if not correct else \"\",\n        ),\n        correct,\n    )\n\n\ndef display_one_flower(image, title, subplot, red=False, titlesize=16):\n    plt.subplot(*subplot)\n    plt.axis(\"off\")\n    plt.imshow(image)\n    if len(title) > 0:\n        plt.title(\n            title,\n            fontsize=int(titlesize) if not red else int(titlesize / 1.2),\n            color=\"red\" if red else \"black\",\n            fontdict={\"verticalalignment\": \"center\"},\n            pad=int(titlesize / 1.5),\n        )\n    return (subplot[0], subplot[1], subplot[2] + 1)\n\n\ndef display_batch_of_images(databatch, predictions=None):\n    \"\"\"This will work with:\n    display_batch_of_images(images)\n    display_batch_of_images(images, predictions)\n    display_batch_of_images((images, labels))\n    display_batch_of_images((images, labels), predictions)\n    \"\"\"\n    # data\n    images, labels = batch_to_numpy_images_and_labels(databatch)\n    if labels is None:\n        labels = [None for _ in enumerate(images)]\n\n    # auto-squaring: this will drop data that does not fit into square\n    # or square-ish rectangle\n    rows = int(math.sqrt(len(images)))\n    cols = len(images) // rows\n\n    # size and spacing\n    FIGSIZE = 13.0\n    SPACING = 0.1\n    subplot = (rows, cols, 1)\n    if rows < cols:\n        plt.figure(figsize=(FIGSIZE, FIGSIZE / cols * rows))\n    else:\n        plt.figure(figsize=(FIGSIZE / rows * cols, FIGSIZE))\n\n    # display\n    for i, (image, label) in enumerate(\n        zip(images[: rows * cols], labels[: rows * cols])\n    ):\n        title = \"\" if label is None else CLASSES[label]\n        correct = True\n        if predictions is not None:\n            title, correct = title_from_label_and_target(predictions[i], label)\n        dynamic_titlesize = (\n            FIGSIZE * SPACING / max(rows, cols) * 40 + 3\n        )  # magic formula tested to work from 1x1 to 10x10 images\n        subplot = display_one_flower(\n            image, title, subplot, not correct, titlesize=dynamic_titlesize\n        )\n\n    # layout\n    plt.tight_layout()\n    if label is None and predictions is None:\n        plt.subplots_adjust(wspace=0, hspace=0)\n    else:\n        plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n    plt.show()\n\n\n### Defining the evolution of loss parameter\ndef lr_function(epoch):\n    start_lr = 1e-3\n    min_lr = 5e-5\n    max_lr = 1e-3\n    rampup_epochs = 5\n    sustain_epochs = 0\n    exp_decay = 0.8\n\n    def lr(epoch, start_lr, min_lr, max_lr, rampup_epochs, sustain_epochs, exp_decay):\n        if epoch < rampup_epochs:\n            lr = (max_lr - start_lr) / rampup_epochs * epoch + start_lr\n        elif epoch < rampup_epochs + sustain_epochs:\n            lr = max_lr\n        else:  # E\n            lr = (max_lr - min_lr) * exp_decay ** (\n                epoch - rampup_epochs - sustain_epochs\n            ) + min_lr\n        return lr\n\n    return lr(epoch, start_lr, min_lr, max_lr, rampup_epochs, sustain_epochs, exp_decay)\n\n\ncallbacks = [\n    tf.keras.callbacks.EarlyStopping(monitor='val_loss',\n                                    patience=5,\n                                    restore_best_weights=True),\n    tf.keras.callbacks.LearningRateScheduler(\n        lambda epoch: lr_function(epoch), verbose=True  # B\n    )\n]\n\n### Define function to display results\ndef display_training_curves(training, validation, title, subplot):\n    if subplot % 10 == 1:\n        plt.subplots(figsize=(10, 10), facecolor=\"#F0F0F0\")\n        plt.tight_layout()\n    ax = plt.subplot(subplot)\n    ax.set_facecolor(\"#F8F8F8\")\n    ax.plot(training)\n    ax.plot(validation)\n    ax.set_title(\"model \" + title)\n    ax.set_ylabel(title)\n    ax.set_xlabel(\"epoch\")\n    ax.legend([\"train \", \"valid\"])\n        \ndef count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec\n    # files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:32:29.919599Z","iopub.status.idle":"2022-11-09T22:32:29.920005Z","shell.execute_reply.started":"2022-11-09T22:32:29.919790Z","shell.execute_reply":"2022-11-09T22:32:29.919808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_TEST_IMAGES = 7382\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:32:29.921460Z","iopub.status.idle":"2022-11-09T22:32:29.921870Z","shell.execute_reply.started":"2022-11-09T22:32:29.921655Z","shell.execute_reply":"2022-11-09T22:32:29.921674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize Data","metadata":{}},{"cell_type":"code","source":"ds_train = get_training_dataset()\nds_iter = iter(ds_train.unbatch().batch(20))\none_batch = next(ds_iter)\ndisplay_batch_of_images(one_batch)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:32:29.923017Z","iopub.status.idle":"2022-11-09T22:32:29.923386Z","shell.execute_reply.started":"2022-11-09T22:32:29.923194Z","shell.execute_reply":"2022-11-09T22:32:29.923212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Models","metadata":{}},{"cell_type":"markdown","source":"# DenseNet201","metadata":{}},{"cell_type":"code","source":"print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:32:29.924294Z","iopub.status.idle":"2022-11-09T22:32:29.924673Z","shell.execute_reply.started":"2022-11-09T22:32:29.924472Z","shell.execute_reply":"2022-11-09T22:32:29.924497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nwith strategy.scope():\n    pretrained_model = tf.keras.applications.DenseNet201(\n        weights=\"imagenet\", include_top=False, input_shape=(size, size, 3)\n    )\n    pretrained_model.trainable = True  # transfer learning\n    model3= tf.keras.Sequential(\n        [\n            pretrained_model,\n            # tf.keras.layers.BatchNormalization(),\n            tf.keras.layers.GlobalAveragePooling2D(),\n            # tf.keras.layers.Dropout(0.1),\n            tf.keras.layers.Dense(104, activation=\"softmax\"),\n        ]\n    )\n\nmodel3.compile(\n    optimizer=\"adam\",\n    loss=\"sparse_categorical_crossentropy\",\n    metrics=[\"sparse_categorical_accuracy\"],\n)\n\nhistorical = model3.fit(\n    training_dataset,\n    steps_per_epoch=STEPS_PER_EPOCH,\n    epochs=EPOCHS,\n    validation_data=validation_dataset,\n    callbacks=callbacks,\n)\n\ndisplay_training_curves(\n    historical.history[\"loss\"],\n    historical.history[\"val_loss\"],\n    \"loss\",\n    211,\n)\n\ndisplay_training_curves(\n    historical.history[\"sparse_categorical_accuracy\"],\n    historical.history[\"val_sparse_categorical_accuracy\"],\n    \"accuracy\",\n    212,\n)","metadata":{"execution":{"iopub.status.busy":"2022-10-29T07:15:49.629664Z","iopub.execute_input":"2022-10-29T07:15:49.630503Z","iopub.status.idle":"2022-10-29T07:23:11.987915Z","shell.execute_reply.started":"2022-10-29T07:15:49.630431Z","shell.execute_reply":"2022-10-29T07:23:11.986877Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# RESNET","metadata":{}},{"cell_type":"markdown","source":"with strategy.scope():\n    pretrained_model = tf.keras.applications.ResNet152V2(\n        weights=\"imagenet\", include_top=False, input_shape=(512, 512, 3)\n    )\n    pretrained_model.trainable = True  # transfer learning\n\n    model = tf.keras.Sequential(\n        [\n            pretrained_model,\n            # tf.keras.layers.BatchNormalization(),\n            tf.keras.layers.GlobalAveragePooling2D(),\n            # tf.keras.layers.Dropout(0.2),\n            tf.keras.layers.Dense(104, activation=\"softmax\"),\n        ]\n    )\n\nmodel.compile(\n    optimizer=\"adam\",\n    loss=\"sparse_categorical_crossentropy\",\n    metrics=[\"sparse_categorical_accuracy\"],\n)\n\nhistorical = model.fit(\n    training_dataset,\n    steps_per_epoch=STEPS_PER_EPOCH,\n    epochs=EPOCHS,\n    validation_data=validation_dataset,\n    callbacks=callbacks,\n)\n\ndisplay_training_curves(\n    historical.history[\"loss\"],\n    historical.history[\"val_loss\"],\n    \"loss\",\n    211,\n)\n\ndisplay_training_curves(\n    historical.history[\"sparse_categorical_accuracy\"],\n    historical.history[\"val_sparse_categorical_accuracy\"],\n    \"accuracy\",\n    212,\n)","metadata":{}},{"cell_type":"markdown","source":"# Xception","metadata":{}},{"cell_type":"markdown","source":"with strategy.scope():\n    pretrained_model = tf.keras.applications.Xception(\n        weights=\"imagenet\", include_top=False, input_shape=(512, 512, 3)\n    )\n    pretrained_model.trainable = True  # transfer learning\n\n    model = tf.keras.Sequential(\n        [\n            pretrained_model,\n            # tf.keras.layers.BatchNormalization(),\n            tf.keras.layers.GlobalAveragePooling2D(),\n            # tf.keras.layers.Dropout(0.2),\n            tf.keras.layers.Dense(104, activation=\"softmax\"),\n        ]\n    )\n\nmodel.compile(\n    optimizer=\"adam\",\n    loss=\"sparse_categorical_crossentropy\",\n    metrics=[\"sparse_categorical_accuracy\"],\n)\n\nhistorical = model.fit(\n    training_dataset,\n    steps_per_epoch=STEPS_PER_EPOCH,\n    epochs=EPOCHS,\n    validation_data=validation_dataset,\n    callbacks=callbacks,\n)\n\ndisplay_training_curves(\n    historical.history[\"loss\"],\n    historical.history[\"val_loss\"],\n    \"loss\",\n    211,\n)\n\ndisplay_training_curves(\n    historical.history[\"sparse_categorical_accuracy\"],\n    historical.history[\"val_sparse_categorical_accuracy\"],\n    \"accuracy\",\n    212,\n)","metadata":{}},{"cell_type":"markdown","source":"with strategy.scope():    \n    pretrained_model = efficientnet.EfficientNetB7(weights='noisy-student', \n                                                         include_top=False ,\n                                                         input_shape=(512,512,3))\n    pretrained_model.trainable = True # tramsfer learning","metadata":{"execution":{"iopub.execute_input":"2022-10-23T22:02:28.330514Z","iopub.status.busy":"2022-10-23T22:02:28.330232Z","iopub.status.idle":"2022-10-23T22:02:59.881319Z","shell.execute_reply":"2022-10-23T22:02:59.879651Z","shell.execute_reply.started":"2022-10-23T22:02:28.330486Z"}}},{"cell_type":"markdown","source":"# EfficientNetV2XL","metadata":{}},{"cell_type":"markdown","source":"This model has been downloaded from this link : https://github.com/leondgarse/keras_cv_attention_models\nThe tensorflow's version on kaggle is not sufficiently updated to get this model via keras.applications, we thus get it, and load it using load_model function.","metadata":{"execution":{"iopub.execute_input":"2022-10-25T10:01:32.155042Z","iopub.status.busy":"2022-10-25T10:01:32.154757Z","iopub.status.idle":"2022-10-25T10:01:32.159762Z","shell.execute_reply":"2022-10-25T10:01:32.158468Z","shell.execute_reply.started":"2022-10-25T10:01:32.155009Z"}}},{"cell_type":"code","source":"with strategy.scope():\n    modelv2xl = tf.keras.models.load_model(\n        \"/kaggle/input/effnetv2xl/efficientnetv2-xl-21k-ft1k.h5\", compile=True\n    )\n    modelv2xl.trainable = False\n    model1 = tf.keras.Sequential(\n        [\n            modelv2xl,\n            # tf.keras.layers.BatchNormalization(),\n            #tf.keras.layers.Dropout(0.2),\n            # tf.keras.layers.Reshape((10,10,10)),\n            # tf.keras.layers.GlobalAveragePooling2D(),\n            tf.keras.layers.Dense(3000, activation=\"relu\"),\n            tf.keras.layers.Dropout(0.2),\n            tf.keras.layers.Dense(104, activation=\"softmax\"), \n            #tf.keras.layers.Dense(104, activation=\"relu\"),\n        ]\n    )\nmodel1.summary()\n\nmodel1.compile(\n    optimizer=\"adam\",\n    loss=\"sparse_categorical_crossentropy\",\n    metrics=[\"sparse_categorical_accuracy\"],\n)\n\nhistorical = model1.fit(\n    training_dataset,\n    steps_per_epoch=STEPS_PER_EPOCH,\n    epochs=EPOCHS,\n    validation_data=validation_dataset,\n    callbacks=callbacks,\n)\n\ndisplay_training_curves(\n    historical.history[\"loss\"],\n    historical.history[\"val_loss\"],\n    \"loss\",\n    211,\n)\n\ndisplay_training_curves(\n    historical.history[\"sparse_categorical_accuracy\"],\n    historical.history[\"val_sparse_categorical_accuracy\"],\n    \"accuracy\",\n    212,\n)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:32:29.928258Z","iopub.status.idle":"2022-11-09T22:32:29.928686Z","shell.execute_reply.started":"2022-11-09T22:32:29.928472Z","shell.execute_reply":"2022-11-09T22:32:29.928495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model1.save(\"/kaggle/working/model_effnetv2xl.h5\", save_format=\"h5\")","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:32:29.931675Z","iopub.status.idle":"2022-11-09T22:32:29.932385Z","shell.execute_reply.started":"2022-11-09T22:32:29.932152Z","shell.execute_reply":"2022-11-09T22:32:29.932180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EfficientNetB7","metadata":{}},{"cell_type":"markdown","source":"with strategy.scope():\n    pretrained_model = efficientnet.EfficientNetB7(\n        weights=\"noisy-student\", include_top=False, input_shape=(512, 512, 3)\n    )\n    pretrained_model.trainable = True  # transfer learning\n\n    model2 = tf.keras.Sequential(\n        [\n            pretrained_model,\n            # tf.keras.layers.BatchNormalization(),\n            # tf.keras.layers.Dropout(0.2),\n            tf.keras.layers.Dense(2048, activation=\"relu\"),\n            tf.keras.layers.Dropout(0.2),\n            tf.keras.layers.GlobalAveragePooling2D(),\n            tf.keras.layers.Dense(104, activation=\"softmax\"),\n        ]\n    )\nmodel2.summary()\n\nmodel2.compile(\n    optimizer=\"adam\",\n    loss=\"sparse_categorical_crossentropy\",\n    metrics=[\"sparse_categorical_accuracy\"],\n)\n\nhistorical = model2.fit(\n    training_dataset,\n    steps_per_epoch=STEPS_PER_EPOCH,\n    epochs=EPOCHS,\n    validation_data=validation_dataset,\n    callbacks=callbacks,\n)\n\ndisplay_training_curves(\n    historical.history[\"loss\"],\n    historical.history[\"val_loss\"],\n    \"loss\",\n    211,\n)\n\ndisplay_training_curves(\n    historical.history[\"sparse_categorical_accuracy\"],\n    historical.history[\"val_sparse_categorical_accuracy\"],\n    \"accuracy\",\n    212,\n)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:32:29.934139Z","iopub.status.idle":"2022-11-09T22:32:29.934568Z","shell.execute_reply.started":"2022-11-09T22:32:29.934348Z","shell.execute_reply":"2022-11-09T22:32:29.934371Z"}}},{"cell_type":"markdown","source":"model2.save(\"/kaggle/working/model_effnetb7.h5\", save_format=\"h5\")","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:32:29.936361Z","iopub.status.idle":"2022-11-09T22:32:29.937409Z","shell.execute_reply.started":"2022-11-09T22:32:29.937169Z","shell.execute_reply":"2022-11-09T22:32:29.937197Z"}}},{"cell_type":"markdown","source":"# EfficientNetV2L","metadata":{}},{"cell_type":"markdown","source":"with strategy.scope():\n    modelv2l = tf.keras.models.load_model(\n        \"/kaggle/input/effnetv2l/efficientnetv2-l-imagenet.h5\", compile=True\n    )\n    modelv2l.trainable = False\n    model = tf.keras.Sequential(\n        [\n            modelv2l,\n            # tf.keras.layers.BatchNormalization(),\n            # tf.keras.layers.Dropout(0.2),\n            # tf.keras.layers.Reshape((10,10,10)),\n            # tf.keras.layers.GlobalAveragePooling2D(),\n            tf.keras.layers.Dense(104, activation=\"softmax\"),\n        ]\n    )\nmodel.summary()\n\nmodel.compile(\n    optimizer=\"adam\",\n    loss=\"sparse_categorical_crossentropy\",\n    metrics=[\"sparse_categorical_accuracy\"],\n)\n\nhistorical = model.fit(\n    training_dataset,\n    steps_per_epoch=STEPS_PER_EPOCH,\n    epochs=EPOCHS,\n    validation_data=validation_dataset,\n    callbacks=callbacks,\n)\n\ndisplay_training_curves(\n    historical.history[\"loss\"],\n    historical.history[\"val_loss\"],\n    \"loss\",\n    211,\n)\n\ndisplay_training_curves(\n    historical.history[\"sparse_categorical_accuracy\"],\n    historical.history[\"val_sparse_categorical_accuracy\"],\n    \"accuracy\",\n    212,\n)","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import f1_score\nNUM_VALIDATION_IMAGES = int(count_data_items(VAL_FILENAMES))\ncmdataset = get_validation_dataset(ordered=True) # since we are splitting the dataset and iterating separately on images and labels, order matters.\nimages_ds = cmdataset.map(lambda image, label: image)\nlabels_ds = cmdataset.map(lambda image, label: label).unbatch()\ncm_correct_labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES))).numpy() # get everything as one batch\nm1 = model1.predict(images_ds)\nm2 = model3.predict(images_ds)\nscores = []\nfor alpha in np.linspace(0,1,100):\n    cm_probabilities = alpha*m1+(1-alpha)*m2\n    cm_predictions = np.argmax(cm_probabilities, axis=-1)\n    scores.append(f1_score(cm_correct_labels, cm_predictions, labels=range(104), average='macro'))\n\nbest_alpha = np.argmax(scores)/100\nprint('Best alpha: ' + str(best_alpha))","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:32:29.938738Z","iopub.status.idle":"2022-11-09T22:32:29.939483Z","shell.execute_reply.started":"2022-11-09T22:32:29.939253Z","shell.execute_reply":"2022-11-09T22:32:29.939278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predictions\n\nThis/kaggle/create a file that can be submitted to the competition.","metadata":{}},{"cell_type":"code","source":"test_ds = get_test_dataset(\n    ordered=True\n)  # since we are splitting the dataset and iterating separately on images and ids, order matters.\n\nprint(\"Computing predictions...\")\ntest_images_ds = test_ds.map(lambda image, idnum: image)\nprobs1 = model1.predict(test_images_ds)\nprobs2 = model3.predict(test_images_ds)\nprobabilities = best_alpha*probs1 + (1-best_alpha)*probs2\npredictions = np.argmax(probabilities, axis=-1)\nprint(predictions)\n\nprint(\"Generating submission.csv file...\")\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = (\n    next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype(\"U\")\n)  # all in one batch\nnp.savetxt(\n    \"submission.csv\",\n    np.rec.fromarrays([test_ids, predictions]),\n    fmt=[\"%s\", \"%d\"],\n    delimiter=\",\",\n    header=\"id,label\",\n    comments=\"\",\n)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:32:29.941010Z","iopub.status.idle":"2022-11-09T22:32:29.941506Z","shell.execute_reply.started":"2022-11-09T22:32:29.941263Z","shell.execute_reply":"2022-11-09T22:32:29.941286Z"},"trusted":true},"execution_count":null,"outputs":[]}]}