{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip uninstall tensorflow -y\n!pip install tensorflow==2.7.0\n!pip install cloud-tpu-client\n!pip install livelossplot\n\nimport numpy as np \nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport os\nimport re\nimport glob\n\nimport tensorflow as tf\nprint(\"Tensorflow version \" + tf.__version__)\nimport tensorflow_datasets as tfds\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.layers import Input, Conv2D, MaxPooling2D, Flatten, Dense, BatchNormalization, GlobalAveragePooling2D, Dropout, AveragePooling2D, CenterCrop\nfrom tensorflow.keras.models import Model, Sequential\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\nfrom tensorflow.keras.optimizers import Adam, RMSprop\nfrom tensorflow.keras.regularizers import L2, L1, L1L2\nfrom tensorflow.keras.metrics import TopKCategoricalAccuracy, top_k_categorical_accuracy, SparseTopKCategoricalAccuracy, SparseCategoricalAccuracy\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.applications import EfficientNetB0, EfficientNetB7\nfrom tensorflow.keras.applications.efficientnet import preprocess_input\nfrom kaggle_datasets import KaggleDatasets\nfrom cloud_tpu_client import Client\nClient().configure_tpu_version(tf.version.VERSION, restart_type='ifNeeded')\nAUTO = tf.data.experimental.AUTOTUNE\n\nfrom livelossplot import PlotLossesKerasTF\nfrom livelossplot.outputs import MatplotlibPlot\nfrom skimage.io import imread, imshow\nfrom sklearn.metrics import accuracy_score","metadata":{"execution":{"iopub.status.busy":"2021-11-29T10:37:38.204561Z","iopub.execute_input":"2021-11-29T10:37:38.204988Z","iopub.status.idle":"2021-11-29T10:39:22.873932Z","shell.execute_reply.started":"2021-11-29T10:37:38.204878Z","shell.execute_reply":"2021-11-29T10:39:22.87313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:  # detect TPUs\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.cluster_spec().as_dict()['worker'])\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\n\nexcept ValueError:  # detect GPUs\n    raise BaseException(\n        'ERROR: Not connected to a TPU runtime; switching to GPU strategy')\n    strategy = tf.distribute.MirroredStrategy()  # for GPU or multi-GPU machines\n    # default strategy that works on CPU and single GPU\n    strategy = tf.distribute.get_strategy()\n    # for clusters of multi-GPU machines\n    strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy()\n\nprint(\"Number of accelerators: \", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2021-11-29T10:39:22.875677Z","iopub.execute_input":"2021-11-29T10:39:22.875903Z","iopub.status.idle":"2021-11-29T10:39:45.65628Z","shell.execute_reply.started":"2021-11-29T10:39:22.875878Z","shell.execute_reply":"2021-11-29T10:39:45.655461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMAGE_SIZE = [512, 512]\n\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\n\n\nGCS_DS_PATH = KaggleDatasets().get_gcs_path('tpu-getting-started')\nGCS_PATH_SELECT = {  # available image sizes\n    192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',\n    224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',\n    331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',\n    512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'\n}\nGCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]\n\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')\nVALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\n# predictions on this dataset should be submitted for the competition\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec')\n\nCLASSES = ['pink primrose',    'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea',     'wild geranium',     'tiger lily',           'moon orchid',              'bird of paradise', 'monkshood',        'globe thistle',         # 00 - 09\n           'snapdragon',       \"colt's foot\",               'king protea',      'spear thistle', 'yellow iris',       'globe-flower',         'purple coneflower',        'peruvian lily',    'balloon flower',   'giant white arum lily',  # 10 - 19\n           'fire lily',        'pincushion flower',         'fritillary',       'red ginger',    'grape hyacinth',    'corn poppy',           'prince of wales feathers', 'stemless gentian', 'artichoke',        'sweet william',         # 20 - 29\n           'carnation',        'garden phlox',              'love in the mist', 'cosmos',        'alpine sea holly',  'ruby-lipped cattleya', 'cape flower',              'great masterwort', 'siam tulip',       'lenten rose',           # 30 - 39\n           'barberton daisy',  'daffodil',                  'sword lily',       'poinsettia',    'bolero deep blue',  'wallflower',           'marigold',                 'buttercup',        'daisy',            'common dandelion',      # 40 - 49\n           'petunia',          'wild pansy',                'primula',          'sunflower',     'lilac hibiscus',    'bishop of llandaff',   'gaura',                    'geranium',         'orange dahlia',    'pink-yellow dahlia',    # 50 - 59\n           'cautleya spicata', 'japanese anemone',          'black-eyed susan', 'silverbush',    'californian poppy', 'osteospermum',         'spring crocus',            'iris',             'windflower',       'tree poppy',            # 60 - 69\n           'gazania',          'azalea',                    'water lily',       'rose',          'thorn apple',       'morning glory',        'passion flower',           'lotus',            'toad lily',        'anthurium',             # 70 - 79\n           'frangipani',       'clematis',                  'hibiscus',         'columbine',     'desert-rose',       'tree mallow',          'magnolia',                 'cyclamen ',        'watercress',       'canna lily',            # 80 - 89\n           'hippeastrum ',     'bee balm',                  'pink quill',       'foxglove',      'bougainvillea',     'camellia',             'mallow',                   'mexican petunia',  'bromelia',         'blanket flower',        # 90 - 99\n           'trumpet creeper',  'blackberry lily',           'common tulip',     'wild rose']                                                                                                                                               # 100 - 102","metadata":{"execution":{"iopub.status.busy":"2021-11-29T10:39:45.658056Z","iopub.execute_input":"2021-11-29T10:39:45.65833Z","iopub.status.idle":"2021-11-29T10:39:46.260051Z","shell.execute_reply.started":"2021-11-29T10:39:45.658276Z","shell.execute_reply":"2021-11-29T10:39:46.259247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1))\n         for filename in filenames]\n    return np.sum(n)\n\n\nNUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\n# The \"-(-//)\" trick rounds up instead of down :-)\nVALIDATION_STEPS = -(-NUM_VALIDATION_IMAGES // BATCH_SIZE)\n# The \"-(-//)\" trick rounds up instead of down :-)\nTEST_STEPS = -(-NUM_TEST_IMAGES // BATCH_SIZE)\nprint('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(\n    NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))","metadata":{"execution":{"iopub.status.busy":"2021-11-29T10:39:46.261263Z","iopub.execute_input":"2021-11-29T10:39:46.261577Z","iopub.status.idle":"2021-11-29T10:39:46.273515Z","shell.execute_reply.started":"2021-11-29T10:39:46.261539Z","shell.execute_reply":"2021-11-29T10:39:46.272529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# code from https://www.kaggle.com/zephyr44/getting-started-with-100-flowers-on-tpu/edit\n\ndef decode_image(image_data):\n    # image format uint8 [0,255]\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])  # explicit size needed for TPU\n    return image\n\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        # tf.string means bytestring\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        # shape [] means single element\n        \"class\": tf.io.FixedLenFeature([], tf.int64),\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['class'], tf.int32)\n    return image, label  # returns a dataset of (image, label) pairs\n\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        # tf.string means bytestring\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        # shape [] means single element\n        \"id\": tf.io.FixedLenFeature([], tf.string),\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['id']\n    return image, idnum  # returns a dataset of image(s)","metadata":{"execution":{"iopub.status.busy":"2021-11-29T10:39:46.275965Z","iopub.execute_input":"2021-11-29T10:39:46.276572Z","iopub.status.idle":"2021-11-29T10:39:46.286976Z","shell.execute_reply.started":"2021-11-29T10:39:46.276539Z","shell.execute_reply":"2021-11-29T10:39:46.286326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reference: https://github.com/facebookresearch/FixRes/blob/main/transforms_v2.py\ndef processing(image, label, smaller_size, bigger_size, train=True, finetuning=False, fixres=True):\n    \"\"\"\n    Processing for the first stage of FixRes procedure\n        For training : random crop -> random horizontal flip -> resize to a smaller size \n    Processing for the second stage of FixRes procedure = Fine-Tuning : resize and center_crop to the normal size\n    \"\"\"\n    ratio = int(256 / 224)\n    if not finetuning:\n        if train:\n            channels = 3\n            begin, size, _ = tf.image.sample_distorted_bounding_box(\n                tf.shape(image),\n                tf.zeros([0, 0, 4], tf.float32),\n                area_range=(0.2, 1.0),\n                min_object_covered=0,\n                use_image_if_no_bounding_boxes=True,\n            )\n            image = tf.slice(image, begin, size)\n            image.set_shape([None, None, channels])\n            image = tf.image.resize(image, [smaller_size, smaller_size])\n            image = tf.image.random_flip_left_right(image)\n            image = tf.image.random_saturation(image, 0.7, 1.3)\n            image = tf.image.random_brightness(image, 0.3)\n        if fixres:\n            image = tf.image.resize(\n                image, [ratio*smaller_size, ratio*smaller_size])\n            image = layers.CenterCrop(\n                smaller_size, smaller_size)(image[None, ...])[0]\n        elif not fixres:\n            image = tf.image.resize(\n                image, [ratio*bigger_size, ratio*bigger_size])\n            image = layers.CenterCrop(\n                bigger_size, bigger_size)(image[None, ...])[0]\n\n    elif finetuning:\n        # if train:\n        #    image = tf.image.random_flip_left_right(image)\n        image = tf.image.resize(image, [ratio*bigger_size, ratio*bigger_size])\n        image = layers.CenterCrop(\n            bigger_size, bigger_size)(image[None, ...])[0]\n\n    return image, label\n\n\ndef generate_augmented_dataset(filenames, smaller_size, bigger_size, train=True, finetuning=False, fixres=True, labeled=True):\n    ignore_order = tf.data.Options()\n    if labeled:\n        ignore_order.experimental_deterministic = False  # disable order, increase speed\n    # automatically interleaves reads from multiple files\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n    # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(\n        read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    dataset = dataset.map(\n        lambda x, y: processing(\n            x, y, smaller_size, bigger_size, train, finetuning, fixres),\n        num_parallel_calls=AUTO)\n    if train:\n        dataset = dataset.repeat()  # the training dataset must repeat for several epochs\n        dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    # prefetch next batch while training (autotune prefetch buffer size)\n    dataset = dataset.prefetch(AUTO)\n\n    return dataset","metadata":{"execution":{"iopub.status.busy":"2021-11-29T10:39:46.288235Z","iopub.execute_input":"2021-11-29T10:39:46.288887Z","iopub.status.idle":"2021-11-29T10:39:46.306225Z","shell.execute_reply.started":"2021-11-29T10:39:46.28886Z","shell.execute_reply":"2021-11-29T10:39:46.305509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"smaller_size = int(IMAGE_SIZE[0]/2)\nbigger_size = IMAGE_SIZE[0]\n\nfixres_train_data = generate_augmented_dataset(\n    TRAINING_FILENAMES, smaller_size, bigger_size, train=True, finetuning=False)\nfixres_val_data = generate_augmented_dataset(\n    VALIDATION_FILENAMES, smaller_size, bigger_size, train=False, finetuning=False)\n\nfinetune_train_data = generate_augmented_dataset(\n    TRAINING_FILENAMES, smaller_size, bigger_size, train=True, finetuning=True)\nfinetune_val_data = generate_augmented_dataset(\n    VALIDATION_FILENAMES, smaller_size, bigger_size, train=False, finetuning=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-29T10:39:46.307331Z","iopub.execute_input":"2021-11-29T10:39:46.308107Z","iopub.status.idle":"2021-11-29T10:39:47.266907Z","shell.execute_reply.started":"2021-11-29T10:39:46.308076Z","shell.execute_reply":"2021-11-29T10:39:47.266047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_model(input_shape, num_classes=len(CLASSES), lr=1e-4):\n    with strategy.scope():\n        inputs = Input((input_shape, input_shape, 3))\n        # On récupère les couches du modèle EfficientNet\n        efficientnet = EfficientNetB7(weights='imagenet', include_top=False)\n\n        # On freeze les couches pré-entrainés excepté les\n\n        print(efficientnet.layers[-11:])\n        for layer in efficientnet.layers[:-11]:\n            layer.trainable = False\n        for layer in efficientnet.layers[-11:]:\n            layer.trainable = True\n\n        # Régularisation L2\n        regularizer = L2()\n        for layer in efficientnet.layers:\n            for attr in ['kernel_regularizer']:\n                if hasattr(layer, attr):\n                    setattr(layer, attr, regularizer)\n\n        x = preprocess_input(inputs)\n        x = efficientnet(x)\n\n        # On ajoute les dernières couches\n        top_layers = Dropout(0.5)(x)\n        top_layers = GlobalAveragePooling2D()(top_layers)\n        top_layers = Dropout(0.5)(top_layers)\n        output_top_layers = Dense(\n            num_classes, activation='softmax')(top_layers)\n\n        model = Model(inputs=inputs, outputs=output_top_layers)\n\n        model.compile(\n            optimizer=keras.optimizers.Adam(learning_rate=lr),\n            loss=\"sparse_categorical_crossentropy\",\n            metrics=[\"sparse_categorical_accuracy\",\n                     SparseTopKCategoricalAccuracy(k=3)],\n            # steps_per_execution=16\n        )\n\n        return model","metadata":{"execution":{"iopub.status.busy":"2021-11-29T10:39:47.268026Z","iopub.execute_input":"2021-11-29T10:39:47.268253Z","iopub.status.idle":"2021-11-29T10:39:47.27846Z","shell.execute_reply.started":"2021-11-29T10:39:47.268227Z","shell.execute_reply":"2021-11-29T10:39:47.277843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_and_evaluate(model, name, train_data, val_data,  epochs):\n    reduce_lr = ReduceLROnPlateau(\n        monitor='val_loss', factor=0.1, patience=3, min_lr=0.00001, min_delta=0.001)\n    early_stopping_monitor = EarlyStopping(\n        monitor='val_loss', patience=6, min_delta=0, verbose=1, mode='auto')\n    checkpoint = ModelCheckpoint(\n        filepath=f'{name}.weights.best.hdf5', save_best_only=True)\n    # outputs=[MatplotlibPlot(figpath =f'{model_name}.png')]\n    plot = PlotLossesKerasTF()\n    callbacks = [reduce_lr, early_stopping_monitor, plot, checkpoint]\n\n    model.fit(train_data,\n              validation_data=val_data,\n              epochs=epochs,\n              steps_per_epoch=STEPS_PER_EPOCH,\n              validation_steps=VALIDATION_STEPS,\n              callbacks=callbacks\n              )\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-11-29T10:39:47.279583Z","iopub.execute_input":"2021-11-29T10:39:47.27998Z","iopub.status.idle":"2021-11-29T10:39:47.295522Z","shell.execute_reply.started":"2021-11-29T10:39:47.279953Z","shell.execute_reply":"2021-11-29T10:39:47.294712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# FixRes Model","metadata":{}},{"cell_type":"code","source":"model_name = 'Flower_Classification_wFixRes'\nmodel = build_model(smaller_size)\nprint(model.summary())","metadata":{"execution":{"iopub.status.busy":"2021-11-29T10:39:47.296608Z","iopub.execute_input":"2021-11-29T10:39:47.296808Z","iopub.status.idle":"2021-11-29T10:40:52.795106Z","shell.execute_reply.started":"2021-11-29T10:39:47.296784Z","shell.execute_reply":"2021-11-29T10:40:52.794184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = train_and_evaluate(\n    model, model_name, fixres_train_data, fixres_val_data, epochs=50)","metadata":{"execution":{"iopub.status.busy":"2021-11-29T10:40:52.796587Z","iopub.execute_input":"2021-11-29T10:40:52.796911Z","iopub.status.idle":"2021-11-29T10:57:24.647838Z","shell.execute_reply.started":"2021-11-29T10:40:52.79687Z","shell.execute_reply":"2021-11-29T10:57:24.646834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_name = 'Flower_Classification_wFixRes_finetune'\nfinetune_model = build_model(bigger_size, lr=1e-5)\nfinetune_model.load_weights(f'Flower_Classification_wFixRes.weights.best.hdf5')\n\nfor layer in finetune_model.layers[1].layers[:-2]:\n    layer.trainable = False\nfor layer in finetune_model.layers[1].layers[-2:]:\n    layer.trainable = True\nfor layer in finetune_model.layers[2:7]:\n    layer.trainable = True\n\nprint(finetune_model.summary())","metadata":{"execution":{"iopub.status.busy":"2021-11-29T10:57:24.650678Z","iopub.execute_input":"2021-11-29T10:57:24.651019Z","iopub.status.idle":"2021-11-29T10:58:21.335234Z","shell.execute_reply.started":"2021-11-29T10:57:24.650959Z","shell.execute_reply":"2021-11-29T10:58:21.334148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Use a lower learning rate during fine-tuning.\nfinetune_model = train_and_evaluate(\n    finetune_model,\n    model_name,\n    finetune_train_data,\n    finetune_val_data,\n    epochs=100\n)","metadata":{"execution":{"iopub.status.busy":"2021-11-29T10:58:21.336748Z","iopub.execute_input":"2021-11-29T10:58:21.336975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = generate_augmented_dataset(\n    TEST_FILENAMES, smaller_size, bigger_size, train=False, finetuning=True, fixres=True, labeled=False)\n\nprint('Computing predictions...')\ntest_images_ds = test_data.map(lambda image, idnum: image)\nprobabilities = finetune_model.predict(test_images_ds)\npredictions = np.argmax(probabilities, axis=-1)\nprint(predictions)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Generating submission.csv file...')\n\n# Get image ids from test set and convert to unicode\ntest_ids_ds = test_data.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U')\n\n# Write the submission file\nnp.savetxt(\n    'submission.csv',\n    np.rec.fromarrays([test_ids, predictions]),\n    fmt=['%s', '%d'],\n    delimiter=',',\n    header='id,label',\n    comments='',\n)\n\n# Look at the first few predictions\n!head submission.csv","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}